diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index 78ad98c7d..941789844 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -4,16 +4,17 @@ on: [push, pull_request] jobs: check: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 env: CLANG_VERSION: 10 steps: - uses: actions/checkout@v2 - name: Install clang-format run: | + codename=$( lsb_release --codename --short ) sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null < +# +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) +################################################################################ + +include(CheckCXXSymbolExists) + +if(WIN32) + check_cxx_symbol_exists("_M_AMD64" "" SOCI_TARGET_ARCH_X64) + if(NOT RTC_ARCH_X64) + check_cxx_symbol_exists("_M_IX86" "" SOCI_TARGET_ARCH_X86) + endif(NOT RTC_ARCH_X64) + # add check for arm here + # see http://msdn.microsoft.com/en-us/library/b0084kay.aspx +else(WIN32) + check_cxx_symbol_exists("__i386__" "" SOCI_TARGET_ARCH_X86) + check_cxx_symbol_exists("__x86_64__" "" SOCI_TARGET_ARCH_X64) + check_cxx_symbol_exists("__arm__" "" SOCI_TARGET_ARCH_ARM) +endif(WIN32) + +if(NOT DEFINED LIB_SUFFIX) + if(SOCI_TARGET_ARCH_X64) + set(_lib_suffix "64") + else() + set(_lib_suffix "") + endif() + set(LIB_SUFFIX ${_lib_suffix} CACHE STRING "Specifies suffix for the lib directory") +endif() + +# +# C++11 Option +# + +if(NOT SOCI_CXX_C11) + set (SOCI_CXX_C11 OFF CACHE BOOL "Build to the C++11 standard") +endif() + +# +# Force compilation flags and set desired warnings level +# + +if (MSVC) + add_definitions(-D_CRT_SECURE_NO_DEPRECATE) + add_definitions(-D_CRT_SECURE_NO_WARNINGS) + add_definitions(-D_CRT_NONSTDC_NO_WARNING) + add_definitions(-D_SCL_SECURE_NO_WARNINGS) + + if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]") + string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /we4266") + endif() + +else() + + set(SOCI_GCC_CLANG_COMMON_FLAGS "") + # "-pedantic -Werror -Wno-error=parentheses -Wall -Wextra -Wpointer-arith -Wcast-align -Wcast-qual -Wfloat-equal -Woverloaded-virtual -Wredundant-decls -Wno-long-long") + + + if (SOCI_CXX_C11) + set(SOCI_CXX_VERSION_FLAGS "-std=c++11") + else() + set(SOCI_CXX_VERSION_FLAGS "-std=gnu++98") + endif() + + if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" OR "${CMAKE_CXX_COMPILER}" MATCHES "clang") + + if(NOT CMAKE_CXX_COMPILER_VERSION LESS 3.1 AND SOCI_ASAN) + set(SOCI_GCC_CLANG_COMMON_FLAGS "${SOCI_GCC_CLANG_COMMON_FLAGS} -fsanitize=address") + endif() + + # enforce C++11 for Clang + set(SOCI_CXX_C11 ON) + set(SOCI_CXX_VERSION_FLAGS "-std=c++11") + add_definitions(-DCATCH_CONFIG_CPP11_NO_IS_ENUM) + + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SOCI_GCC_CLANG_COMMON_FLAGS} ${SOCI_CXX_VERSION_FLAGS}") + + elseif(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX) + + if(NOT CMAKE_CXX_COMPILER_VERSION LESS 4.8 AND SOCI_ASAN) + set(SOCI_GCC_CLANG_COMMON_FLAGS "${SOCI_GCC_CLANG_COMMON_FLAGS} -fsanitize=address") + endif() + + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SOCI_GCC_CLANG_COMMON_FLAGS} ${SOCI_CXX_VERSION_FLAGS} ") + if (CMAKE_COMPILER_IS_GNUCXX) + if (CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-variadic-macros") + endif() + endif() + + else() + message(WARNING "Unknown toolset - using default flags to build SOCI") + endif() + +endif() + +# Set SOCI_HAVE_* variables for soci-config.h generator +set(SOCI_HAVE_CXX_C11 ${SOCI_CXX_C11} CACHE INTERNAL "Enables C++11 support") diff --git a/Builds/CMake/deps/Postgres.cmake b/Builds/CMake/deps/Postgres.cmake index 7ad7f169a..bb94832a4 100644 --- a/Builds/CMake/deps/Postgres.cmake +++ b/Builds/CMake/deps/Postgres.cmake @@ -13,7 +13,7 @@ if(reporting) ExternalProject_Add(postgres_src PREFIX ${nih_cache_path} GIT_REPOSITORY https://github.com/postgres/postgres.git - GIT_TAG master + GIT_TAG REL_14_5 CONFIGURE_COMMAND ./configure --without-readline > /dev/null BUILD_COMMAND ${CMAKE_COMMAND} -E env --unset=MAKELEVEL make UPDATE_COMMAND "" diff --git a/Builds/CMake/deps/Soci.cmake b/Builds/CMake/deps/Soci.cmake index fa05a1157..d165d6e1f 100644 --- a/Builds/CMake/deps/Soci.cmake +++ b/Builds/CMake/deps/Soci.cmake @@ -51,7 +51,8 @@ else() # This patch process is likely fragile and should be reviewed carefully # whenever we update the GIT_TAG above. PATCH_COMMAND - ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/soci_patch.cmake + ${CMAKE_COMMAND} -D RIPPLED_SOURCE=${CMAKE_CURRENT_SOURCE_DIR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/soci_patch.cmake CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} diff --git a/Builds/CMake/deps/cassandra.cmake b/Builds/CMake/deps/cassandra.cmake index 8f1e799dc..4563a3413 100644 --- a/Builds/CMake/deps/cassandra.cmake +++ b/Builds/CMake/deps/cassandra.cmake @@ -11,7 +11,7 @@ if(reporting) ExternalProject_Add(zlib_src PREFIX ${nih_cache_path} GIT_REPOSITORY https://github.com/madler/zlib.git - GIT_TAG master + GIT_TAG v1.2.12 INSTALL_COMMAND "" BUILD_BYPRODUCTS /${ep_lib_prefix}z.a LOG_BUILD TRUE @@ -45,7 +45,7 @@ if(reporting) ExternalProject_Add(krb5_src PREFIX ${nih_cache_path} GIT_REPOSITORY https://github.com/krb5/krb5.git - GIT_TAG master + GIT_TAG krb5-1.20-final UPDATE_COMMAND "" CONFIGURE_COMMAND autoreconf src && CFLAGS=-fcommon ./src/configure --enable-static --disable-shared > /dev/null BUILD_IN_SOURCE 1 @@ -80,7 +80,7 @@ if(reporting) ExternalProject_Add(libuv_src PREFIX ${nih_cache_path} GIT_REPOSITORY https://github.com/libuv/libuv.git - GIT_TAG v1.x + GIT_TAG v1.44.2 INSTALL_COMMAND "" BUILD_BYPRODUCTS /${ep_lib_prefix}uv_a.a LOG_BUILD TRUE @@ -106,7 +106,7 @@ if(reporting) ExternalProject_Add(cassandra_src PREFIX ${nih_cache_path} GIT_REPOSITORY https://github.com/datastax/cpp-driver.git - GIT_TAG master + GIT_TAG 2.16.2 CMAKE_ARGS -DLIBUV_ROOT_DIR=${BINARY_DIR} -DLIBUV_LIBARY=${BINARY_DIR}/libuv_a.a diff --git a/Builds/CMake/soci_patch.cmake b/Builds/CMake/soci_patch.cmake index 57c46e582..0c2a75c0d 100644 --- a/Builds/CMake/soci_patch.cmake +++ b/Builds/CMake/soci_patch.cmake @@ -2,6 +2,16 @@ # so as to remove type range check exceptions that cause # us trouble when using boost::optional to select int values +# Soci's CMake setup leaves flags in place that will cause warnings to +# be treated as errors, but some compiler versions throw "new" warnings +# that then cause the build to fail. Simplify that until soci fixes +# those warnings. +if (RIPPLED_SOURCE) + execute_process( COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${RIPPLED_SOURCE}/Builds/CMake/SociConfig.cmake.patched + cmake/SociConfig.cmake ) +endif () + # Some versions of CMake erroneously patch external projects on every build. # If the patch makes no changes, skip it. This workaround can be # removed once we stop supporting vulnerable versions of CMake. diff --git a/Builds/VisualStudio2017/CMakeSettings-example.json b/Builds/VisualStudio2019/CMakeSettings-example.json similarity index 91% rename from Builds/VisualStudio2017/CMakeSettings-example.json rename to Builds/VisualStudio2019/CMakeSettings-example.json index b2889ddf5..b90bfce6b 100644 --- a/Builds/VisualStudio2017/CMakeSettings-example.json +++ b/Builds/VisualStudio2019/CMakeSettings-example.json @@ -3,7 +3,7 @@ "configurations": [ { "name": "x64-Debug", - "generator": "Visual Studio 15 2017 Win64", + "generator": "Visual Studio 16 2019", "configurationType": "Debug", "inheritEnvironments": [ "msvc_x64_x64" ], "buildRoot": "${thisFileDir}\\build\\${name}", @@ -23,7 +23,7 @@ }, { "name": "x64-Release", - "generator": "Visual Studio 15 2017 Win64", + "generator": "Visual Studio 16 2019", "configurationType": "Release", "inheritEnvironments": [ "msvc_x64_x64" ], "buildRoot": "${thisFileDir}\\build\\${name}", diff --git a/Builds/VisualStudio2017/README.md b/Builds/VisualStudio2019/README.md similarity index 86% rename from Builds/VisualStudio2017/README.md rename to Builds/VisualStudio2019/README.md index ec8fb1c08..e369eac67 100644 --- a/Builds/VisualStudio2017/README.md +++ b/Builds/VisualStudio2019/README.md @@ -1,4 +1,4 @@ -# Visual Studio 2017 Build Instructions +# Visual Studio 2019 Build Instructions ## Important @@ -14,26 +14,26 @@ need these software components | Component | Minimum Recommended Version | |-----------|-----------------------| -| [Visual Studio 2017](README.md#install-visual-studio-2017)| 15.5.4 | +| [Visual Studio 2019](README.md#install-visual-studio-2019)| 15.5.4 | | [Git for Windows](README.md#install-git-for-windows)| 2.16.1 | | [OpenSSL Library](README.md#install-openssl) | 1.1.1L | | [Boost library](README.md#build-boost) | 1.70.0 | | [CMake for Windows](README.md#optional-install-cmake-for-windows)* | 3.12 | -\* Only needed if not using the integrated CMake in VS 2017 and prefer generating dedicated project/solution files. +\* Only needed if not using the integrated CMake in VS 2019 and prefer generating dedicated project/solution files. ## Install Software -### Install Visual Studio 2017 +### Install Visual Studio 2019 If not already installed on your system, download your choice of installer from -the [Visual Studio 2017 +the [Visual Studio 2019 Download](https://www.visualstudio.com/downloads/download-visual-studio-vs) page, run the installer, and follow the directions. **You may need to choose the `Desktop development with C++` workload to install all necessary C++ features.** -Any version of Visual Studio 2017 may be used to build rippled. The **Visual -Studio 2017 Community** edition is available free of charge (see [the product +Any version of Visual Studio 2019 may be used to build rippled. The **Visual +Studio 2019 Community** edition is available free of charge (see [the product page](https://www.visualstudio.com/products/visual-studio-community-vs) for licensing details), while paid editions may be used for an initial free-trial period. @@ -55,7 +55,7 @@ OpenSSL.](http://slproweb.com/products/Win32OpenSSL.html) There will several `Win64` bit variants available, you want the non-light `v1.1` line. As of this writing, you **should** select -* Win64 OpenSSL v1.1.1L +* Win64 OpenSSL v1.1.1q and should **not** select @@ -82,11 +82,11 @@ to get the correct 32-/64-bit variant. Boost 1.70 or later is required. -After [downloading boost](http://www.boost.org/users/download/) and unpacking it -to `c:\lib`. As of this writing, the most recent version of boost is 1.70.0, -which will unpack into a directory named `boost_1_70_0`. We recommended either +[Download boost](http://www.boost.org/users/download/) and unpack it +to `c:\lib`. As of this writing, the most recent version of boost is 1.80.0, +which will unpack into a directory named `boost_1_80_0`. We recommended either renaming this directory to `boost`, or creating a junction link `mklink /J boost -boost_1_70_0`, so that you can more easily switch between versions. +boost_1_80_0`, so that you can more easily switch between versions. Next, open **Developer Command Prompt** and type the following commands @@ -101,11 +101,11 @@ is not affected by changes in outside files. Therefore, it is necessary to build the required boost static libraries using this command: ```powershell -bjam -j --toolset=msvc-14.1 address-model=64 architecture=x86 link=static threading=multi runtime-link=shared,static stage +b2 -j --toolset=msvc-14.2 address-model=64 architecture=x86 link=static threading=multi runtime-link=shared,static stage ``` where you should replace `` with the number of parallel -invocations to use build, e.g. `bjam -j4 ...` would use up to 4 concurrent build +invocations to use build, e.g. `bjam -j8 ...` would use up to 8 concurrent build shell commands for the build. Building the boost libraries may take considerable time. When the build process @@ -115,7 +115,7 @@ library paths as they will be required later. ### (Optional) Install CMake for Windows [CMake](http://cmake.org) is a cross platform build system generator. Visual -Studio 2017 includes an integrated version of CMake that avoids having to +Studio 2019 includes an integrated version of CMake that avoids having to manually run CMake, but it is undergoing continuous improvement. Users that prefer to use standard Visual Studio project and solution files need to install a dedicated version of CMake to generate them. The latest version can be found @@ -141,7 +141,7 @@ repository and optionally switch to the *master* branch. Type the following at the bash prompt: ```powershell -git clone git@github.com:ripple/rippled.git +git clone git@github.com:XRPLF/rippled.git cd rippled ``` If you receive an error about not having the "correct access rights" make sure @@ -160,7 +160,7 @@ To test the latest release candidate, choose the `release` branch. git checkout release ``` -If you are doing development work and want the latest set of untested features, +If you are doing development work and want the latest set of beta features, you can consider using the `develop` branch instead. ``` @@ -177,14 +177,14 @@ To begin, simply: cloned rippled folder. 2. Right-click on `CMakeLists.txt` in the **Solution Explorer - Folder View** to generate a `CMakeSettings.json` file. A sample settings file is provided - [here](/Builds/VisualStudio2017/CMakeSettings-example.json). Customize the + [here](/Builds/VisualStudio2019/CMakeSettings-example.json). Customize the settings for `BOOST_ROOT`, `OPENSSL_ROOT` to match the install paths if they differ from those in the file. 4. Select either the `x64-Release` or `x64-Debug` configuration from the - **Project Setings** drop-down. This should invoke the built-in CMake project + **Project Settings** drop-down. This should invoke the built-in CMake project generator. If not, you can right-click on the `CMakeLists.txt` file and - choose **Cache | Generate Cache**. -5. Select either the `rippled.exe` (unity) or `rippled_classic.exe` (non-unity) + choose **Configure rippled**. +5. Select the `rippled.exe` option in the **Select Startup Item** drop-down. This will be the target built when you press F7. Alternatively, you can choose a target to build from the top-level **CMake | Build** menu. Note that at this time, there are other @@ -216,9 +216,9 @@ execute the following commands within your `rippled` cloned repository: ``` mkdir build\cmake cd build\cmake -cmake ..\.. -G"Visual Studio 15 2017 Win64" -DBOOST_ROOT="C:\lib\boost_1_70_0" -DOPENSSL_ROOT="C:\lib\OpenSSL-Win64" -DCMAKE_GENERATOR_TOOLSET=host=x64 +cmake ..\.. -G"Visual Studio 16 2019" -Ax64 -DBOOST_ROOT="C:\lib\boost" -DOPENSSL_ROOT="C:\lib\OpenSSL-Win64" -DCMAKE_GENERATOR_TOOLSET=host=x64 ``` -Now launch Visual Studio 2017 and select **File | Open | Project/Solution**. +Now launch Visual Studio 2019 and select **File | Open | Project/Solution**. Navigate to the `build\cmake` folder created above and select the `rippled.sln` file. You can then choose whether to build the `Debug` or `Release` solution configuration. diff --git a/Builds/levelization/results/loops.txt b/Builds/levelization/results/loops.txt index d1838c55c..cb137f497 100644 --- a/Builds/levelization/results/loops.txt +++ b/Builds/levelization/results/loops.txt @@ -14,7 +14,7 @@ Loop: ripple.app ripple.overlay ripple.overlay ~= ripple.app Loop: ripple.app ripple.peerfinder - ripple.peerfinder ~= ripple.app + ripple.app > ripple.peerfinder Loop: ripple.app ripple.rpc ripple.rpc > ripple.app diff --git a/Builds/levelization/results/ordering.txt b/Builds/levelization/results/ordering.txt index ed6b4e57c..401040fc2 100644 --- a/Builds/levelization/results/ordering.txt +++ b/Builds/levelization/results/ordering.txt @@ -113,7 +113,6 @@ test.consensus > ripple.basics test.consensus > ripple.beast test.consensus > ripple.consensus test.consensus > ripple.ledger -test.consensus > ripple.rpc test.consensus > test.csf test.consensus > test.toplevel test.consensus > test.unit_test diff --git a/Builds/linux/README.md b/Builds/linux/README.md index bb5bc7618..15a84a33e 100644 --- a/Builds/linux/README.md +++ b/Builds/linux/README.md @@ -7,10 +7,11 @@ the [rippled-package-builder](https://github.com/ripple/rippled-package-builder) repository. Note: Ubuntu 16.04 users may need to update their compiler (see the dependencies -section). For non Ubuntu distributions, the steps below should work be +section). For non Ubuntu distributions, the steps below should work by installing the appropriate dependencies using that distribution's package management tools. + ## Dependencies gcc-8 or later is required. @@ -28,8 +29,6 @@ $ apt-get install -y autoconf flex bison ``` Advanced users can choose to install newer versions of gcc, or the clang compiler. -At this time, rippled only supports protobuf version 2. Using version 3 of -protobuf will give errors. ### Build Boost @@ -42,7 +41,7 @@ $ tar -xzf boost_1_70_0.tar.gz $ cd boost_1_70_0 $ ./bootstrap.sh $ ./b2 headers -$ ./b2 -j +$ ./b2 -j $(echo $(nproc)-2 | bc) ``` ### (Optional) Dependencies for Building Source Documentation @@ -88,8 +87,8 @@ git checkout develop If you didn't persistently set the `BOOST_ROOT` environment variable to the directory in which you compiled boost, then you should set it temporarily. -For example, you built Boost in your home directory `~/boost_1_70_0`, you -would do for any shell in which you want to build: +For example, if you built Boost in your home directory `~/boost_1_70_0`, you +would run the following shell command: ``` export BOOST_ROOT=~/boost_1_70_0 @@ -104,8 +103,8 @@ All builds should be done in a separate directory from the source tree root (a subdirectory is fine). For example, from the root of the ripple source tree: ``` -mkdir my_build -cd my_build +mkdir build +cd build ``` followed by: @@ -153,7 +152,7 @@ Several other infrequently used options are available - run `ccmake` or Once you have generated the build system, you can run the build via cmake: ``` -cmake --build . -- -j +cmake --build . -- -j $(echo $(nproc)-2 | bc) ``` the `-j` parameter in this example tells the build tool to compile several @@ -174,7 +173,7 @@ building, e.g.: ``` cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/opt/local .. -cmake --build . --target install -- -j +cmake --build . --target install -- -j $(echo $(nproc)-2 | bc) ``` We recommend specifying `CMAKE_INSTALL_PREFIX` when configuring in order to @@ -184,7 +183,7 @@ the installation by specifying the `DESTDIR` env variable during the install pha e.g.: ``` -DESTDIR=~/mylibs cmake --build . --target install -- -j +DESTDIR=~/mylibs cmake --build . --target install -- -j $(echo $(nproc)-2 | bc) ``` in which case, the files would be installed in the `CMAKE_INSTALL_PREFIX` within @@ -213,13 +212,13 @@ git submodule add -b master https://github.com/ripple/rippled.git vendor/rippled change the `vendor/rippled` path as desired for your repo layout. Furthermore, change the branch name if you want to track a different rippled branch, such as `develop`. - + Second, to bring this submodule into your project, just add the rippled subdirectory: ``` add_subdirectory (vendor/rippled) ``` - + ##### Option 2: installed rippled + find_package First, follow the "Optional Installation" instructions above to @@ -239,3 +238,32 @@ change the `/opt/local` module path above to match your chosen installation pref `rippled` builds a set of unit tests into the server executable. To run these unit tests after building, pass the `--unittest` option to the compiled `rippled` executable. The executable will exit with summary info after running the unit tests. + +## Workaround for a compile error in soci + +Compilation errors have been observed with Apple Clang 13.1.6+ and soci v4.x. soci compiles with the `-Werror` flag which causes warnings to be treated as errors. These warnings pertain to style (not correctness). However, they cause the cmake process to fail. + +Here's an example of how this looks: +``` +.../rippled/.nih_c/unix_makefiles/AppleClang_13.1.6.13160021/Debug/src/soci/src/core/session.cpp:450:66: note: in instantiation of function template specialization 'soci::use' requested here + return prepare << backEnd_->get_column_descriptions_query(), use(table_name, "t"); + ^ +1 error generated. +``` + +Please apply the below patch (courtesy of Scott Determan) to remove these errors. `.nih_c/unix_makefiles/AppleClang_13.1.6.13160021/Debug/src/soci/cmake/SociConfig.cmake` file needs to be edited. This file is an example for Mac OS and it might be slightly different for other OS/Architectures. + +``` +diff --git a/cmake/SociConfig.cmake b/cmake/SociConfig.cmake +index 97d907e4..11bcd1f3 100644 +--- a/cmake/SociConfig.cmake ++++ b/cmake/SociConfig.cmake +@@ -58,8 +58,8 @@ if (MSVC) + + else() + +- set(SOCI_GCC_CLANG_COMMON_FLAGS +- "-pedantic -Werror -Wno-error=parentheses -Wall -Wextra -Wpointer-arith -Wcast-align -Wcast-qual -Wfloat-equal -Woverloaded-virtual -Wredundant-decls -Wno-long-long") ++ set(SOCI_GCC_CLANG_COMMON_FLAGS "") ++ # "-pedantic -Werror -Wno-error=parentheses -Wall -Wextra -Wpointer-arith -Wcast-align -Wcast-qual -Wfloat-equal -Woverloaded-virtual -Wredundant-decls -Wno-long-long") +``` diff --git a/CMakeLists.txt b/CMakeLists.txt index c5e750d7b..d3b494c10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ endif () project (rippled) set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) # make GIT_COMMIT_HASH define available to all sources @@ -21,6 +21,12 @@ if(Git_FOUND) endif() endif() #git +if (thread_safety_analysis) + add_compile_options(-Wthread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS -DRIPPLE_ENABLE_THREAD_SAFETY_ANNOTATIONS) + add_compile_options("-stdlib=libc++") + add_link_options("-stdlib=libc++") +endif() + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/Builds/CMake/deps") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..06b9d622c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,67 @@ +# Contributing +The XRP Ledger has many and diverse stakeholders, and everyone deserves a chance to contribute meaningful changes to the code that runs the XRPL. +To contribute, please: +1. Fork the repository under your own user. +2. Create a new branch on which to write your changes. Please note that changes which alter transaction processing must be composed via and guarded using [Amendments](https://xrpl.org/amendments.html). Changes which are _read only_ i.e. RPC, or changes which are only refactors and maintain the existing behaviour do not need to be made through an Amendment. +3. Write and test your code. +4. Ensure that your code compiles with the provided build engine and update the provided build engine as part of your PR where needed and where appropriate. +5. Write test cases for your code and include those in `src/test` such that they are runnable from the command line using `./rippled -u`. (Some changes will not be able to be tested this way.) +6. Ensure your code passes automated checks (e.g. clang-format and levelization.) +7. Squash your commits (i.e. rebase) into as few commits as is reasonable to describe your changes at a high level (typically a single commit for a small change.) +8. Open a PR to the main repository onto the _develop_ branch, and follow the provided template. + +# Major Changes +If your code change is a major feature, a breaking change or in some other way makes a significant alteration to the way the XRPL will operate, then you must first write an XLS document (XRP Ledger Standard) describing your change. +To do this: +1. Go to [XLS Standards](https://github.com/XRPLF/XRPL-Standards/discussions). +2. Choose the next available standard number. +3. Open a discussion with the appropriate title to propose your draft standard. +4. Link your XLS in your PR. + +# Style guide +This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent rather than a set of _thou shalt not_ commandments. + +## Formatting +All code must conform to `clang-format` version 10, unless the result would be unreasonably difficult to read or maintain. +To change your code to conform use `clang-format -i `. + +## Avoid +1. Proliferation of nearly identical code. +2. Proliferation of new files and classes. +3. Complex inheritance and complex OOP patterns. +4. Unmanaged memory allocation and raw pointers. +5. Macros and non-trivial templates (unless they add significant value.) +6. Lambda patterns (unless these add significant value.) +7. CPU or architecture-specific code unless there is a good reason to include it, and where it is used guard it with macros and provide explanatory comments. +8. Importing new libraries unless there is a very good reason to do so. + +## Seek to +9. Extend functionality of existing code rather than creating new code. +10. Prefer readability over terseness where important logic is concerned. +11. Inline functions that are not used or are not likely to be used elsewhere in the codebase. +12. Use clear and self-explanatory names for functions, variables, structs and classes. +13. Use TitleCase for classes, structs and filenames, camelCase for function and variable names, lower case for namespaces and folders. +14. Provide as many comments as you feel that a competent programmer would need to understand what your code does. + +# Maintainers +Maintainers are ecosystem participants with elevated access to the repository. They are able to push new code, make decisions on when a release should be made, etc. + +## Code Review +New contributors' PRs must be reviewed by at least two of the maintainers. Well established prior contributors can be reviewed by a single maintainer. + +## Adding and Removing +New maintainers can be proposed by two existing maintainers, subject to a vote by a quorum of the existing maintainers. A minimum of 50% support and a 50% participation is required. In the event of a tie vote, the addition of the new maintainer will be rejected. + +Existing maintainers can resign, or be subject to a vote for removal at the behest of two existing maintainers. A minimum of 60% agreement and 50% participation are required. The XRP Ledger Foundation will have the ability, for cause, to remove an existing maintainer without a vote. + +## Existing Maintainers +* [JoelKatz](https://github.com/JoelKatz) (Ripple) +* [Manojsdoshi](https://github.com/manojsdoshi) (Ripple) +* [N3tc4t](https://github.com/n3tc4t) (XRPL Labs) +* [Nikolaos D Bougalis](https://github.com/nbougalis) +* [Nixer89](https://github.com/nixer89) (XRP Ledger Foundation) +* [RichardAH](https://github.com/RichardAH) (XRPL Labs + XRP Ledger Foundation) +* [Seelabs](https://github.com/seelabs) (Ripple) +* [Silkjaer](https://github.com/Silkjaer) (XRP Ledger Foundation) +* [WietseWind](https://github.com/WietseWind) (XRPL Labs + XRP Ledger Foundation) +* [Ximinez](https://github.com/ximinez) (Ripple) diff --git a/README.md b/README.md index f4cb4e2a1..267ecfb25 100644 --- a/README.md +++ b/README.md @@ -55,5 +55,6 @@ git-subtree. See those directories' README files for more details. * [XRP Ledger Dev Portal](https://xrpl.org/) * [Setup and Installation](https://xrpl.org/install-rippled.html) -* [Source Documentation (Doxygen)](https://ripple.github.io/rippled) +* [Source Documentation (Doxygen)](https://xrplf.github.io/rippled/) +* [Mailing List for Release Announcements](https://groups.google.com/g/ripple-server) * [Learn more about the XRP Ledger (YouTube)](https://www.youtube.com/playlist?list=PLJQ55Tj1hIVZtJ_JdTvSum2qMTsedWkNi) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index d509f9d7e..4403110e0 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -5,7 +5,224 @@ This document contains the release notes for `rippled`, the reference server implementation of the XRP Ledger protocol. To learn more about how to build, run or update a `rippled` server, visit https://xrpl.org/install-rippled.html -Have new ideas? Need help with setting up your node? Come visit us [here](https://github.com/ripple/rippled/issues/new/choose) +Have new ideas? Need help with setting up your node? Come visit us [here](https://github.com/xrplf/rippled/issues/new/choose) + +# Introducing XRP Ledger version 1.9.4 + +Version 1.9.4 of `rippled`, the reference implementation of the XRP Ledger protocol is now available. This release introduces an amendment that removes the ability for an NFT issuer to indicate that trust lines should be automatically created for royalty payments from secondary sales of NFTs, in response to a bug report that indicated how this functionality could be abused to mount a denial of service attack against the issuer. + +## Action Required + +This release introduces a new amendment to the XRP Ledger protocol, **`fixRemoveNFTokenAutoTrustLine`** to mitigate a potential denial-of-service attack against NFT issuers that minted NFTs and allowed secondary trading of those NFTs to create trust lines for any asset. + +This amendment is open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators. + +If you operate an XRP Ledger server, then you should upgrade to version 1.9.4 within two weeks, to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network. + +For more information about NFTs on the XRP Ledger, see [NFT Conceptual Overview](https://xrpl.org/nft-conceptual-overview.html). + + +## Install / Upgrade + +On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html). + +## Changelog + +## Contributions + +The primary change in this release is the following bug fix: + +- **Introduce fixRemoveNFTokenAutoTrustLine amendment**: Introduces the `fixRemoveNFTokenAutoTrustLine` amendment, which disables the `tfTrustLine` flag, which a malicious attacker could exploit to mount denial-of-service attacks against NFT issuers that specified the flag on their NFTs. ([#4301](https://github.com/XRPLF/rippled/4301)) + + +### GitHub + +The public source code repository for `rippled` is hosted on GitHub at . + +We welcome all contributions and invite everyone to join the community of XRP Ledger developers and help us build the Internet of Value. + +### Credits + +The following people contributed directly to this release: + +- Scott Schurr +- Howard Hinnant +- Scott Determan +- Ikko Ashimine + + +# Introducing XRP Ledger version 1.9.3 + +Version 1.9.3 of `rippled`, the reference server implementation of the XRP Ledger protocol is now available. This release corrects minor technical flaws with the code that loads configured amendment votes after a startup and the copy constructor of `PublicKey`. + +## Install / Upgrade + +On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html). + +## Changelog + +## Contributions + +This release contains the following bug fixes: + +- **Change by-value to by-reference to persist vote**: A minor technical flaw, caused by use of a copy instead of a reference, resulted in operator-configured "yes" votes to not be properly loaded after a restart. ([#4256](https://github.com/XRPLF/rippled/pull/4256)) +- **Properly handle self-assignment of PublicKey**: The `PublicKey` copy assignment operator mishandled the case where a `PublicKey` would be assigned to itself, and could result in undefined behavior. + +### GitHub + +The public source code repository for `rippled` is hosted on GitHub at . + +We welcome contributions, big and small, and invite everyone to join the community of XRP Ledger developers and help us build the Internet of Value. + +### Credits + +The following people contributed directly to this release: + +- Howard Hinnant +- Crypto Brad Garlinghouse +- Wo Jake <87929946+wojake@users.noreply.github.com> + + +# Introducing XRP Ledger version 1.9.2 + +Version 1.9.2 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release includes several fixes and improvements, including a second new fix amendment to correct a bug in Non-Fungible Tokens (NFTs) code, a new API method for order book changes, less noisy logging, and other small fixes. + + + + +## Action Required + +This release introduces a two new amendments to the XRP Ledger protocol. The first, **fixNFTokenNegOffer**, fixes a bug in code associated with the **NonFungibleTokensV1** amendment, originally introduced in [version 1.9.0](https://xrpl.org/blog/2022/rippled-1.9.0.html). The second, **NonFungibleTokensV1_1**, is a "roll-up" amendment that enables the **NonFungibleTokensV1** feature plus the two fix amendments associated with it, **fixNFTokenDirV1** and **fixNFTokenNegOffer**. + +If you want to enable NFT code on the XRP Ledger Mainnet, you can vote in favor of only the **NonFungibleTokensV1_1** amendment to support enabling the feature and fixes together, without risk that the unfixed NFT code may become enabled first. + +These amendments are now open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators. + +If you operate an XRP Ledger server, then you should upgrade to version 1.9.2 within two weeks, to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network. + +For more information about NFTs on the XRP Ledger, see [NFT Conceptual Overview](https://xrpl.org/nft-conceptual-overview.html). + +## Install / Upgrade + +On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html). + +## Changelog + +This release contains the following features and improvements. + +- **Introduce fixNFTokenNegOffer amendment.** This amendment fixes a bug in the Non-Fungible Tokens (NFTs) functionality provided by the NonFungibleTokensV1 amendment (not currently enabled on Mainnet). The bug allowed users to place offers to buy tokens for negative amounts of money when using Brokered Mode. Anyone who accepted such an offer would transfer the token _and_ pay money. This amendment explicitly disallows offers to buy or sell NFTs for negative amounts of money, and returns an appropriate error code. This also corrects the error code returned when placing offers to buy or sell NFTs for negative amounts in Direct Mode. ([8266d9d](https://github.com/XRPLF/rippled/commit/8266d9d598d19f05e1155956b30ca443c27e119e)) +- **Introduce `NonFungibleTokensV1_1` amendment.** This amendment encompasses three NFT-related amendments: the original NonFungibleTokensV1 amendment (from version 1.9.0), the fixNFTokenDirV1 amendment (from version 1.9.1), and the new fixNFTokenNegOffer amendment from this release. This amendment contains no changes other than enabling those three amendments together; this allows validators to vote in favor of _only_ enabling the feature and fixes at the same time. ([59326bb](https://github.com/XRPLF/rippled/commit/59326bbbc552287e44b3a0d7b8afbb1ddddb3e3b)) +- **Handle invalid port numbers.** If the user specifies a URL with an invalid port number, the server would silently attempt to use port 0 instead. Now it raises an error instead. This affects admin API methods and config file parameters for downloading history shards and specifying validator list sites. ([#4213](https://github.com/XRPLF/rippled/pull/4213)) +- **Reduce log noisiness.** Decreased the severity of benign log messages in several places: "addPathsForType" messages during regular operation, expected errors during unit tests, and missing optional documentation components when compiling from source. ([#4178](https://github.com/XRPLF/rippled/pull/4178), [#4166](https://github.com/XRPLF/rippled/pull/4166), [#4180](https://github.com/XRPLF/rippled/pull/4180)) +- **Fix race condition in history shard implementation and support clang's ThreadSafetyAnalysis tool.** Added build settings so that developers can use this feature of the clang compiler to analyze the code for correctness, and fix an error found by this tool, which was the source of rare crashes in unit tests. ([#4188](https://github.com/XRPLF/rippled/pull/4188)) +- **Prevent crash when rotating a database with missing data.** When rotating databases, a missing entry could cause the server to crash. While there should never be a missing database entry, this change keeps the server running by aborting database rotation. ([#4182](https://github.com/XRPLF/rippled/pull/4182)) +- **Fix bitwise comparison in OfferCreate.** Fixed an expression that incorrectly used a bitwise comparison for two boolean values rather than a true boolean comparison. The outcome of the two comparisons is equivalent, so this is not a transaction processing change, but the bitwise comparison relied on compilers to implicitly fix the expression. ([#4183](https://github.com/XRPLF/rippled/pull/4183)) +- **Disable cluster timer when not in a cluster.** Disabled a timer that was unused on servers not running in clustered mode. The functionality of clustered servers is unchanged. ([#4173](https://github.com/XRPLF/rippled/pull/4173)) +- **Limit how often to process peer discovery messages.** In the peer-to-peer network, servers periodically share IP addresses of their peers with each other to facilitate peer discovery. It is not necessary to process these types of messages too often; previously, the code tracked whether it needed to process new messages of this type but always processed them anyway. With this change, the server no longer processes peer discovery messages if it has done so recently. ([#4202](https://github.com/XRPLF/rippled/pull/4202)) +- **Improve STVector256 deserialization.** Optimized the processing of this data type in protocol messages. This data type is used in several types of ledger entry that are important for bookkeeping, including directory pages that track other ledger types, amendments tracking, and the ledger hashes history. ([#4204](https://github.com/XRPLF/rippled/pull/4204)) +- **Fix and refactor spinlock code.** The spinlock code, which protects the `SHAMapInnerNode` child lists, had a mistake that allowed the same child to be repeatedly locked under some circumstances. Fixed this bug and improved the spinlock code to make it easier to use correctly and easier to verify that the code works correctly. ([#4201](https://github.com/XRPLF/rippled/pull/4201)) +- **Improve comments and contributor documentation.** Various minor documentation changes including some to reflect the fact that the source code repository is now owned by the XRP Ledger Foundation. ([#4214](https://github.com/XRPLF/rippled/pull/4214), [#4179](https://github.com/XRPLF/rippled/pull/4179), [#4222](https://github.com/XRPLF/rippled/pull/4222)) +- **Introduces a new API book_changes to provide information in a format that is useful for building charts that highlight DEX activity at a per-ledger level.** ([#4212](https://github.com/XRPLF/rippled/pull/4212)) + +## Contributions + +### GitHub + +The public source code repository for `rippled` is hosted on GitHub at . + +We welcome contributions, big and small, and invite everyone to join the community of XRP Ledger developers and help us build the Internet of Value. + +### Credits + +The following people contributed directly to this release: + +- Chenna Keshava B S +- Ed Hennis +- Ikko Ashimine +- Nik Bougalis +- Richard Holland +- Scott Schurr +- Scott Determan + +For a real-time view of all lifetime contributors, including links to the commits made by each, please visit the "Contributors" section of the GitHub repository: . + +# Introducing XRP Ledger version 1.9.1 + +Version 1.9.1 of `rippled`, the reference server implementation of the XRP Ledger protocol, is now available. This release includes several important fixes, including a fix for a syncing issue from 1.9.0, a new fix amendment to correct a bug in the new Non-Fungible Tokens (NFTs) code, and a new amendment to allow multi-signing by up to 32 signers. + + + + +## Action Required + +This release introduces two new amendments to the XRP Ledger protocol. These amendments are now open for voting according to the XRP Ledger's [amendment process](https://xrpl.org/amendments.html), which enables protocol changes following two weeks of >80% support from trusted validators. + +If you operate an XRP Ledger server, then you should upgrade to version 1.9.1 within two weeks, to ensure service continuity. The exact time that protocol changes take effect depends on the voting decisions of the decentralized network. + +The **fixNFTokenDirV1** amendment fixes a bug in code associated with the **NonFungibleTokensV1** amendment, so the fixNFTokenDirV1 amendment should be enabled first. All validator operators are encouraged to [configure amendment voting](https://xrpl.org/configure-amendment-voting.html) to oppose the NonFungibleTokensV1 amendment until _after_ the fixNFTokenDirV1 amendment has become enabled. For more information about NFTs on the XRP Ledger, see [NFT Conceptual Overview](https://xrpl.org/nft-conceptual-overview.html). + +The **ExpandedSignerList** amendment extends the ledger's built-in multi-signing functionality so that each list can contain up to 32 entries instead of the current limit of 8. Additionally, this amendment allows each signer to have an arbitrary 256-bit data field associated with it. This data can be used to identify the signer or provide other metadata that is useful for organizations, smart contracts, or other purposes. + +## Install / Upgrade + +On supported platforms, see the [instructions on installing or updating `rippled`](https://xrpl.org/install-rippled.html). + +## Changelog + +This release contains the following features and improvements. + +## New Features and Amendments + +- **Introduce fixNFTokenDirV1 Amendment** - This amendment fixes an off-by-one error that occurred in some corner cases when determining which `NFTokenPage` an `NFToken` object belongs on. It also adjusts the constraints of `NFTokenPage` invariant checks, so that certain error cases fail with a suitable error code such as `tecNO_SUITABLE_TOKEN_PAGE` instead of failing with a `tecINVARIANT_FAILED` error code. ([#4155](https://github.com/ripple/rippled/pull/4155)) + +- **Introduce ExpandedSignerList Amendment** - This amendment expands the maximum signer list size to 32 entries and allows each signer to have an optional 256-bit `WalletLocator` field containing arbitrary data. ([#4097](https://github.com/ripple/rippled/pull/4097)) + +- **Pause online deletion rather than canceling it if the server fails health check** - The server stops performing online deletion of old ledger history if the server fails its internal health check during this time. Online deletion can now resume after the server recovers, rather than having to start over. ([#4139](https://github.com/ripple/rippled/pull/4139)) + + +## Bug Fixes and Performance Improvements + +- **Fix performance issues introduced in 1.9.0** - Readjusts some parameters of the ledger acquisition engine to revert some changes introduced in 1.9.0 that had adverse effects on some systems, including causing some systems to fail to sync to the network. ([#4152](https://github.com/ripple/rippled/pull/4152)) + +- **Improve Memory Efficiency of Path Finding** - Finding paths for cross-currency payments is a resource-intensive operation. While that remains true, this fix improves memory usage of pathfinding by discarding trust line results that cannot be used before those results are fully loaded or cached. ([#4111](https://github.com/ripple/rippled/pull/4111)) + +- **Fix incorrect CMake behavior on Windows when platform is unspecified or x64** - Fixes handling of platform selection when using the cmake-gui tool to build on Windows. The generator expects `Win64` but the GUI only provides `x64` as an option, which raises an error. This fix only raises an error if the platform is `Win32` instead, allowing the generation of solution files to succeed. ([#4150](https://github.com/ripple/rippled/pull/4150)) + +- **Fix test failures with newer MSVC compilers on Windows** - Fixes some cases where the API handler code used string pointer comparisons, which may not work correctly with some versions of the MSVC compiler. ([#4149](https://github.com/ripple/rippled/pull/4149)) + +- **Update minimum Boost version to 1.71.0** - This release is compatible with Boost library versions 1.71.0 through 1.77.0. The build configuration and documentation have been updated to reflect this. ([#4134](https://github.com/ripple/rippled/pull/4134)) + +- **Fix unit test failures for DatabaseDownloader** - Increases a timeout in the `DatabaseDownloader` code and adjusts unit tests so that the code does not return spurious failures, and more data is logged if it does fail. ([#4021](https://github.com/ripple/rippled/pull/4021)) + +- **Refactor relational database interface** - Improves code comments, naming, and organization of the module that interfaces with relational databases (such as the SQLite database used for tracking transaction history). ([#3965](https://github.com/ripple/rippled/pull/3965)) + + +## Contributions + +### GitHub + +The public source code repository for `rippled` is hosted on GitHub at . + +We welcome contributions, big and small, and invite everyone to join the community of XRP Ledger developers and help us build the Internet of Value. + + +### Credits + +The following people contributed directly to this release: + +- Devon White +- Ed Hennis +- Gregory Popovitch +- Mark Travis +- Manoj Doshi +- Nik Bougalis +- Richard Holland +- Scott Schurr + +For a real-time view of all lifetime contributors, including links to the commits made by each, please visit the "Contributors" section of the GitHub repository: . + +We welcome external contributions and are excited to see the broader XRP Ledger community continue to grow and thrive. + # Change log diff --git a/cfg/rippled-example.cfg b/cfg/rippled-example.cfg index f87ad44c7..fda1671e4 100644 --- a/cfg/rippled-example.cfg +++ b/cfg/rippled-example.cfg @@ -1140,17 +1140,10 @@ # The online delete process checks periodically # that rippled is still in sync with the network, # and that the validated ledger is less than -# 'age_threshold_seconds' old. By default, if it -# is not the online delete process aborts and -# tries again later. If 'recovery_wait_seconds' -# is set and rippled is out of sync, but likely to -# recover quickly, then online delete will wait -# this number of seconds for rippled to get back -# into sync before it aborts. -# Set this value if the node is otherwise staying -# in sync, or recovering quickly, but the online -# delete process is unable to finish. -# Default is unset. +# 'age_threshold_seconds' old. If not, then continue +# sleeping for this number of seconds and +# checking until healthy. +# Default is 5. # # Optional keys for Cassandra: # @@ -1636,10 +1629,10 @@ ip = 127.0.0.1 admin = 127.0.0.1 protocol = ws -#[port_grpc] -#port = 50051 -#ip = 0.0.0.0 -#secure_gateway = 127.0.0.1 +[port_grpc] +port = 50051 +ip = 127.0.0.1 +secure_gateway = 127.0.0.1 #[port_ws_public] #port = 6005 diff --git a/src/ripple/app/consensus/RCLConsensus.cpp b/src/ripple/app/consensus/RCLConsensus.cpp index be8f2af8c..aec747e09 100644 --- a/src/ripple/app/consensus/RCLConsensus.cpp +++ b/src/ripple/app/consensus/RCLConsensus.cpp @@ -86,9 +86,11 @@ RCLConsensus::Adaptor::Adaptor( , inboundTransactions_{inboundTransactions} , j_(journal) , validatorKeys_(validatorKeys) - , valCookie_{rand_int( - 1, - std::numeric_limits::max())} + , valCookie_( + 1 + + rand_int( + crypto_prng(), + std::numeric_limits::max() - 1)) , nUnlVote_(validatorKeys_.nodeID, j_) { assert(valCookie_ != 0); @@ -211,15 +213,10 @@ RCLConsensus::Adaptor::propose(RCLCxPeerPos::Proposal const& proposal) prop.set_nodepubkey( validatorKeys_.publicKey.data(), validatorKeys_.publicKey.size()); - auto signingHash = sha512Half( - HashPrefix::proposal, - std::uint32_t(proposal.proposeSeq()), - proposal.closeTime().time_since_epoch().count(), - proposal.prevLedger(), - proposal.position()); - auto sig = signDigest( - validatorKeys_.publicKey, validatorKeys_.secretKey, signingHash); + validatorKeys_.publicKey, + validatorKeys_.secretKey, + proposal.signingHash()); prop.set_signature(sig.data(), sig.size()); @@ -421,7 +418,9 @@ RCLConsensus::Adaptor::onAccept( Json::Value&& consensusJson) { app_.getJobQueue().addJob( - jtACCEPT, "acceptLedger", [=, cj = std::move(consensusJson)]() mutable { + jtACCEPT, + "acceptLedger", + [=, this, cj = std::move(consensusJson)]() mutable { // Note that no lock is held or acquired during this job. // This is because generic Consensus guarantees that once a ledger // is accepted, the consensus results and capture by reference state @@ -632,7 +631,7 @@ RCLConsensus::Adaptor::doAccept( auto const lastVal = ledgerMaster_.getValidatedLedger(); std::optional rules; if (lastVal) - rules.emplace(*lastVal, app_.config().features); + rules = makeRulesGivenLedger(*lastVal, app_.config().features); else rules.emplace(app_.config().features); app_.openLedger().accept( diff --git a/src/ripple/app/consensus/RCLCxPeerPos.cpp b/src/ripple/app/consensus/RCLCxPeerPos.cpp index 709e78988..ee5a45b94 100644 --- a/src/ripple/app/consensus/RCLCxPeerPos.cpp +++ b/src/ripple/app/consensus/RCLCxPeerPos.cpp @@ -32,29 +32,23 @@ RCLCxPeerPos::RCLCxPeerPos( Slice const& signature, uint256 const& suppression, Proposal&& proposal) - : data_{std::make_shared( - publicKey, - signature, - suppression, - std::move(proposal))} + : publicKey_(publicKey) + , suppression_(suppression) + , proposal_(std::move(proposal)) { -} + // The maximum allowed size of a signature is 72 bytes; we verify + // this elsewhere, but we want to be extra careful here: + assert(signature.size() != 0 && signature.size() <= signature_.capacity()); -uint256 -RCLCxPeerPos::signingHash() const -{ - return sha512Half( - HashPrefix::proposal, - std::uint32_t(proposal().proposeSeq()), - proposal().closeTime().time_since_epoch().count(), - proposal().prevLedger(), - proposal().position()); + if (signature.size() != 0 && signature.size() <= signature_.capacity()) + signature_.assign(signature.begin(), signature.end()); } bool RCLCxPeerPos::checkSign() const { - return verifyDigest(publicKey(), signingHash(), signature(), false); + return verifyDigest( + publicKey(), proposal_.signingHash(), signature(), false); } Json::Value @@ -88,16 +82,4 @@ proposalUniqueId( return s.getSHA512Half(); } -RCLCxPeerPos::Data::Data( - PublicKey const& publicKey, - Slice const& signature, - uint256 const& suppress, - Proposal&& proposal) - : publicKey_{publicKey} - , signature_{signature} - , suppression_{suppress} - , proposal_{std::move(proposal)} -{ -} - } // namespace ripple diff --git a/src/ripple/app/consensus/RCLCxPeerPos.h b/src/ripple/app/consensus/RCLCxPeerPos.h index 9d448aac4..e82a85d42 100644 --- a/src/ripple/app/consensus/RCLCxPeerPos.h +++ b/src/ripple/app/consensus/RCLCxPeerPos.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -61,10 +62,6 @@ public: uint256 const& suppress, Proposal&& proposal); - //! Create the signing hash for the proposal - uint256 - signingHash() const; - //! Verify the signing hash of the proposal bool checkSign() const; @@ -73,27 +70,27 @@ public: Slice signature() const { - return data_->signature_; + return {signature_.data(), signature_.size()}; } //! Public key of peer that sent the proposal PublicKey const& publicKey() const { - return data_->publicKey_; + return publicKey_; } //! Unique id used by hash router to suppress duplicates uint256 const& suppressionID() const { - return data_->suppression_; + return suppression_; } Proposal const& proposal() const { - return data_->proposal_; + return proposal_; } //! JSON representation of proposal @@ -101,21 +98,10 @@ public: getJson() const; private: - struct Data : public CountedObject - { - PublicKey publicKey_; - Buffer signature_; - uint256 suppression_; - Proposal proposal_; - - Data( - PublicKey const& publicKey, - Slice const& signature, - uint256 const& suppress, - Proposal&& proposal); - }; - - std::shared_ptr data_; + PublicKey publicKey_; + uint256 suppression_; + Proposal proposal_; + boost::container::static_vector signature_; template void diff --git a/src/ripple/app/ledger/AcceptedLedger.cpp b/src/ripple/app/ledger/AcceptedLedger.cpp index 526704d18..4f308653d 100644 --- a/src/ripple/app/ledger/AcceptedLedger.cpp +++ b/src/ripple/app/ledger/AcceptedLedger.cpp @@ -31,11 +31,9 @@ AcceptedLedger::AcceptedLedger( transactions_.reserve(256); auto insertAll = [&](auto const& txns) { - auto const& idcache = app.accountIDCache(); - for (auto const& item : txns) transactions_.emplace_back(std::make_unique( - ledger, item.first, item.second, idcache)); + ledger, item.first, item.second)); }; if (app.config().reporting()) diff --git a/src/ripple/app/ledger/AcceptedLedgerTx.cpp b/src/ripple/app/ledger/AcceptedLedgerTx.cpp index f0408b0c0..613a91e43 100644 --- a/src/ripple/app/ledger/AcceptedLedgerTx.cpp +++ b/src/ripple/app/ledger/AcceptedLedgerTx.cpp @@ -28,8 +28,7 @@ namespace ripple { AcceptedLedgerTx::AcceptedLedgerTx( std::shared_ptr const& ledger, std::shared_ptr const& txn, - std::shared_ptr const& met, - AccountIDCache const& accountCache) + std::shared_ptr const& met) : mTxn(txn) , mMeta(txn->getTransactionID(), ledger->seq(), *met) , mAffected(mMeta.getAffectedAccounts()) @@ -52,7 +51,7 @@ AcceptedLedgerTx::AcceptedLedgerTx( { Json::Value& affected = (mJson[jss::affected] = Json::arrayValue); for (auto const& account : mAffected) - affected.append(accountCache.toBase58(account)); + affected.append(toBase58(account)); } if (mTxn->getTxnType() == ttOFFER_CREATE) diff --git a/src/ripple/app/ledger/AcceptedLedgerTx.h b/src/ripple/app/ledger/AcceptedLedgerTx.h index 7d6897857..2995d447b 100644 --- a/src/ripple/app/ledger/AcceptedLedgerTx.h +++ b/src/ripple/app/ledger/AcceptedLedgerTx.h @@ -46,8 +46,7 @@ public: AcceptedLedgerTx( std::shared_ptr const& ledger, std::shared_ptr const&, - std::shared_ptr const&, - AccountIDCache const&); + std::shared_ptr const&); std::shared_ptr const& getTxn() const diff --git a/src/ripple/app/ledger/Ledger.cpp b/src/ripple/app/ledger/Ledger.cpp index 66bea5682..713114485 100644 --- a/src/ripple/app/ledger/Ledger.cpp +++ b/src/ripple/app/ledger/Ledger.cpp @@ -29,8 +29,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include @@ -626,7 +626,7 @@ Ledger::setup(Config const& config) try { - rules_ = Rules(*this, config.features); + rules_ = makeRulesGivenLedger(*this, config.features); } catch (SHAMapMissingNode const&) { @@ -930,9 +930,11 @@ saveValidatedLedger( return true; } - auto res = dynamic_cast( - &app.getRelationalDBInterface()) - ->saveValidatedLedger(ledger, current); + auto const db = dynamic_cast(&app.getRelationalDatabase()); + if (!db) + Throw("Failed to get relational database"); + + auto const res = db->saveValidatedLedger(ledger, current); // Clients can now trust the database for // information about this ledger sequence. @@ -1053,7 +1055,7 @@ std::tuple, std::uint32_t, uint256> getLatestLedger(Application& app) { const std::optional info = - app.getRelationalDBInterface().getNewestLedgerInfo(); + app.getRelationalDatabase().getNewestLedgerInfo(); if (!info) return {std::shared_ptr(), {}, {}}; return {loadLedgerHelper(*info, app, true), info->seq, info->hash}; @@ -1063,7 +1065,7 @@ std::shared_ptr loadByIndex(std::uint32_t ledgerIndex, Application& app, bool acquire) { if (std::optional info = - app.getRelationalDBInterface().getLedgerInfoByIndex(ledgerIndex)) + app.getRelationalDatabase().getLedgerInfoByIndex(ledgerIndex)) { std::shared_ptr ledger = loadLedgerHelper(*info, app, acquire); finishLoadByIndexOrHash(ledger, app.config(), app.journal("Ledger")); @@ -1076,7 +1078,7 @@ std::shared_ptr loadByHash(uint256 const& ledgerHash, Application& app, bool acquire) { if (std::optional info = - app.getRelationalDBInterface().getLedgerInfoByHash(ledgerHash)) + app.getRelationalDatabase().getLedgerInfoByHash(ledgerHash)) { std::shared_ptr ledger = loadLedgerHelper(*info, app, acquire); finishLoadByIndexOrHash(ledger, app.config(), app.journal("Ledger")); @@ -1165,9 +1167,12 @@ flatFetchTransactions(ReadView const& ledger, Application& app) return {}; } - auto nodestoreHashes = dynamic_cast( - &app.getRelationalDBInterface()) - ->getTxHashes(ledger.info().seq); + auto const db = + dynamic_cast(&app.getRelationalDatabase()); + if (!db) + Throw("Failed to get relational database"); + + auto nodestoreHashes = db->getTxHashes(ledger.info().seq); return flatFetchTransactions(app, nodestoreHashes); } diff --git a/src/ripple/app/ledger/impl/InboundLedger.cpp b/src/ripple/app/ledger/impl/InboundLedger.cpp index d24c451a1..3ecba97b1 100644 --- a/src/ripple/app/ledger/impl/InboundLedger.cpp +++ b/src/ripple/app/ledger/impl/InboundLedger.cpp @@ -44,27 +44,27 @@ using namespace std::chrono_literals; enum { // Number of peers to start with - peerCountStart = 4 + peerCountStart = 5 // Number of peers to add on a timeout , - peerCountAdd = 2 + peerCountAdd = 3 // how many timeouts before we give up , - ledgerTimeoutRetriesMax = 10 + ledgerTimeoutRetriesMax = 6 // how many timeouts before we get aggressive , - ledgerBecomeAggressiveThreshold = 6 + ledgerBecomeAggressiveThreshold = 4 // Number of nodes to find initially , - missingNodesFind = 512 + missingNodesFind = 256 // Number of nodes to request for a reply , - reqNodesReply = 256 + reqNodesReply = 128 // Number of nodes to request blindly , @@ -72,7 +72,7 @@ enum { }; // millisecond for each ledger timeout -auto constexpr ledgerAcquireTimeout = 2500ms; +auto constexpr ledgerAcquireTimeout = 3000ms; InboundLedger::InboundLedger( Application& app, @@ -664,15 +664,15 @@ InboundLedger::trigger(std::shared_ptr const& peer, TriggerReason reason) if (reason != TriggerReason::reply) { // If we're querying blind, don't query deep - tmGL.set_querydepth(1); + tmGL.set_querydepth(0); } else if (peer && peer->isHighLatency()) { // If the peer has high latency, query extra deep - tmGL.set_querydepth(3); + tmGL.set_querydepth(2); } else - tmGL.set_querydepth(2); + tmGL.set_querydepth(1); // Get the state data first because it's the most likely to be useful // if we wind up abandoning this fetch. diff --git a/src/ripple/app/ledger/impl/LedgerMaster.cpp b/src/ripple/app/ledger/impl/LedgerMaster.cpp index 3bae67f65..ad08b18dd 100644 --- a/src/ripple/app/ledger/impl/LedgerMaster.cpp +++ b/src/ripple/app/ledger/impl/LedgerMaster.cpp @@ -34,8 +34,7 @@ #include #include #include -#include -#include +#include #include #include #include @@ -278,10 +277,10 @@ LedgerMaster::getValidatedLedgerAge() #ifdef RIPPLED_REPORTING if (app_.config().reporting()) - return static_cast( - &app_.getRelationalDBInterface()) + return static_cast(&app_.getRelationalDatabase()) ->getValidatedLedgerAge(); #endif + std::chrono::seconds valClose{mValidLedgerSign.load()}; if (valClose == 0s) { @@ -309,8 +308,7 @@ LedgerMaster::isCaughtUp(std::string& reason) #ifdef RIPPLED_REPORTING if (app_.config().reporting()) - return static_cast( - &app_.getRelationalDBInterface()) + return static_cast(&app_.getRelationalDatabase()) ->isCaughtUp(reason); #endif @@ -743,7 +741,7 @@ LedgerMaster::tryFill(std::shared_ptr ledger) mCompleteLedgers.insert(range(minHas, maxHas)); } maxHas = minHas; - ledgerHashes = app_.getRelationalDBInterface().getHashesByIndex( + ledgerHashes = app_.getRelationalDatabase().getHashesByIndex( (seq < 500) ? 0 : (seq - 499), seq); it = ledgerHashes.find(seq); @@ -927,8 +925,8 @@ LedgerMaster::setFullLedger( { // Check the SQL database's entry for the sequence before this // ledger, if it's not this ledger's parent, invalidate it - uint256 prevHash = app_.getRelationalDBInterface().getHashByIndex( - ledger->info().seq - 1); + uint256 prevHash = + app_.getRelationalDatabase().getHashByIndex(ledger->info().seq - 1); if (prevHash.isNonZero() && prevHash != ledger->info().parentHash) clearLedger(ledger->info().seq - 1); } @@ -1664,7 +1662,7 @@ LedgerMaster::getValidatedLedger() #ifdef RIPPLED_REPORTING if (app_.config().reporting()) { - auto seq = app_.getRelationalDBInterface().getMaxLedgerSeq(); + auto seq = app_.getRelationalDatabase().getMaxLedgerSeq(); if (!seq) return {}; return getLedgerBySeq(*seq); @@ -1700,8 +1698,7 @@ LedgerMaster::getCompleteLedgers() { #ifdef RIPPLED_REPORTING if (app_.config().reporting()) - return static_cast( - &app_.getRelationalDBInterface()) + return static_cast(&app_.getRelationalDatabase()) ->getCompleteLedgers(); #endif std::lock_guard sl(mCompleteLock); @@ -1746,7 +1743,7 @@ LedgerMaster::getHashBySeq(std::uint32_t index) if (hash.isNonZero()) return hash; - return app_.getRelationalDBInterface().getHashByIndex(index); + return app_.getRelationalDatabase().getHashByIndex(index); } std::optional @@ -1967,7 +1964,7 @@ LedgerMaster::fetchForHistory( fillInProgress = mFillInProgress; } if (fillInProgress == 0 && - app_.getRelationalDBInterface().getHashByIndex(seq - 1) == + app_.getRelationalDatabase().getHashByIndex(seq - 1) == ledger->info().parentHash) { { @@ -2363,7 +2360,7 @@ LedgerMaster::getFetchPackCacheSize() const std::optional LedgerMaster::minSqlSeq() { - return app_.getRelationalDBInterface().getMinLedgerSeq(); + return app_.getRelationalDatabase().getMinLedgerSeq(); } } // namespace ripple diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 2256ee31b..dce11bc38 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -45,17 +45,19 @@ #include #include #include -#include -#include +#include +#include #include #include #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -165,6 +167,8 @@ public: std::unique_ptr logs_; std::unique_ptr timeKeeper_; + std::uint64_t const instanceCookie_; + beast::Journal m_journal; std::unique_ptr perfLog_; Application::MutexType m_masterMutex; @@ -177,7 +181,6 @@ public: NodeStoreScheduler m_nodeStoreScheduler; std::unique_ptr m_shaMapStore; PendingSaves pendingSaves_; - AccountIDCache accountIDCache_; std::optional openLedger_; NodeCache m_tempNodeCache; @@ -219,7 +222,7 @@ public: boost::asio::steady_timer sweepTimer_; boost::asio::steady_timer entropyTimer_; - std::unique_ptr mRelationalDBInterface; + std::unique_ptr mRelationalDatabase; std::unique_ptr mWalletDB; std::unique_ptr overlay_; @@ -274,6 +277,11 @@ public: , config_(std::move(config)) , logs_(std::move(logs)) , timeKeeper_(std::move(timeKeeper)) + , instanceCookie_( + 1 + + rand_int( + crypto_prng(), + std::numeric_limits::max() - 1)) , m_journal(logs_->journal("Application")) // PerfLog must be started before any other threads are launched. @@ -327,8 +335,6 @@ public: m_nodeStoreScheduler, logs_->journal("SHAMapStore"))) - , accountIDCache_(128000) - , m_tempNodeCache( "NodeCache", 16384, @@ -485,6 +491,8 @@ public: config_->reporting() ? std::make_unique(*this) : nullptr) { + initAccountIdCache(config_->getValueFor(SizedItem::accountIdCacheSize)); + add(m_resourceManager.get()); // @@ -508,13 +516,13 @@ public: //-------------------------------------------------------------------------- bool - setup() override; + setup(boost::program_options::variables_map const& cmdline) override; void start(bool withTimers) override; void run() override; void - signalStop() override; + signalStop(std::string msg = "") override; bool checkSigs() const override; void @@ -526,6 +534,12 @@ public: //-------------------------------------------------------------------------- + std::uint64_t + instanceID() const override + { + return instanceCookie_; + } + Logs& logs() override { @@ -841,12 +855,6 @@ public: return pendingSaves_; } - AccountIDCache const& - accountIDCache() const override - { - return accountIDCache_; - } - OpenLedger& openLedger() override { @@ -877,11 +885,11 @@ public: return *txQ_; } - RelationalDBInterface& - getRelationalDBInterface() override + RelationalDatabase& + getRelationalDatabase() override { - assert(mRelationalDBInterface.get() != nullptr); - return *mRelationalDBInterface; + assert(mRelationalDatabase.get() != nullptr); + return *mRelationalDatabase; } DatabaseCon& @@ -907,14 +915,14 @@ public: //-------------------------------------------------------------------------- bool - initRDBMS() + initRelationalDatabase() { assert(mWalletDB.get() == nullptr); try { - mRelationalDBInterface = - RelationalDBInterface::init(*this, *config_, *m_jobQueue); + mRelationalDatabase = + RelationalDatabase::init(*this, *config_, *m_jobQueue); // wallet database auto setup = setup_DatabaseCon(*config_, m_journal); @@ -1041,7 +1049,7 @@ public: doSweep() { if (!config_->standalone() && - !getRelationalDBInterface().transactionDbHasSpace(*config_)) + !getRelationalDatabase().transactionDbHasSpace(*config_)) { signalStop(); } @@ -1066,8 +1074,7 @@ public: cachedSLEs_.sweep(); #ifdef RIPPLED_REPORTING - if (auto pg = dynamic_cast( - &*mRelationalDBInterface)) + if (auto pg = dynamic_cast(&*mRelationalDatabase)) pg->sweep(); #endif @@ -1109,7 +1116,7 @@ private: // TODO Break this up into smaller, more digestible initialization segments. bool -ApplicationImp::setup() +ApplicationImp::setup(boost::program_options::variables_map const& cmdline) { // We want to intercept CTRL-C and the standard termination signal SIGTERM // and terminate the process. This handler will NEVER be invoked twice. @@ -1147,8 +1154,10 @@ ApplicationImp::setup() if (logs_->threshold() > kDebug) logs_->threshold(kDebug); } - JLOG(m_journal.info()) << "process starting: " - << BuildInfo::getFullVersionString(); + + JLOG(m_journal.info()) << "Process starting: " + << BuildInfo::getFullVersionString() + << ", Instance Cookie: " << instanceCookie_; if (numberOfThreads(*config_) < 2) { @@ -1162,7 +1171,7 @@ ApplicationImp::setup() if (!config_->standalone()) timeKeeper_->run(config_->SNTP_SERVERS); - if (!initRDBMS() || !initNodeStore()) + if (!initRelationalDatabase() || !initNodeStore()) return false; if (shardStore_) @@ -1266,7 +1275,7 @@ ApplicationImp::setup() if (!config().reporting()) m_orderBookDB.setup(getLedgerMaster().getCurrentLedger()); - nodeIdentity_ = getNodeIdentity(*this); + nodeIdentity_ = getNodeIdentity(*this, cmdline); if (!cluster_->load(config().section(SECTION_CLUSTER_NODES))) { @@ -1619,8 +1628,7 @@ ApplicationImp::run() ledgerCleaner_->stop(); if (reportingETL_) reportingETL_->stop(); - if (auto pg = dynamic_cast( - &*mRelationalDBInterface)) + if (auto pg = dynamic_cast(&*mRelationalDatabase)) pg->stop(); m_nodeStore->stop(); perfLog_->stop(); @@ -1629,10 +1637,17 @@ ApplicationImp::run() } void -ApplicationImp::signalStop() +ApplicationImp::signalStop(std::string msg) { if (!isTimeToStop.exchange(true)) + { + if (msg.empty()) + JLOG(m_journal.warn()) << "Server stopping"; + else + JLOG(m_journal.warn()) << "Server stopping: " << msg; + stoppingCondition_.notify_all(); + } } bool @@ -2137,7 +2152,7 @@ ApplicationImp::nodeToShards() void ApplicationImp::setMaxDisallowedLedger() { - auto seq = getRelationalDBInterface().getMaxLedgerSeq(); + auto seq = getRelationalDatabase().getMaxLedgerSeq(); if (seq) maxDisallowedLedger_ = *seq; diff --git a/src/ripple/app/main/Application.h b/src/ripple/app/main/Application.h index 0fc927ff6..d8cb7d318 100644 --- a/src/ripple/app/main/Application.h +++ b/src/ripple/app/main/Application.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -89,7 +90,6 @@ class PathRequests; class PendingSaves; class PublicKey; class SecretKey; -class AccountIDCache; class STLedgerEntry; class TimeKeeper; class TransactionMaster; @@ -99,7 +99,7 @@ class ValidatorList; class ValidatorSite; class Cluster; -class RelationalDBInterface; +class RelationalDatabase; class DatabaseCon; class SHAMapStore; @@ -136,13 +136,14 @@ public: virtual ~Application() = default; virtual bool - setup() = 0; + setup(boost::program_options::variables_map const& options) = 0; + virtual void start(bool withTimers) = 0; virtual void run() = 0; virtual void - signalStop() = 0; + signalStop(std::string msg = "") = 0; virtual bool checkSigs() const = 0; virtual void @@ -154,6 +155,10 @@ public: // --- // + /** Returns a 64-bit instance identifier, generated at startup */ + virtual std::uint64_t + instanceID() const = 0; + virtual Logs& logs() = 0; virtual Config& @@ -245,14 +250,12 @@ public: getSHAMapStore() = 0; virtual PendingSaves& pendingSaves() = 0; - virtual AccountIDCache const& - accountIDCache() const = 0; virtual OpenLedger& openLedger() = 0; virtual OpenLedger const& openLedger() const = 0; - virtual RelationalDBInterface& - getRelationalDBInterface() = 0; + virtual RelationalDatabase& + getRelationalDatabase() = 0; virtual std::chrono::milliseconds getIOLatency() = 0; diff --git a/src/ripple/app/main/DBInit.h b/src/ripple/app/main/DBInit.h index 00cfc104d..3d2f42717 100644 --- a/src/ripple/app/main/DBInit.h +++ b/src/ripple/app/main/DBInit.h @@ -72,12 +72,22 @@ inline constexpr std::array LgrDBInit{ // Transaction database holds transactions and public keys inline constexpr auto TxDBName{"transaction.db"}; -inline constexpr std::array TxDBPragma +// In C++17 omitting the explicit template parameters caused +// a crash +inline constexpr std::array TxDBPragma { "PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;", "PRAGMA max_page_count=2147483646;", + #if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP) "PRAGMA mmap_size=17179869184;" +#else + + // Provide an explicit `no-op` SQL statement + // in order to keep the size of the array + // constant regardless of the preprocessor + // condition evaluation + "PRAGMA sqlite_noop_statement;" #endif }; @@ -117,12 +127,22 @@ inline constexpr std::array TxDBInit{ // The Ledger Meta database maps ledger hashes to shard indexes inline constexpr auto LgrMetaDBName{"ledger_meta.db"}; -inline constexpr std::array LgrMetaDBPragma +// In C++17 omitting the explicit template parameters caused +// a crash +inline constexpr std::array LgrMetaDBPragma { "PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;", "PRAGMA max_page_count=2147483646;", + #if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP) "PRAGMA mmap_size=17179869184;" +#else + + // Provide an explicit `no-op` SQL statement + // in order to keep the size of the array + // constant regardless of the preprocessor + // condition evaluation + "PRAGMA sqlite_noop_statement;" #endif }; @@ -141,12 +161,22 @@ inline constexpr std::array LgrMetaDBInit{ // Transaction Meta database maps transaction IDs to shard indexes inline constexpr auto TxMetaDBName{"transaction_meta.db"}; -inline constexpr std::array TxMetaDBPragma +// In C++17 omitting the explicit template parameters caused +// a crash +inline constexpr std::array TxMetaDBPragma { "PRAGMA page_size=4096;", "PRAGMA journal_size_limit=1582080;", "PRAGMA max_page_count=2147483646;", + #if (ULONG_MAX > UINT_MAX) && !defined(NO_SQLITE_MMAP) "PRAGMA mmap_size=17179869184;" +#else + + // Provide an explicit `no-op` SQL statement + // in order to keep the size of the array + // constant regardless of the preprocessor + // condition evaluation + "PRAGMA sqlite_noop_statement;" #endif }; diff --git a/src/ripple/app/main/GRPCServer.cpp b/src/ripple/app/main/GRPCServer.cpp index aef2612c0..fdef8c1ce 100644 --- a/src/ripple/app/main/GRPCServer.cpp +++ b/src/ripple/app/main/GRPCServer.cpp @@ -590,94 +590,6 @@ GRPCServerImpl::setupListeners() requests.push_back(std::move(callData)); }; - { - using cd = CallData< - org::xrpl::rpc::v1::GetFeeRequest, - org::xrpl::rpc::v1::GetFeeResponse>; - - addToRequests(std::make_shared( - service_, - *cq_, - app_, - &org::xrpl::rpc::v1::XRPLedgerAPIService::AsyncService:: - RequestGetFee, - doFeeGrpc, - &org::xrpl::rpc::v1::XRPLedgerAPIService::Stub::GetFee, - RPC::NEEDS_CURRENT_LEDGER, - Resource::feeReferenceRPC, - secureGatewayIPs_)); - } - { - using cd = CallData< - org::xrpl::rpc::v1::GetAccountInfoRequest, - org::xrpl::rpc::v1::GetAccountInfoResponse>; - - addToRequests(std::make_shared( - service_, - *cq_, - app_, - &org::xrpl::rpc::v1::XRPLedgerAPIService::AsyncService:: - RequestGetAccountInfo, - doAccountInfoGrpc, - &org::xrpl::rpc::v1::XRPLedgerAPIService::Stub::GetAccountInfo, - RPC::NO_CONDITION, - Resource::feeReferenceRPC, - secureGatewayIPs_)); - } - { - using cd = CallData< - org::xrpl::rpc::v1::GetTransactionRequest, - org::xrpl::rpc::v1::GetTransactionResponse>; - - addToRequests(std::make_shared( - service_, - *cq_, - app_, - &org::xrpl::rpc::v1::XRPLedgerAPIService::AsyncService:: - RequestGetTransaction, - doTxGrpc, - &org::xrpl::rpc::v1::XRPLedgerAPIService::Stub::GetTransaction, - RPC::NEEDS_NETWORK_CONNECTION, - Resource::feeReferenceRPC, - secureGatewayIPs_)); - } - { - using cd = CallData< - org::xrpl::rpc::v1::SubmitTransactionRequest, - org::xrpl::rpc::v1::SubmitTransactionResponse>; - - addToRequests(std::make_shared( - service_, - *cq_, - app_, - &org::xrpl::rpc::v1::XRPLedgerAPIService::AsyncService:: - RequestSubmitTransaction, - doSubmitGrpc, - &org::xrpl::rpc::v1::XRPLedgerAPIService::Stub::SubmitTransaction, - RPC::NEEDS_CURRENT_LEDGER, - Resource::feeMediumBurdenRPC, - secureGatewayIPs_)); - } - - { - using cd = CallData< - org::xrpl::rpc::v1::GetAccountTransactionHistoryRequest, - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse>; - - addToRequests(std::make_shared( - service_, - *cq_, - app_, - &org::xrpl::rpc::v1::XRPLedgerAPIService::AsyncService:: - RequestGetAccountTransactionHistory, - doAccountTxGrpc, - &org::xrpl::rpc::v1::XRPLedgerAPIService::Stub:: - GetAccountTransactionHistory, - RPC::NO_CONDITION, - Resource::feeMediumBurdenRPC, - secureGatewayIPs_)); - } - { using cd = CallData< org::xrpl::rpc::v1::GetLedgerRequest, diff --git a/src/ripple/app/main/Main.cpp b/src/ripple/app/main/Main.cpp index 14befa63e..f25b83fd5 100644 --- a/src/ripple/app/main/Main.cpp +++ b/src/ripple/app/main/Main.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include @@ -133,8 +133,10 @@ printHelp(const po::options_description& desc) " account_objects [] [strict]\n" " account_offers | [] " "[strict]\n" - " account_tx accountID [ledger_min [ledger_max [limit " - "[offset]]]] [binary] [count] [descending]\n" + " account_tx accountID [ledger_index_min [ledger_index_max " + "[limit " + "]]] [binary]\n" + " book_changes []\n" " book_offers [ " "[ [ []]]]]\n" " can_delete [||now|always|never]\n" @@ -159,6 +161,7 @@ printHelp(const po::options_description& desc) " ledger_request \n" " log_level [[] ]\n" " logrotate\n" + " manifest \n" " node_to_shard [status|start|stop]\n" " peers\n" " ping\n" @@ -178,6 +181,7 @@ printHelp(const po::options_description& desc) " submit_multisigned \n" " tx \n" " validation_create [||]\n" + " validator_info\n" " validators\n" " validator_list_sites\n" " version\n" @@ -368,6 +372,10 @@ run(int argc, char** argv) "conf", po::value(), "Specify the configuration file.")( "debug", "Enable normally suppressed debug logging")( "help,h", "Display this message.")( + "newnodeid", "Generate a new node identity for this server.")( + "nodeid", + po::value(), + "Specify the node identity for this server.")( "quorum", po::value(), "Override the minimum validation quorum.")( @@ -752,7 +760,7 @@ run(int argc, char** argv) auto app = make_Application( std::move(config), std::move(logs), std::move(timeKeeper)); - if (!app->setup()) + if (!app->setup(vm)) return -1; // With our configuration parsed, ensure we have diff --git a/src/ripple/app/main/NodeIdentity.cpp b/src/ripple/app/main/NodeIdentity.cpp index 5f7cca7a5..e66b9e840 100644 --- a/src/ripple/app/main/NodeIdentity.cpp +++ b/src/ripple/app/main/NodeIdentity.cpp @@ -19,28 +19,39 @@ #include #include -#include -#include +#include #include #include -#include #include namespace ripple { std::pair -getNodeIdentity(Application& app) +getNodeIdentity( + Application& app, + boost::program_options::variables_map const& cmdline) { - // If a seed is specified in the configuration file use that directly. - if (app.config().exists(SECTION_NODE_SEED)) + std::optional seed; + + if (cmdline.count("nodeid")) { - auto const seed = parseBase58( + seed = parseGenericSeed(cmdline["nodeid"].as(), false); + + if (!seed) + Throw("Invalid 'nodeid' in command line"); + } + else if (app.config().exists(SECTION_NODE_SEED)) + { + seed = parseBase58( app.config().section(SECTION_NODE_SEED).lines().front()); if (!seed) - Throw("NodeIdentity: Bad [" SECTION_NODE_SEED - "] specified"); + Throw("Invalid [" SECTION_NODE_SEED + "] in configuration file"); + } + if (seed) + { auto secretKey = generateSecretKey(KeyType::secp256k1, *seed); auto publicKey = derivePublicKey(KeyType::secp256k1, secretKey); @@ -48,6 +59,10 @@ getNodeIdentity(Application& app) } auto db = app.getWalletDB().checkoutDb(); + + if (cmdline.count("newnodeid") != 0) + clearNodeIdentity(*db); + return getNodeIdentity(*db); } diff --git a/src/ripple/app/main/NodeIdentity.h b/src/ripple/app/main/NodeIdentity.h index 60deeed85..b82b3657a 100644 --- a/src/ripple/app/main/NodeIdentity.h +++ b/src/ripple/app/main/NodeIdentity.h @@ -23,13 +23,20 @@ #include #include #include +#include #include namespace ripple { -/** The cryptographic credentials identifying this server instance. */ +/** The cryptographic credentials identifying this server instance. + + @param app The application object + @param cmdline The command line parameters passed into the application. + */ std::pair -getNodeIdentity(Application& app); +getNodeIdentity( + Application& app, + boost::program_options::variables_map const& cmdline); } // namespace ripple diff --git a/src/ripple/app/misc/FeeVote.h b/src/ripple/app/misc/FeeVote.h index 543f4cdb6..d8948a150 100644 --- a/src/ripple/app/misc/FeeVote.h +++ b/src/ripple/app/misc/FeeVote.h @@ -46,10 +46,10 @@ public: static constexpr FeeUnit32 reference_fee_units{10}; /** The account reserve requirement in drops. */ - XRPAmount account_reserve{20 * DROPS_PER_XRP}; + XRPAmount account_reserve{10 * DROPS_PER_XRP}; /** The per-owned item reserve requirement in drops. */ - XRPAmount owner_reserve{5 * DROPS_PER_XRP}; + XRPAmount owner_reserve{2 * DROPS_PER_XRP}; }; virtual ~FeeVote() = default; diff --git a/src/ripple/app/misc/NetworkOPs.cpp b/src/ripple/app/misc/NetworkOPs.cpp index 4b44cf431..8dff1af7b 100644 --- a/src/ripple/app/misc/NetworkOPs.cpp +++ b/src/ripple/app/misc/NetworkOPs.cpp @@ -37,9 +37,8 @@ #include #include #include -#include -#include -#include +#include +#include #include #include #include @@ -63,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -503,6 +503,11 @@ public: bool unsubLedger(std::uint64_t uListener) override; + bool + subBookChanges(InfoSub::ref ispListener) override; + bool + unsubBookChanges(std::uint64_t uListener) override; + bool subServer(InfoSub::ref ispListener, Json::Value& jvResult, bool admin) override; @@ -744,9 +749,10 @@ private: sValidations, // Received validations. sPeerStatus, // Peer status changes. sConsensusPhase, // Consensus phase + sBookChanges, // Per-ledger order book changes - sLastEntry = sConsensusPhase // as this name implies, any new entry - // must be ADDED ABOVE this one + sLastEntry = sBookChanges // as this name implies, any new entry + // must be ADDED ABOVE this one }; std::array mStreamMaps; @@ -913,7 +919,10 @@ void NetworkOPsImp::setStateTimer() { setHeartbeatTimer(); - setClusterTimer(); + + // Only do this work if a cluster is configured + if (app_.cluster().size() != 0) + setClusterTimer(); } void @@ -966,6 +975,7 @@ void NetworkOPsImp::setClusterTimer() { using namespace std::chrono_literals; + setTimer( clusterTimer_, 10s, @@ -1051,7 +1061,11 @@ NetworkOPsImp::processHeartbeatTimer() void NetworkOPsImp::processClusterTimer() { + if (app_.cluster().size() == 0) + return; + using namespace std::chrono_literals; + bool const update = app_.cluster().update( app_.nodeIdentity().first, "", @@ -1749,7 +1763,7 @@ NetworkOPsImp::switchLastClosedLedger( auto const lastVal = app_.getLedgerMaster().getValidatedLedger(); std::optional rules; if (lastVal) - rules.emplace(*lastVal, app_.config().features); + rules = makeRulesGivenLedger(*lastVal, app_.config().features); else rules.emplace(app_.config().features); app_.openLedger().accept( @@ -2317,10 +2331,6 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters) if (!app_.config().SERVER_DOMAIN.empty()) info[jss::server_domain] = app_.config().SERVER_DOMAIN; - if (!app_.config().reporting()) - if (auto const netid = app_.overlay().networkID()) - info[jss::network_id] = static_cast(*netid); - info[jss::build_version] = BuildInfo::getVersionString(); info[jss::server_state] = strOperatingMode(admin); @@ -2463,6 +2473,9 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters) if (!app_.config().reporting()) { + if (auto const netid = app_.overlay().networkID()) + info[jss::network_id] = static_cast(*netid); + auto const escalationMetrics = app_.getTxQ().getMetrics(*app_.openLedger().current()); @@ -2899,6 +2912,24 @@ NetworkOPsImp::pubLedger(std::shared_ptr const& lpAccepted) } } + if (!mStreamMaps[sBookChanges].empty()) + { + Json::Value jvObj = ripple::RPC::computeBookChanges(lpAccepted); + + auto it = mStreamMaps[sBookChanges].begin(); + while (it != mStreamMaps[sBookChanges].end()) + { + InfoSub::pointer p = it->second.lock(); + if (p) + { + p->send(jvObj, true); + ++it; + } + else + it = mStreamMaps[sBookChanges].erase(it); + } + } + { static bool firstTime = true; if (firstTime) @@ -3366,8 +3397,9 @@ NetworkOPsImp::addAccountHistoryJob(SubAccountHistoryInfoWeak subInfo) #ifdef RIPPLED_REPORTING if (app_.config().reporting()) { - if (dynamic_cast( - &app_.getRelationalDBInterface())) + // Use a dynamic_cast to return DatabaseType::None + // on failure. + if (dynamic_cast(&app_.getRelationalDatabase())) { return DatabaseType::Postgres; } @@ -3375,16 +3407,18 @@ NetworkOPsImp::addAccountHistoryJob(SubAccountHistoryInfoWeak subInfo) } else { - if (dynamic_cast( - &app_.getRelationalDBInterface())) + // Use a dynamic_cast to return DatabaseType::None + // on failure. + if (dynamic_cast(&app_.getRelationalDatabase())) { return DatabaseType::Sqlite; } return DatabaseType::None; } #else - if (dynamic_cast( - &app_.getRelationalDBInterface())) + // Use a dynamic_cast to return DatabaseType::None + // on failure. + if (dynamic_cast(&app_.getRelationalDatabase())) { return DatabaseType::Sqlite; } @@ -3470,17 +3504,16 @@ NetworkOPsImp::addAccountHistoryJob(SubAccountHistoryInfoWeak subInfo) auto getMoreTxns = [&](std::uint32_t minLedger, std::uint32_t maxLedger, - std::optional - marker) + std::optional marker) -> std::optional>> { + RelationalDatabase::AccountTxs, + std::optional>> { switch (dbType) { case Postgres: { - auto db = static_cast( - &app_.getRelationalDBInterface()); - RelationalDBInterface::AccountTxArgs args; + auto db = static_cast( + &app_.getRelationalDatabase()); + RelationalDatabase::AccountTxArgs args; args.account = accountId; LedgerRange range{minLedger, maxLedger}; args.ledger = range; @@ -3496,7 +3529,7 @@ NetworkOPsImp::addAccountHistoryJob(SubAccountHistoryInfoWeak subInfo) } if (auto txns = - std::get_if( + std::get_if( &txResult.transactions); txns) { @@ -3512,9 +3545,9 @@ NetworkOPsImp::addAccountHistoryJob(SubAccountHistoryInfoWeak subInfo) } } case Sqlite: { - auto db = static_cast( - &app_.getRelationalDBInterface()); - RelationalDBInterface::AccountTxPageOptions options{ + auto db = static_cast( + &app_.getRelationalDatabase()); + RelationalDatabase::AccountTxPageOptions options{ accountId, minLedger, maxLedger, marker, 0, true}; return db->newestAccountTxPage(options); } @@ -3575,7 +3608,7 @@ NetworkOPsImp::addAccountHistoryJob(SubAccountHistoryInfoWeak subInfo) return; } - std::optional marker{}; + std::optional marker{}; while (!subInfo.index_->stopHistorical_) { auto dbResult = @@ -3875,6 +3908,16 @@ NetworkOPsImp::subLedger(InfoSub::ref isrListener, Json::Value& jvResult) .second; } +// <-- bool: true=added, false=already there +bool +NetworkOPsImp::subBookChanges(InfoSub::ref isrListener) +{ + std::lock_guard sl(mSubLock); + return mStreamMaps[sBookChanges] + .emplace(isrListener->getSeq(), isrListener) + .second; +} + // <-- bool: true=erased, false=was not there bool NetworkOPsImp::unsubLedger(std::uint64_t uSeq) @@ -3883,6 +3926,14 @@ NetworkOPsImp::unsubLedger(std::uint64_t uSeq) return mStreamMaps[sLedger].erase(uSeq); } +// <-- bool: true=erased, false=was not there +bool +NetworkOPsImp::unsubBookChanges(std::uint64_t uSeq) +{ + std::lock_guard sl(mSubLock); + return mStreamMaps[sBookChanges].erase(uSeq); +} + // <-- bool: true=added, false=already there bool NetworkOPsImp::subManifests(InfoSub::ref isrListener) diff --git a/src/ripple/app/misc/SHAMapStore.h b/src/ripple/app/misc/SHAMapStore.h index 7a999012c..c42e5f5a5 100644 --- a/src/ripple/app/misc/SHAMapStore.h +++ b/src/ripple/app/misc/SHAMapStore.h @@ -55,7 +55,7 @@ public: clampFetchDepth(std::uint32_t fetch_depth) const = 0; virtual std::unique_ptr - makeNodeStore(std::int32_t readThreads) = 0; + makeNodeStore(int readThreads) = 0; /** Highest ledger that may be deleted. */ virtual LedgerIndex diff --git a/src/ripple/app/misc/SHAMapStoreImp.cpp b/src/ripple/app/misc/SHAMapStoreImp.cpp index 56a817934..d5cb07792 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.cpp +++ b/src/ripple/app/misc/SHAMapStoreImp.cpp @@ -17,17 +17,18 @@ */ //============================================================================== +#include + #include #include -#include -#include -#include +#include +#include #include #include #include -#include - #include +#include +#include #include @@ -138,7 +139,7 @@ SHAMapStoreImp::SHAMapStoreImp( if (get_if_exists(section, "age_threshold_seconds", temp)) ageThreshold_ = std::chrono::seconds{temp}; if (get_if_exists(section, "recovery_wait_seconds", temp)) - recoveryWaitTime_.emplace(std::chrono::seconds{temp}); + recoveryWaitTime_ = std::chrono::seconds{temp}; get_if_exists(section, "advisory_delete", advisoryDelete_); @@ -166,7 +167,7 @@ SHAMapStoreImp::SHAMapStoreImp( } std::unique_ptr -SHAMapStoreImp::makeNodeStore(std::int32_t readThreads) +SHAMapStoreImp::makeNodeStore(int readThreads) { auto nscfg = app_.config().section(ConfigSection::nodeDatabase()); @@ -268,7 +269,7 @@ SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node) true); if (!(++nodeCount % checkHealthInterval_)) { - if (health()) + if (healthWait() == stopping) return false; } @@ -326,7 +327,7 @@ SHAMapStoreImp::run() bool const readyToRotate = validatedSeq >= lastRotated + deleteInterval_ && - canDelete_ >= lastRotated - 1 && !health(); + canDelete_ >= lastRotated - 1 && healthWait() == keepGoing; // Make sure we don't delete ledgers currently being // imported into the ShardStore @@ -358,47 +359,39 @@ SHAMapStoreImp::run() << ledgerMaster_->getValidatedLedgerAge().count() << 's'; clearPrior(lastRotated); - switch (health()) - { - case Health::stopping: - return; - case Health::unhealthy: - continue; - case Health::ok: - default:; - } + if (healthWait() == stopping) + return; JLOG(journal_.debug()) << "copying ledger " << validatedSeq; std::uint64_t nodeCount = 0; - validatedLedger->stateMap().snapShot(false)->visitNodes(std::bind( - &SHAMapStoreImp::copyNode, - this, - std::ref(nodeCount), - std::placeholders::_1)); - switch (health()) + + try { - case Health::stopping: - return; - case Health::unhealthy: - continue; - case Health::ok: - default:; + validatedLedger->stateMap().snapShot(false)->visitNodes( + std::bind( + &SHAMapStoreImp::copyNode, + this, + std::ref(nodeCount), + std::placeholders::_1)); } + catch (SHAMapMissingNode const& e) + { + JLOG(journal_.error()) + << "Missing node while copying ledger before rotate: " + << e.what(); + continue; + } + + if (healthWait() == stopping) + return; // Only log if we completed without a "health" abort JLOG(journal_.debug()) << "copied ledger " << validatedSeq << " nodecount " << nodeCount; JLOG(journal_.debug()) << "freshening caches"; freshenCaches(); - switch (health()) - { - case Health::stopping: - return; - case Health::unhealthy: - continue; - case Health::ok: - default:; - } + if (healthWait() == stopping) + return; // Only log if we completed without a "health" abort JLOG(journal_.debug()) << validatedSeq << " freshened caches"; @@ -408,15 +401,8 @@ SHAMapStoreImp::run() << validatedSeq << " new backend " << newBackend->getName(); clearCaches(validatedSeq); - switch (health()) - { - case Health::stopping: - return; - case Health::unhealthy: - continue; - case Health::ok: - default:; - } + if (healthWait() == stopping) + return; lastRotated = validatedSeq; @@ -580,7 +566,7 @@ SHAMapStoreImp::clearSql( min = *m; } - if (min > lastRotated || health() != Health::ok) + if (min > lastRotated || healthWait() == stopping) return; if (min == lastRotated) { @@ -601,11 +587,11 @@ SHAMapStoreImp::clearSql( JLOG(journal_.trace()) << "End: Delete up to " << deleteBatch_ << " rows with LedgerSeq < " << min << " from: " << TableName; - if (health()) + if (healthWait() == stopping) return; if (min < lastRotated) std::this_thread::sleep_for(backOff_); - if (health()) + if (healthWait() == stopping) return; } JLOG(journal_.debug()) << "finished deleting from: " << TableName; @@ -645,23 +631,21 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated) ledgerMaster_->clearPriorLedgers(lastRotated); JLOG(journal_.trace()) << "End: Clear internal ledgers up to " << lastRotated; - if (health()) + if (healthWait() == stopping) return; - RelationalDBInterfaceSqlite* iface = - dynamic_cast( - &app_.getRelationalDBInterface()); + SQLiteDatabase* const db = + dynamic_cast(&app_.getRelationalDatabase()); + + if (!db) + Throw("Failed to get relational database"); clearSql( lastRotated, "Ledgers", - [&iface]() -> std::optional { - return iface->getMinLedgerSeq(); - }, - [&iface](LedgerIndex min) -> void { - iface->deleteBeforeLedgerSeq(min); - }); - if (health()) + [db]() -> std::optional { return db->getMinLedgerSeq(); }, + [db](LedgerIndex min) -> void { db->deleteBeforeLedgerSeq(min); }); + if (healthWait() == stopping) return; if (!app_.config().useTxTables()) @@ -670,70 +654,48 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated) clearSql( lastRotated, "Transactions", - [&iface]() -> std::optional { - return iface->getTransactionsMinLedgerSeq(); + [&db]() -> std::optional { + return db->getTransactionsMinLedgerSeq(); }, - [&iface](LedgerIndex min) -> void { - iface->deleteTransactionsBeforeLedgerSeq(min); + [&db](LedgerIndex min) -> void { + db->deleteTransactionsBeforeLedgerSeq(min); }); - if (health()) + if (healthWait() == stopping) return; clearSql( lastRotated, "AccountTransactions", - [&iface]() -> std::optional { - return iface->getAccountTransactionsMinLedgerSeq(); + [&db]() -> std::optional { + return db->getAccountTransactionsMinLedgerSeq(); }, - [&iface](LedgerIndex min) -> void { - iface->deleteAccountTransactionsBeforeLedgerSeq(min); + [&db](LedgerIndex min) -> void { + db->deleteAccountTransactionsBeforeLedgerSeq(min); }); - if (health()) + if (healthWait() == stopping) return; } -SHAMapStoreImp::Health -SHAMapStoreImp::health() +SHAMapStoreImp::HealthResult +SHAMapStoreImp::healthWait() { + auto age = ledgerMaster_->getValidatedLedgerAge(); + OperatingMode mode = netOPs_->getOperatingMode(); + std::unique_lock lock(mutex_); + while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_)) { - std::lock_guard lock(mutex_); - if (stop_) - return Health::stopping; - } - if (!netOPs_) - return Health::ok; - assert(deleteInterval_); - - if (healthy_) - { - auto age = ledgerMaster_->getValidatedLedgerAge(); - OperatingMode mode = netOPs_->getOperatingMode(); - if (recoveryWaitTime_ && mode == OperatingMode::SYNCING && - age < ageThreshold_) - { - JLOG(journal_.warn()) - << "Waiting " << recoveryWaitTime_->count() - << "s for node to get back into sync with network. state: " - << app_.getOPs().strOperatingMode(mode, false) << ". age " - << age.count() << 's'; - std::this_thread::sleep_for(*recoveryWaitTime_); - - age = ledgerMaster_->getValidatedLedgerAge(); - mode = netOPs_->getOperatingMode(); - } - if (mode != OperatingMode::FULL || age > ageThreshold_) - { - JLOG(journal_.warn()) << "Not deleting. state: " - << app_.getOPs().strOperatingMode(mode, false) - << ". age " << age.count() << 's'; - healthy_ = false; - } + lock.unlock(); + JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count() + << "s for node to stabilize. state: " + << app_.getOPs().strOperatingMode(mode, false) + << ". age " << age.count() << 's'; + std::this_thread::sleep_for(recoveryWaitTime_); + age = ledgerMaster_->getValidatedLedgerAge(); + mode = netOPs_->getOperatingMode(); + lock.lock(); } - if (healthy_) - return Health::ok; - else - return Health::unhealthy; + return stop_ ? stopping : keepGoing; } void diff --git a/src/ripple/app/misc/SHAMapStoreImp.h b/src/ripple/app/misc/SHAMapStoreImp.h index e3528faaa..995ee0267 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.h +++ b/src/ripple/app/misc/SHAMapStoreImp.h @@ -22,8 +22,8 @@ #include #include -#include -#include +#include +#include #include #include @@ -40,8 +40,6 @@ class NetworkOPs; class SHAMapStoreImp : public SHAMapStore { private: - enum Health : std::uint8_t { ok = 0, stopping, unhealthy }; - class SavedStateDB { public: @@ -106,12 +104,11 @@ private: std::uint32_t deleteBatch_ = 100; std::chrono::milliseconds backOff_{100}; std::chrono::seconds ageThreshold_{60}; - /// If set, and the node is out of sync during an - /// online_delete health check, sleep the thread - /// for this time and check again so the node can - /// recover. + /// If the node is out of sync during an online_delete healthWait() + /// call, sleep the thread for this time, and continue checking until + /// recovery. /// See also: "recovery_wait_seconds" in rippled-example.cfg - std::optional recoveryWaitTime_; + std::chrono::seconds recoveryWaitTime_{5}; // these do not exist upon SHAMapStore creation, but do exist // as of run() or before @@ -136,7 +133,7 @@ public: } std::unique_ptr - makeNodeStore(std::int32_t readThreads) override; + makeNodeStore(int readThreads) override; LedgerIndex setCanDelete(LedgerIndex seq) override @@ -201,7 +198,7 @@ private: { dbRotating_->fetchNodeObject( key, 0, NodeStore::FetchType::synchronous, true); - if (!(++check % checkHealthInterval_) && health()) + if (!(++check % checkHealthInterval_) && healthWait() == stopping) return true; } @@ -225,16 +222,16 @@ private: void clearPrior(LedgerIndex lastRotated); - // If rippled is not healthy, defer rotate-delete. - // If already unhealthy, do not change state on further check. - // Assume that, once unhealthy, a necessary step has been - // aborted, so the online-delete process needs to restart - // at next ledger. - // If recoveryWaitTime_ is set, this may sleep to give rippled - // time to recover, so never call it from any thread other than - // the main "run()". - Health - health(); + /** + * This is a health check for online deletion that waits until rippled is + * stable before returning. It returns an indication of whether the server + * is stopping. + * + * @return Whether the server is stopping. + */ + enum HealthResult { stopping, keepGoing }; + [[nodiscard]] HealthResult + healthWait(); public: void diff --git a/src/ripple/app/misc/impl/AccountTxPaging.cpp b/src/ripple/app/misc/impl/AccountTxPaging.cpp index 5c1e80170..433463e28 100644 --- a/src/ripple/app/misc/impl/AccountTxPaging.cpp +++ b/src/ripple/app/misc/impl/AccountTxPaging.cpp @@ -31,7 +31,7 @@ namespace ripple { void convertBlobsToTxResult( - RelationalDBInterface::AccountTxs& to, + RelationalDatabase::AccountTxs& to, std::uint32_t ledger_index, std::string const& status, Blob const& rawTxn, diff --git a/src/ripple/app/misc/impl/AccountTxPaging.h b/src/ripple/app/misc/impl/AccountTxPaging.h index ad3c40e56..6b8f235b5 100644 --- a/src/ripple/app/misc/impl/AccountTxPaging.h +++ b/src/ripple/app/misc/impl/AccountTxPaging.h @@ -20,7 +20,7 @@ #ifndef RIPPLE_APP_MISC_IMPL_ACCOUNTTXPAGING_H_INCLUDED #define RIPPLE_APP_MISC_IMPL_ACCOUNTTXPAGING_H_INCLUDED -#include +#include #include #include #include @@ -31,7 +31,7 @@ namespace ripple { void convertBlobsToTxResult( - RelationalDBInterface::AccountTxs& to, + RelationalDatabase::AccountTxs& to, std::uint32_t ledger_index, std::string const& status, Blob const& rawTxn, diff --git a/src/ripple/app/misc/impl/AmendmentTable.cpp b/src/ripple/app/misc/impl/AmendmentTable.cpp index be59320be..93113af80 100644 --- a/src/ripple/app/misc/impl/AmendmentTable.cpp +++ b/src/ripple/app/misc/impl/AmendmentTable.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include @@ -425,7 +425,7 @@ AmendmentTableImpl::AmendmentTableImpl( } else // up-vote { - auto s = add(amend_hash, lock); + AmendmentState& s = add(amend_hash, lock); JLOG(j_.debug()) << "Amendment {" << *amendment_name << ", " << amend_hash << "} is upvoted."; diff --git a/src/ripple/app/misc/impl/Manifest.cpp b/src/ripple/app/misc/impl/Manifest.cpp index d5fcde19e..931b63215 100644 --- a/src/ripple/app/misc/impl/Manifest.cpp +++ b/src/ripple/app/misc/impl/Manifest.cpp @@ -18,7 +18,7 @@ //============================================================================== #include -#include +#include #include #include #include diff --git a/src/ripple/app/misc/impl/Transaction.cpp b/src/ripple/app/misc/impl/Transaction.cpp index ee391c7a9..9adef982d 100644 --- a/src/ripple/app/misc/impl/Transaction.cpp +++ b/src/ripple/app/misc/impl/Transaction.cpp @@ -21,9 +21,8 @@ #include #include #include -#include -#include -#include +#include +#include #include #include #include @@ -134,9 +133,15 @@ Transaction::load( Transaction::Locator Transaction::locate(uint256 const& id, Application& app) { - return dynamic_cast( - &app.getRelationalDBInterface()) - ->locateTransaction(id); + auto const db = + dynamic_cast(&app.getRelationalDatabase()); + + if (!db) + { + Throw("Failed to get relational database"); + } + + return db->locateTransaction(id); } std::variant< @@ -148,9 +153,14 @@ Transaction::load( std::optional> const& range, error_code_i& ec) { - return dynamic_cast( - &app.getRelationalDBInterface()) - ->getTransaction(id, range, ec); + auto const db = dynamic_cast(&app.getRelationalDatabase()); + + if (!db) + { + Throw("Failed to get relational database"); + } + + return db->getTransaction(id, range, ec); } // options 1 to include the date of the transaction diff --git a/src/ripple/app/misc/impl/TxQ.cpp b/src/ripple/app/misc/impl/TxQ.cpp index 0eeec8d62..59559cf24 100644 --- a/src/ripple/app/misc/impl/TxQ.cpp +++ b/src/ripple/app/misc/impl/TxQ.cpp @@ -78,8 +78,6 @@ increase(FeeLevel64 level, std::uint32_t increasePercent) ////////////////////////////////////////////////////////////////////////// -constexpr FeeLevel64 TxQ::baseLevel; - std::size_t TxQ::FeeMetrics::update( Application& app, diff --git a/src/ripple/app/paths/AccountCurrencies.cpp b/src/ripple/app/paths/AccountCurrencies.cpp index 2892ff869..18452725b 100644 --- a/src/ripple/app/paths/AccountCurrencies.cpp +++ b/src/ripple/app/paths/AccountCurrencies.cpp @@ -33,18 +33,23 @@ accountSourceCurrencies( if (includeXRP) currencies.insert(xrpCurrency()); - for (auto const& rspEntry : lrCache->getRippleLines(account)) + if (auto const lines = + lrCache->getRippleLines(account, LineDirection::outgoing)) { - auto& saBalance = rspEntry.getBalance(); - - // Filter out non - if (saBalance > beast::zero - // Have IOUs to send. - || (rspEntry.getLimitPeer() - // Peer extends credit. - && ((-saBalance) < rspEntry.getLimitPeer()))) // Credit left. + for (auto const& rspEntry : *lines) { - currencies.insert(saBalance.getCurrency()); + auto& saBalance = rspEntry.getBalance(); + + // Filter out non + if (saBalance > beast::zero + // Have IOUs to send. + || + (rspEntry.getLimitPeer() + // Peer extends credit. + && ((-saBalance) < rspEntry.getLimitPeer()))) // Credit left. + { + currencies.insert(saBalance.getCurrency()); + } } } @@ -64,12 +69,16 @@ accountDestCurrencies( currencies.insert(xrpCurrency()); // Even if account doesn't exist - for (auto const& rspEntry : lrCache->getRippleLines(account)) + if (auto const lines = + lrCache->getRippleLines(account, LineDirection::outgoing)) { - auto& saBalance = rspEntry.getBalance(); + for (auto const& rspEntry : *lines) + { + auto& saBalance = rspEntry.getBalance(); - if (saBalance < rspEntry.getLimit()) // Can take more - currencies.insert(saBalance.getCurrency()); + if (saBalance < rspEntry.getLimit()) // Can take more + currencies.insert(saBalance.getCurrency()); + } } currencies.erase(badCurrency()); diff --git a/src/ripple/app/paths/PathRequest.cpp b/src/ripple/app/paths/PathRequest.cpp index e5b15fd9d..02b46c81e 100644 --- a/src/ripple/app/paths/PathRequest.cpp +++ b/src/ripple/app/paths/PathRequest.cpp @@ -552,9 +552,16 @@ PathRequest::findPaths( continueCallback); mContext[issue] = ps; - auto& sourceAccount = !isXRP(issue.account) - ? issue.account - : isXRP(issue.currency) ? xrpAccount() : *raSrcAccount; + auto const& sourceAccount = [&] { + if (!isXRP(issue.account)) + return issue.account; + + if (isXRP(issue.currency)) + return xrpAccount(); + + return *raSrcAccount; + }(); + STAmount saMaxAmount = saSendMax.value_or( STAmount({issue.currency, sourceAccount}, 1u, 0, true)); @@ -675,10 +682,8 @@ PathRequest::doUpdate( destCurrencies.append(to_string(c)); } - newStatus[jss::source_account] = - app_.accountIDCache().toBase58(*raSrcAccount); - newStatus[jss::destination_account] = - app_.accountIDCache().toBase58(*raDstAccount); + newStatus[jss::source_account] = toBase58(*raSrcAccount); + newStatus[jss::destination_account] = toBase58(*raDstAccount); newStatus[jss::destination_amount] = saDstAmount.getJson(JsonOptions::none); newStatus[jss::full_reply] = !fast; @@ -748,6 +753,8 @@ PathRequest::doUpdate( jvStatus = newStatus; } + JLOG(m_journal.debug()) + << iIdentifier << " update finished " << (fast ? "fast" : "normal"); return newStatus; } diff --git a/src/ripple/app/paths/Pathfinder.cpp b/src/ripple/app/paths/Pathfinder.cpp index 4e81bebd3..556622ee7 100644 --- a/src/ripple/app/paths/Pathfinder.cpp +++ b/src/ripple/app/paths/Pathfinder.cpp @@ -708,6 +708,7 @@ int Pathfinder::getPathsOut( Currency const& currency, AccountID const& account, + LineDirection direction, bool isDstCurrency, AccountID const& dstAccount, std::function const& continueCallback) @@ -735,33 +736,37 @@ Pathfinder::getPathsOut( { count = app_.getOrderBookDB().getBookSize(issue); - for (auto const& rspEntry : mRLCache->getRippleLines(account)) + if (auto const lines = mRLCache->getRippleLines(account, direction)) { - if (currency != rspEntry.getLimit().getCurrency()) + for (auto const& rspEntry : *lines) { - } - else if ( - rspEntry.getBalance() <= beast::zero && - (!rspEntry.getLimitPeer() || - -rspEntry.getBalance() >= rspEntry.getLimitPeer() || - (bAuthRequired && !rspEntry.getAuth()))) - { - } - else if (isDstCurrency && dstAccount == rspEntry.getAccountIDPeer()) - { - count += 10000; // count a path to the destination extra - } - else if (rspEntry.getNoRipplePeer()) - { - // This probably isn't a useful path out - } - else if (rspEntry.getFreezePeer()) - { - // Not a useful path out - } - else - { - ++count; + if (currency != rspEntry.getLimit().getCurrency()) + { + } + else if ( + rspEntry.getBalance() <= beast::zero && + (!rspEntry.getLimitPeer() || + -rspEntry.getBalance() >= rspEntry.getLimitPeer() || + (bAuthRequired && !rspEntry.getAuth()))) + { + } + else if ( + isDstCurrency && dstAccount == rspEntry.getAccountIDPeer()) + { + count += 10000; // count a path to the destination extra + } + else if (rspEntry.getNoRipplePeer()) + { + // This probably isn't a useful path out + } + else if (rspEntry.getFreezePeer()) + { + // Not a useful path out + } + else + { + ++count; + } } } } @@ -791,8 +796,8 @@ Pathfinder::addPathsForType( PathType const& pathType, std::function const& continueCallback) { - JLOG(j_.warn()) << "addPathsForType " - << CollectionAndDelimiter(pathType, ", "); + JLOG(j_.debug()) << "addPathsForType " + << CollectionAndDelimiter(pathType, ", "); // See if the set of paths for this type already exists. auto it = mPaths.find(pathType); if (it != mPaths.end()) @@ -976,117 +981,128 @@ Pathfinder::addLink( bool const bIsNoRippleOut(isNoRippleOut(currentPath)); bool const bDestOnly(addFlags & afAC_LAST); - auto& rippleLines(mRLCache->getRippleLines(uEndAccount)); - - AccountCandidates candidates; - candidates.reserve(rippleLines.size()); - - for (auto const& rs : rippleLines) + if (auto const lines = mRLCache->getRippleLines( + uEndAccount, + bIsNoRippleOut ? LineDirection::incoming + : LineDirection::outgoing)) { - if (continueCallback && !continueCallback()) - return; - auto const& acct = rs.getAccountIDPeer(); + auto& rippleLines = *lines; - if (hasEffectiveDestination && (acct == mDstAccount)) - { - // We skipped the gateway - continue; - } + AccountCandidates candidates; + candidates.reserve(rippleLines.size()); - bool bToDestination = acct == mEffectiveDst; - - if (bDestOnly && !bToDestination) - { - continue; - } - - if ((uEndCurrency == rs.getLimit().getCurrency()) && - !currentPath.hasSeen(acct, uEndCurrency, acct)) - { - // path is for correct currency and has not been seen - if (rs.getBalance() <= beast::zero && - (!rs.getLimitPeer() || - -rs.getBalance() >= rs.getLimitPeer() || - (bRequireAuth && !rs.getAuth()))) - { - // path has no credit - } - else if (bIsNoRippleOut && rs.getNoRipple()) - { - // Can't leave on this path - } - else if (bToDestination) - { - // destination is always worth trying - if (uEndCurrency == mDstAmount.getCurrency()) - { - // this is a complete path - if (!currentPath.empty()) - { - JLOG(j_.trace()) - << "complete path found ae: " - << currentPath.getJson( - JsonOptions::none); - addUniquePath(mCompletePaths, currentPath); - } - } - else if (!bDestOnly) - { - // this is a high-priority candidate - candidates.push_back( - {AccountCandidate::highPriority, acct}); - } - } - else if (acct == mSrcAccount) - { - // going back to the source is bad - } - else - { - // save this candidate - int out = getPathsOut( - uEndCurrency, - acct, - bIsEndCurrency, - mEffectiveDst, - continueCallback); - if (out) - candidates.push_back({out, acct}); - } - } - } - - if (!candidates.empty()) - { - std::sort( - candidates.begin(), - candidates.end(), - std::bind( - compareAccountCandidate, - mLedger->seq(), - std::placeholders::_1, - std::placeholders::_2)); - - int count = candidates.size(); - // allow more paths from source - if ((count > 10) && (uEndAccount != mSrcAccount)) - count = 10; - else if (count > 50) - count = 50; - - auto it = candidates.begin(); - while (count-- != 0) + for (auto const& rs : rippleLines) { if (continueCallback && !continueCallback()) return; - // Add accounts to incompletePaths - STPathElement pathElement( - STPathElement::typeAccount, - it->account, - uEndCurrency, - it->account); - incompletePaths.assembleAdd(currentPath, pathElement); - ++it; + auto const& acct = rs.getAccountIDPeer(); + LineDirection const direction = rs.getDirectionPeer(); + + if (hasEffectiveDestination && (acct == mDstAccount)) + { + // We skipped the gateway + continue; + } + + bool bToDestination = acct == mEffectiveDst; + + if (bDestOnly && !bToDestination) + { + continue; + } + + if ((uEndCurrency == rs.getLimit().getCurrency()) && + !currentPath.hasSeen(acct, uEndCurrency, acct)) + { + // path is for correct currency and has not been + // seen + if (rs.getBalance() <= beast::zero && + (!rs.getLimitPeer() || + -rs.getBalance() >= rs.getLimitPeer() || + (bRequireAuth && !rs.getAuth()))) + { + // path has no credit + } + else if (bIsNoRippleOut && rs.getNoRipple()) + { + // Can't leave on this path + } + else if (bToDestination) + { + // destination is always worth trying + if (uEndCurrency == mDstAmount.getCurrency()) + { + // this is a complete path + if (!currentPath.empty()) + { + JLOG(j_.trace()) + << "complete path found ae: " + << currentPath.getJson( + JsonOptions::none); + addUniquePath( + mCompletePaths, currentPath); + } + } + else if (!bDestOnly) + { + // this is a high-priority candidate + candidates.push_back( + {AccountCandidate::highPriority, acct}); + } + } + else if (acct == mSrcAccount) + { + // going back to the source is bad + } + else + { + // save this candidate + int out = getPathsOut( + uEndCurrency, + acct, + direction, + bIsEndCurrency, + mEffectiveDst, + continueCallback); + if (out) + candidates.push_back({out, acct}); + } + } + } + + if (!candidates.empty()) + { + std::sort( + candidates.begin(), + candidates.end(), + std::bind( + compareAccountCandidate, + mLedger->seq(), + std::placeholders::_1, + std::placeholders::_2)); + + int count = candidates.size(); + // allow more paths from source + if ((count > 10) && (uEndAccount != mSrcAccount)) + count = 10; + else if (count > 50) + count = 50; + + auto it = candidates.begin(); + while (count-- != 0) + { + if (continueCallback && !continueCallback()) + return; + // Add accounts to incompletePaths + STPathElement pathElement( + STPathElement::typeAccount, + it->account, + uEndCurrency, + it->account); + incompletePaths.assembleAdd( + currentPath, pathElement); + ++it; + } } } } diff --git a/src/ripple/app/paths/Pathfinder.h b/src/ripple/app/paths/Pathfinder.h index 45da9ec11..375e5e246 100644 --- a/src/ripple/app/paths/Pathfinder.h +++ b/src/ripple/app/paths/Pathfinder.h @@ -144,6 +144,7 @@ private: getPathsOut( Currency const& currency, AccountID const& account, + LineDirection direction, bool isDestCurrency, AccountID const& dest, std::function const& continueCallback); diff --git a/src/ripple/app/paths/RippleLineCache.cpp b/src/ripple/app/paths/RippleLineCache.cpp index a0b26ba28..2487924ff 100644 --- a/src/ripple/app/paths/RippleLineCache.cpp +++ b/src/ripple/app/paths/RippleLineCache.cpp @@ -26,39 +26,101 @@ namespace ripple { RippleLineCache::RippleLineCache( std::shared_ptr const& ledger, beast::Journal j) - : journal_(j) + : ledger_(ledger), journal_(j) { - mLedger = ledger; - - JLOG(journal_.debug()) << "RippleLineCache created for ledger " - << mLedger->info().seq; + JLOG(journal_.debug()) << "created for ledger " << ledger_->info().seq; } RippleLineCache::~RippleLineCache() { - JLOG(journal_.debug()) << "~RippleLineCache destroyed for ledger " - << mLedger->info().seq << " with " << lines_.size() - << " accounts"; + JLOG(journal_.debug()) << "destroyed for ledger " << ledger_->info().seq + << " with " << lines_.size() << " accounts and " + << totalLineCount_ << " distinct trust lines."; } -std::vector const& -RippleLineCache::getRippleLines(AccountID const& accountID) +std::shared_ptr> +RippleLineCache::getRippleLines( + AccountID const& accountID, + LineDirection direction) { - AccountKey key(accountID, hasher_(accountID)); + auto const hash = hasher_(accountID); + AccountKey key(accountID, direction, hash); + AccountKey otherkey( + accountID, + direction == LineDirection::outgoing ? LineDirection::incoming + : LineDirection::outgoing, + hash); std::lock_guard sl(mLock); - auto [it, inserted] = lines_.emplace(key, std::vector()); + auto [it, inserted] = [&]() { + if (auto otheriter = lines_.find(otherkey); otheriter != lines_.end()) + { + // The whole point of using the direction flag is to reduce the + // number of trust line objects held in memory. Ensure that there is + // only a single set of trustlines in the cache per account. + auto const size = otheriter->second ? otheriter->second->size() : 0; + JLOG(journal_.info()) + << "Request for " + << (direction == LineDirection::outgoing ? "outgoing" + : "incoming") + << " trust lines for account " << accountID << " found " << size + << (direction == LineDirection::outgoing ? " incoming" + : " outgoing") + << " trust lines. " + << (direction == LineDirection::outgoing + ? "Deleting the subset of incoming" + : "Returning the superset of outgoing") + << " trust lines. "; + if (direction == LineDirection::outgoing) + { + // This request is for the outgoing set, but there is already a + // subset of incoming lines in the cache. Erase that subset + // to be replaced by the full set. The full set will be built + // below, and will be returned, if needed, on subsequent calls + // for either value of outgoing. + assert(size <= totalLineCount_); + totalLineCount_ -= size; + lines_.erase(otheriter); + } + else + { + // This request is for the incoming set, but there is + // already a superset of the outgoing trust lines in the cache. + // The path finding engine will disregard the non-rippling trust + // lines, so to prevent them from being stored twice, return the + // outgoing set. + key = otherkey; + return std::pair{otheriter, false}; + } + } + return lines_.emplace(key, nullptr); + }(); if (inserted) - it->second = PathFindTrustLine::getItems(accountID, *mLedger); + { + assert(it->second == nullptr); + auto lines = + PathFindTrustLine::getItems(accountID, *ledger_, direction); + if (lines.size()) + { + it->second = std::make_shared>( + std::move(lines)); + totalLineCount_ += it->second->size(); + } + } - JLOG(journal_.debug()) << "RippleLineCache getRippleLines for ledger " - << mLedger->info().seq << " found " - << it->second.size() << " lines for " - << (inserted ? "new " : "existing ") << accountID - << " out of a total of " << lines_.size() - << " accounts"; + assert(!it->second || (it->second->size() > 0)); + auto const size = it->second ? it->second->size() : 0; + JLOG(journal_.trace()) << "getRippleLines for ledger " + << ledger_->info().seq << " found " << size + << (key.direction_ == LineDirection::outgoing + ? " outgoing" + : " incoming") + << " lines for " << (inserted ? "new " : "existing ") + << accountID << " out of a total of " + << lines_.size() << " accounts and " + << totalLineCount_ << " trust lines"; return it->second; } diff --git a/src/ripple/app/paths/RippleLineCache.h b/src/ripple/app/paths/RippleLineCache.h index e7a7e0f74..590c50082 100644 --- a/src/ripple/app/paths/RippleLineCache.h +++ b/src/ripple/app/paths/RippleLineCache.h @@ -44,27 +44,43 @@ public: std::shared_ptr const& getLedger() const { - return mLedger; + return ledger_; } - std::vector const& - getRippleLines(AccountID const& accountID); + /** Find the trust lines associated with an account. + + @param accountID The account + @param direction Whether the account is an "outgoing" link on the path. + "Outgoing" is defined as the source account, or an account found via a + trustline that has rippling enabled on the @accountID's side. If an + account is "outgoing", all trust lines will be returned. If an account is + not "outgoing", then any trust lines that don't have rippling enabled are + not usable, so only return trust lines that have rippling enabled on + @accountID's side. + @return Returns a vector of the usable trust lines. + */ + std::shared_ptr> + getRippleLines(AccountID const& accountID, LineDirection direction); private: std::mutex mLock; ripple::hardened_hash<> hasher_; - std::shared_ptr mLedger; + std::shared_ptr ledger_; beast::Journal journal_; struct AccountKey final : public CountedObject { AccountID account_; + LineDirection direction_; std::size_t hash_value_; - AccountKey(AccountID const& account, std::size_t hash) - : account_(account), hash_value_(hash) + AccountKey( + AccountID const& account, + LineDirection direction, + std::size_t hash) + : account_(account), direction_(direction), hash_value_(hash) { } @@ -76,7 +92,8 @@ private: bool operator==(AccountKey const& lhs) const { - return hash_value_ == lhs.hash_value_ && account_ == lhs.account_; + return hash_value_ == lhs.hash_value_ && account_ == lhs.account_ && + direction_ == lhs.direction_; } std::size_t @@ -97,8 +114,17 @@ private: }; }; - hash_map, AccountKey::Hash> + // Use a shared_ptr so entries can be removed from the map safely. + // Even though a shared_ptr to a vector will take more memory just a vector, + // most accounts are not going to have any entries (estimated over 90%), so + // vectors will not need to be created for them. This should lead to far + // less memory usage overall. + hash_map< + AccountKey, + std::shared_ptr>, + AccountKey::Hash> lines_; + std::size_t totalLineCount_ = 0; }; } // namespace ripple diff --git a/src/ripple/app/paths/TrustLine.cpp b/src/ripple/app/paths/TrustLine.cpp index f7c25dd24..2d7e5b241 100644 --- a/src/ripple/app/paths/TrustLine.cpp +++ b/src/ripple/app/paths/TrustLine.cpp @@ -63,26 +63,38 @@ PathFindTrustLine::makeItem( namespace detail { template std::vector -getTrustLineItems(AccountID const& accountID, ReadView const& view) +getTrustLineItems( + AccountID const& accountID, + ReadView const& view, + LineDirection direction = LineDirection::outgoing) { std::vector items; forEachItem( view, accountID, - [&items, &accountID](std::shared_ptr const& sleCur) { + [&items, &accountID, &direction]( + std::shared_ptr const& sleCur) { auto ret = T::makeItem(accountID, sleCur); - if (ret) + if (ret && + (direction == LineDirection::outgoing || !ret->getNoRipple())) items.push_back(std::move(*ret)); }); + // This list may be around for a while, so free up any unneeded + // capacity + items.shrink_to_fit(); return items; } } // namespace detail std::vector -PathFindTrustLine::getItems(AccountID const& accountID, ReadView const& view) +PathFindTrustLine::getItems( + AccountID const& accountID, + ReadView const& view, + LineDirection direction) { - return detail::getTrustLineItems(accountID, view); + return detail::getTrustLineItems( + accountID, view, direction); } RPCTrustLine::RPCTrustLine( diff --git a/src/ripple/app/paths/TrustLine.h b/src/ripple/app/paths/TrustLine.h index 44c7f3f10..29430c0d3 100644 --- a/src/ripple/app/paths/TrustLine.h +++ b/src/ripple/app/paths/TrustLine.h @@ -31,6 +31,15 @@ namespace ripple { +/** Describes how an account was found in a path, and how to find the next set +of paths. "Outgoing" is defined as the source account, or an account found via a +trustline that has rippling enabled on the account's side. +"Incoming" is defined as an account found via a trustline that has rippling +disabled on the account's side. Any trust lines for an incoming account that +have rippling disabled are unusable in paths. +*/ +enum class LineDirection : bool { incoming = false, outgoing = true }; + /** Wraps a trust line SLE for convenience. The complication of trust lines is that there is a "low" account and a "high" account. This wraps the @@ -109,6 +118,20 @@ public: return mFlags & (!mViewLowest ? lsfLowNoRipple : lsfHighNoRipple); } + LineDirection + getDirection() const + { + return getNoRipple() ? LineDirection::incoming + : LineDirection::outgoing; + } + + LineDirection + getDirectionPeer() const + { + return getNoRipplePeer() ? LineDirection::incoming + : LineDirection::outgoing; + } + /** Have we set the freeze flag on our peer */ bool getFreeze() const @@ -184,7 +207,10 @@ public: makeItem(AccountID const& accountID, std::shared_ptr const& sle); static std::vector - getItems(AccountID const& accountID, ReadView const& view); + getItems( + AccountID const& accountID, + ReadView const& view, + LineDirection direction); }; // This wrapper is used for the `AccountLines` command and includes the quality diff --git a/src/ripple/app/paths/impl/BookStep.cpp b/src/ripple/app/paths/impl/BookStep.cpp index a80ee13f8..a6b2c5961 100644 --- a/src/ripple/app/paths/impl/BookStep.cpp +++ b/src/ripple/app/paths/impl/BookStep.cpp @@ -1125,10 +1125,10 @@ bookStepEqual(Step const& step, ripple::Book const& book) bool const inXRP = isXRP(book.in.currency); bool const outXRP = isXRP(book.out.currency); if (inXRP && outXRP) - return equalHelper< - XRPAmount, - XRPAmount, - BookPaymentStep>(step, book); + { + assert(0); + return false; // no such thing as xrp/xrp book step + } if (inXRP && !outXRP) return equalHelper< XRPAmount, diff --git a/src/ripple/app/rdb/Download.h b/src/ripple/app/rdb/Download.h new file mode 100644 index 000000000..b72b5ec57 --- /dev/null +++ b/src/ripple/app/rdb/Download.h @@ -0,0 +1,79 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_RDB_DOWNLOAD_H_INCLUDED +#define RIPPLE_APP_RDB_DOWNLOAD_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { + +/** + * @brief openDatabaseBodyDb Opens a database that will store the contents of a + * file being downloaded, returns its descriptor, and starts a new + * download process or continues an existing one. + * @param setup Path to the database and other opening parameters. + * @param path Path of the new file to download. + * @return Pair containing a unique pointer to the database and the amount of + * bytes already downloaded if a download is being continued. + */ +std::pair, std::optional> +openDatabaseBodyDb( + DatabaseCon::Setup const& setup, + boost::filesystem::path const& path); + +/** + * @brief databaseBodyDoPut Saves a new fragment of a downloaded file. + * @param session Session with the database. + * @param data Downloaded fragment of file data to save. + * @param path Path to the file currently being downloaded. + * @param fileSize Size of the portion of the file already downloaded. + * @param part The index of the most recently updated database row. + * @param maxRowSizePad A constant padding value that accounts for other data + * stored in each row of the database. + * @return Index of the most recently updated database row. + */ +std::uint64_t +databaseBodyDoPut( + soci::session& session, + std::string const& data, + std::string const& path, + std::uint64_t fileSize, + std::uint64_t part, + std::uint16_t maxRowSizePad); + +/** + * @brief databaseBodyFinish Finishes the download process and writes the file + * to disk. + * @param session Session with the database. + * @param fout Opened file into which the downloaded data from the database will + * be written. + */ +void +databaseBodyFinish(soci::session& session, std::ofstream& fout); + +} // namespace ripple + +#endif diff --git a/src/ripple/app/rdb/PeerFinder.h b/src/ripple/app/rdb/PeerFinder.h new file mode 100644 index 000000000..06cd80f67 --- /dev/null +++ b/src/ripple/app/rdb/PeerFinder.h @@ -0,0 +1,76 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_RDB_PEERFINDER_H_INCLUDED +#define RIPPLE_APP_RDB_PEERFINDER_H_INCLUDED + +#include +#include +#include + +namespace ripple { + +/** + * @brief initPeerFinderDB Opens a session with the peer finder database. + * @param session Session with the peer finder database. + * @param config Path to the database and other opening parameters. + * @param j Journal. + */ +void +initPeerFinderDB( + soci::session& session, + BasicConfig const& config, + beast::Journal j); + +/** + * @brief updatePeerFinderDB Updates the peer finder database to a new version. + * @param session Session with the database. + * @param currentSchemaVersion New version of the database. + * @param j Journal. + */ +void +updatePeerFinderDB( + soci::session& session, + int currentSchemaVersion, + beast::Journal j); + +/** + * @brief readPeerFinderDB Reads all entries from the peer finder database and + * invokes the given callback for each entry. + * @param session Session with the database. + * @param func Callback to invoke for each entry. + */ +void +readPeerFinderDB( + soci::session& session, + std::function const& func); + +/** + * @brief savePeerFinderDB Saves a new entry to the peer finder database. + * @param session Session with the database. + * @param v Entry to save which contains information about a new peer. + */ +void +savePeerFinderDB( + soci::session& session, + std::vector const& v); + +} // namespace ripple + +#endif diff --git a/src/ripple/app/rdb/README.md b/src/ripple/app/rdb/README.md new file mode 100644 index 000000000..1a68a1ae5 --- /dev/null +++ b/src/ripple/app/rdb/README.md @@ -0,0 +1,102 @@ +# Relational Database Interface + +The guiding principles of the Relational Database Interface are summarized below: + +* All hard-coded SQL statements should be stored in the [files](#source-files) under the `ripple/app/rdb` directory. With the exception of test modules, no hard-coded SQL should be added to any other file in rippled. +* The base class `RelationalDatabase` is inherited by derived classes that each provide an interface for operating on distinct relational database systems. +* For future use, the shard store will be used if the node store is absent. + +## Overview + +Firstly, the interface `RelationalDatabase` is inherited by the classes `SQLiteDatabase` and `PostgresDatabase` which are used to operate the software's main data store (for storing transactions, accounts, ledgers, etc.). Secondly, the files under the `detail` directory provide supplementary functions that are used by these derived classes to access the underlying databases. Lastly, the remaining files in the interface (located at the top level of the module) are used by varied parts of the software to access any secondary relational databases. + +## Configuration + +The config section `[relational_db]` has a property named `backend` whose value designates which database implementation will be used for node or shard databases. Presently the only valid value for this property is `sqlite`: + +``` +[relational_db] +backend=sqlite +``` + +## Source Files + +The Relational Database Interface consists of the following directory structure (as of November 2021): + +``` +src/ripple/app/rdb/ +├── backend +│   ├── detail +│   │   ├── impl +│   │   │   ├── Node.cpp +│   │   │   └── Shard.cpp +│   │   ├── Node.h +│   │   └── Shard.h +│   ├── impl +│   │   ├── PostgresDatabase.cpp +│   │   └── SQLiteDatabase.cpp +│   ├── PostgresDatabase.h +│   └── SQLiteDatabase.h +├── impl +│   ├── Download.cpp +│   ├── PeerFinder.cpp +│   ├── RelationalDatabase.cpp +│   ├── ShardArchive.cpp +│   ├── State.cpp +│   ├── UnitaryShard.cpp +│   ├── Vacuum.cpp +│   └── Wallet.cpp +├── Download.h +├── PeerFinder.h +├── RelationalDatabase.h +├── README.md +├── ShardArchive.h +├── State.h +├── UnitaryShard.h +├── Vacuum.h +└── Wallet.h +``` + +### File Contents +| File | Contents | +| ----------- | ----------- | +| `Node.[h\|cpp]` | Defines/Implements methods used by `SQLiteDatabase` for interacting with SQLite node databases| +| `Shard.[h\|cpp]` | Defines/Implements methods used by `SQLiteDatabase` for interacting with SQLite shard databases | +| `PostgresDatabase.[h\|cpp]` | Defines/Implements the class `PostgresDatabase`/`PostgresDatabaseImp` which inherits from `RelationalDatabase` and is used to operate on the main stores | +|`SQLiteDatabase.[h\|cpp]`| Defines/Implements the class `SQLiteDatabase`/`SQLiteDatabaseImp` which inherits from `RelationalDatabase` and is used to operate on the main stores | +| `Download.[h\|cpp]` | Defines/Implements methods for persisting file downloads to a SQLite database | +| `PeerFinder.[h\|cpp]` | Defines/Implements methods for interacting with the PeerFinder SQLite database | +|`RelationalDatabase.cpp`| Implements the static method `RelationalDatabase::init` which is used to initialize an instance of `RelationalDatabase` | +| `RelationalDatabase.h` | Defines the abstract class `RelationalDatabase`, the primary class of the Relational Database Interface | +| `ShardArchive.[h\|cpp]` | Defines/Implements methods used by `ShardArchiveHandler` for interacting with SQLite databases containing metadata regarding shard downloads | +| `State.[h\|cpp]` | Defines/Implements methods for interacting with the State SQLite database which concerns ledger deletion and database rotation | +| `UnitaryShard.[h\|cpp]` | Defines/Implements methods used by a unitary instance of `Shard` for interacting with the various SQLite databases thereof. These files are distinct from `Shard.[h\|cpp]` which contain methods used by `SQLiteDatabaseImp` | +| `Vacuum.[h\|cpp]` | Defines/Implements a method for performing the `VACUUM` operation on SQLite databases | +| `Wallet.[h\|cpp]` | Defines/Implements methods for interacting with Wallet SQLite databases | + +## Classes + +The abstract class `RelationalDatabase` is the primary class of the Relational Database Interface and is defined in the eponymous header file. This class provides a static method `init()` which, when invoked, creates a concrete instance of a derived class whose type is specified by the system configuration. All other methods in the class are virtual. Presently there exist two classes that derive from `RelationalDatabase`, namely `SQLiteDatabase` and `PostgresDatabase`. + +## Database Methods + +The Relational Database Interface provides three categories of methods for interacting with databases: + +* Free functions for interacting with SQLite databases used by various components of the software. These methods feature a `soci::session` parameter which facilitates connecting to SQLite databases, and are defined and implemented in the following files: + + * `Download.[h\|cpp]` + * `PeerFinder.[h\|cpp]` + * `ShardArchive.[h\|cpp]` + * `State.[h\|cpp]` + * `UnitaryShard.[h\|cpp]` + * `Vacuum.[h\|cpp]` + * `Wallet.[h\|cpp]` + + +* Free functions used exclusively by `SQLiteDatabaseImp` for interacting with SQLite databases owned by the node store or shard store. Unlike the free functions in the files listed above, these are not intended to be invoked directly by clients. Rather, these methods are invoked by derived instances of `RelationalDatabase`. These methods are defined in the following files: + + * `Node.[h|cpp]` + * `Shard.[h|cpp]` + + +* Member functions of `RelationalDatabase`, `SQLiteDatabase`, and `PostgresDatabase` which are used to access the main stores (node store, shard store). The `SQLiteDatabase` class will access the node store by default, but will use shard databases if the node store is not present and the shard store is available. The class `PostgresDatabase` uses only the node store. diff --git a/src/ripple/app/rdb/RelationalDBInterface.md b/src/ripple/app/rdb/RelationalDBInterface.md deleted file mode 100644 index 302c9befe..000000000 --- a/src/ripple/app/rdb/RelationalDBInterface.md +++ /dev/null @@ -1,288 +0,0 @@ -# Relational Database Interface - -Here are main principles of Relational DB interface: - -1) All SQL hard code is in the files described below in Files section. -No hard-coded SQL should be added to any other file in rippled, except related -to tests for specific SQL implementations. -2) Pure interface class `RelationalDBInterface` can have several -implementations for different relational database types. -3) For future use, if the node database is absent, then shard databases will -be used. - -## Configuration - -Section `[relational_db]` of the configuration file contains parameter -`backend`. The value of this parameter is the name of relational database -implementation used for node or shard databases. At the present, the only valid -value of this parameter is `sqlite`. - -## Files - -The following source files are related to Relational DB interface: - -- `ripple/app/rdb/RelationalDBInterface.h` - definition of main pure class of -the interface, `RelationalDBInterface`; -- `ripple/app/rdb/impl/RelationalDBInterface.cpp` - implementation of static -method `init()` of the class `RelationalDBInterface`; -- `ripple/app/rdb/backend/RelationalDBInterfaceSqlite.h` - definition of pure -class `RelationalDBInterfaceSqlite` derived from `RelationalDBInterface`; -this is base class for sqlite implementation of the interface; -- `ripple/app/rdb/backend/RelationalDBInterfaceSqlite.cpp` - implementation of -`RelationalDBInterfaceSqlite`-derived class for the case of sqlite databases; -- `ripple/app/rdb/backend/RelationalDBInterfacePostgres.h` - definition of pure -class `RelationalDBInterfacePostgres` derived from `RelationalDBInterface`; -this is base class for postgres implementation of the interface; -- `ripple/app/rdb/backend/RelationalDBInterfacePostgres.cpp` - implementation -of `RelationalDBInterfacePostgres`-derived class for the case of postgres -databases; -- `ripple/app/rdb/RelationalDBInterface_global.h` - definitions of global -methods for all sqlite databases except of node and shard; -- `ripple/app/rdb/impl/RelationalDBInterface_global.cpp` - implementations of -global methods for all sqlite databases except of node and shard; -- `ripple/app/rdb/RelationalDBInterface_nodes.h` - definitions of global -methods for sqlite node databases; -- `ripple/app/rdb/impl/RelationalDBInterface_nodes.cpp` - implementations of -global methods for sqlite node databases; -- `ripple/app/rdb/RelationalDBInterface_shards.h` - definitions of global -methods for sqlite shard databases; -- `ripple/app/rdb/impl/RelationalDBInterface_shards.cpp` - implementations of -global methods for sqlite shard databases; -- `ripple/app/rdb/RelationalDBInterface_postgres.h` - definitions of internal -methods for postgres databases; -- `ripple/app/rdb/impl/RelationalDBInterface_postgres.cpp` - implementations of -internal methods for postgres databases; - -## Classes - -The main class of the interface is `class RelationalDBInterface`. It is defined -in the file `RelationalDBInterface.h`. This class has static method `init()` -which allow to create proper `RelationalDBInterface`-derived class specified -in the config. All other methods are pure virtual. These methods do not use -database as a parameter. It assumed that implementation of class derived from -`RelationalDBInterface` holds all database pointers inside and uses appropriate -databases (nodes or shards) to get return values required by each method. - -At the present, there are two implementations of the derived classes - -`class RelationalDBInterfaceSqlite` for sqlite database (it is located in the -file `RelationalDBInterfaceSqlite.cpp`) and -`class RelationalDBInterfacePostgres` for postgres database (it is located in -the file `RelationalDBInterfacePostgres.cpp`) - -## Methods - -There are 3 types of methods for SQL interface: - -1) Global methods for work with all databases except of node. In particular, -methods related to shards datavases only. These methods are sqlite-specific. -They use `soci::session` as database pointer parameter. Defined and -implemented in files `RelationalDBInterface_global.*` and -`RelationalDBInterface_shard.*`. - -2) Global methods for work with node databases, and also with shard databases. -For sqlite case, these methods are internal for `RelationalDBInterfaceSqlite` -implementation of the class `RelationalDBInterface`. They use `soci::session` -as database pointer parameter. Defined and implemented in files -`RelationalDBInterface_nodes.*`. For postgres case, these methods are internal -for `RelationalDBInterfacePostgres` implementation of the class -`RelationalDBInterface`. They use `std::shared_ptr` as database pointer -parameter. Defined and implemented in files `RelationalDBInterface_postgres.*`. - -3) Virtual methods of class `RelationalDBInterface` and also derived classes -`RelationalDBInterfaceSqlite` and `RelationalDBInterfacePostgres`. -Calling such a method resulted in calling corresponding method from -`RelationalDBInterface`-derived class. For sqlite case, such a method tries to -retrieve information from node database, and if this database not exists - then -from shard databases. For both node and shard databases, calls to global -methods of type 2) performed. For postgres case, such a method retrieves -information only from node database by calling a global method of type 2). - -## Methods lists - -### Type 1 methods - -#### Files RelationalDBInterface_global.* - -Wallet DB methods: -``` -makeWalletDB -makeTestWalletDB -getManifests -saveManifests -addValidatorManifest -getNodeIdentity -getPeerReservationTable -insertPeerReservation -deletePeerReservation -createFeatureVotes -readAmendments -voteAmendment -``` - -State DB methods: -``` -initStateDB -getCanDelete -setCanDelete -getSavedState -setSavedState -setLastRotated -``` - -DatabaseBody DB methods: -``` -openDatabaseBodyDb -databaseBodyDoPut -databaseBodyFinish -``` - -Vacuum DB method: -``` -doVacuumDB -``` - -PeerFinder DB methods: -``` -initPeerFinderDB -updatePeerFinderDB -readPeerFinderDB -savePeerFinderDB -``` - -#### Files RelationalDBInterface_shards.* - -Shards DB methods: -``` -makeShardCompleteLedgerDBs -makeShardIncompleteLedgerDBs -updateLedgerDBs -``` - -Shard acquire DB methods: -``` -makeAcquireDB -insertAcquireDBIndex -selectAcquireDBLedgerSeqs -selectAcquireDBLedgerSeqsHash -updateAcquireDB -``` - -Shard archive DB methods: -``` -makeArchiveDB -readArchiveDB -insertArchiveDB -deleteFromArchiveDB -dropArchiveDB -``` - -### Type 2 methods - -#### Files RelationalDBInterface_nodes.* - -``` -makeLedgerDBs -getMinLedgerSeq -getMaxLedgerSeq -deleteByLedgerSeq -deleteBeforeLedgerSeq -getRows -getRowsMinMax -saveValidatedLedger -getLedgerInfoByIndex -getOldestLedgerInfo -getNewestLedgerInfo -getLimitedOldestLedgerInfo -getLimitedNewestLedgerInfo -getLedgerInfoByHash -getHashByIndex -getHashesByIndex -getHashesByIndex -getTxHistory -getOldestAccountTxs -getNewestAccountTxs -getOldestAccountTxsB -getNewestAccountTxsB -oldestAccountTxPage -newestAccountTxPage -getTransaction -DbHasSpace -``` - -#### Files RelationalDBInterface_postgres.* - -``` -getMinLedgerSeq -getMaxLedgerSeq -getCompleteLedgers -getValidatedLedgerAge -getNewestLedgerInfo -getLedgerInfoByIndex -getLedgerInfoByHash -getHashByIndex -getHashesByIndex -getTxHashes -getAccountTx -locateTransaction -writeLedgerAndTransactions -getTxHistory -``` - -### Type 3 methods - -#### Files RelationalDBInterface.* - -``` -init -getMinLedgerSeq -getMaxLedgerSeq -getLedgerInfoByIndex -getNewestLedgerInfo -getLedgerInfoByHash -getHashByIndex -getHashesByIndex -getTxHistory -ledgerDbHasSpace -transactionDbHasSpace -``` - -#### Files backend/RelationalDBInterfaceSqlite.* - -``` -getTransactionsMinLedgerSeq -getAccountTransactionsMinLedgerSeq -deleteTransactionByLedgerSeq -deleteBeforeLedgerSeq -deleteTransactionsBeforeLedgerSeq -deleteAccountTransactionsBeforeLedgerSeq -getTransactionCount -getAccountTransactionCount -getLedgerCountMinMax -saveValidatedLedger -getLimitedOldestLedgerInfo -getLimitedNewestLedgerInfo -getOldestAccountTxs -getNewestAccountTxs -getOldestAccountTxsB -getNewestAccountTxsB -oldestAccountTxPage -newestAccountTxPage -oldestAccountTxPageB -newestAccountTxPageB -getTransaction -getKBUsedAll -getKBUsedLedger -getKBUsedTransaction -``` - -#### Files backend/RelationalDBInterfacePostgres.* - -``` -sweep -getCompleteLedgers -getValidatedLedgerAge -writeLedgerAndTransactions -getTxHashes -getAccountTx -locateTransaction -``` diff --git a/src/ripple/app/rdb/RelationalDBInterface_global.h b/src/ripple/app/rdb/RelationalDBInterface_global.h deleted file mode 100644 index 3eb0469ee..000000000 --- a/src/ripple/app/rdb/RelationalDBInterface_global.h +++ /dev/null @@ -1,333 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_RELATIONALDBINTERFACE_GLOBAL_H_INCLUDED -#define RIPPLE_CORE_RELATIONALDBINTERFACE_GLOBAL_H_INCLUDED - -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -/* Wallet DB */ - -/** - * @brief makeWalletDB Opens wallet DB and returns it. - * @param setup Path to database and other opening parameters. - * @return Unique pointer to database descriptor. - */ -std::unique_ptr -makeWalletDB(DatabaseCon::Setup const& setup); - -/** - * @brief makeTestWalletDB Opens test wallet DB with arbitrary name. - * @param setup Path to database and other opening parameters. - * @param dbname Name of database. - * @return Unique pointer to database descriptor. - */ -std::unique_ptr -makeTestWalletDB(DatabaseCon::Setup const& setup, std::string const& dbname); - -/** - * @brief getManifests Loads manifest from wallet DB and stores it in the cache. - * @param session Session with database. - * @param dbTable Name of table in the database to extract manifest from. - * @param mCache Cache to store manifest. - * @param j Journal. - */ -void -getManifests( - soci::session& session, - std::string const& dbTable, - ManifestCache& mCache, - beast::Journal j); - -/** - * @brief saveManifests Saves all given manifests to database. - * @param session Session with database. - * @param dbTable Name of database table to save manifest into. - * @param isTrusted Callback returned true if key is trusted. - * @param map Map to save which points public keys to manifests. - * @param j Journal. - */ -void -saveManifests( - soci::session& session, - std::string const& dbTable, - std::function const& isTrusted, - hash_map const& map, - beast::Journal j); - -/** - * @brief addValidatorManifest Saves manifest of validator to database. - * @param session Session with database. - * @param serialized Manifest of validator in raw format. - */ -void -addValidatorManifest(soci::session& session, std::string const& serialized); - -/** - * @brief getNodeIdentity Returns public and private keys of this node. - * @param session Session with database. - * @return Pair of public and private keys. - */ -std::pair -getNodeIdentity(soci::session& session); - -/** - * @brief getPeerReservationTable Returns peer reservation table. - * @param session Session with database. - * @param j Journal. - * @return Peer reservation hash table. - */ -std::unordered_set, KeyEqual> -getPeerReservationTable(soci::session& session, beast::Journal j); - -/** - * @brief insertPeerReservation Adds entry to peer reservation table. - * @param session Session with database. - * @param nodeId public key of node. - * @param description Description of node. - */ -void -insertPeerReservation( - soci::session& session, - PublicKey const& nodeId, - std::string const& description); - -/** - * @brief deletePeerReservation Deletes entry from peer reservation table. - * @param session Session with database. - * @param nodeId Public key of node to remove. - */ -void -deletePeerReservation(soci::session& session, PublicKey const& nodeId); - -/** - * @brief createFeatureVotes Creates FeatureVote table if it is not exists. - * @param session Session with walletDB database. - * @return true if the table already exists - */ -bool -createFeatureVotes(soci::session& session); - -// For historical reasons the up-vote and down-vote integer representations -// are unintuitive. -enum class AmendmentVote : int { up = 0, down = 1 }; - -/** - * @brief readAmendments Read all amendments from FeatureVotes table. - * @param session Session with walletDB database. - * @param callback Callback called for each amendment passing its hash, name - * and the flag if it should be vetoed as callback parameters - */ -void -readAmendments( - soci::session& session, - std::function amendment_hash, - boost::optional amendment_name, - boost::optional vote)> const& callback); - -/** - * @brief voteAmendment Set veto value for particular amendment. - * @param session Session with walletDB database. - * @param amendment Hash of amendment. - * @param name Name of amendment. - * @param vote Whether to vote in favor of this amendment. - */ -void -voteAmendment( - soci::session& session, - uint256 const& amendment, - std::string const& name, - AmendmentVote vote); - -/* State DB */ - -struct SavedState -{ - std::string writableDb; - std::string archiveDb; - LedgerIndex lastRotated; -}; - -/** - * @brief initStateDB Opens DB session with State DB. - * @param session Structure to open session in. - * @param config Path to database and other opening parameters. - * @param dbName Name of database. - */ -void -initStateDB( - soci::session& session, - BasicConfig const& config, - std::string const& dbName); - -/** - * @brief getCanDelete Returns ledger sequence which can be deleted. - * @param session Session with database. - * @return Ledger sequence. - */ -LedgerIndex -getCanDelete(soci::session& session); - -/** - * @brief setCanDelete Updates ledger sequence which can be deleted. - * @param session Session with database. - * @param canDelete Ledger sequence to save. - * @return Previous value of ledger sequence whic can be deleted. - */ -LedgerIndex -setCanDelete(soci::session& session, LedgerIndex canDelete); - -/** - * @brief getSavedState Returns saved state. - * @param session Session with database. - * @return The SavedState structure which contains names of - * writable DB, archive DB and last rotated ledger sequence. - */ -SavedState -getSavedState(soci::session& session); - -/** - * @brief setSavedState Saves given state. - * @param session Session with database. - * @param state The SavedState structure which contains names of - * writable DB, archive DB and last rotated ledger sequence. - */ -void -setSavedState(soci::session& session, SavedState const& state); - -/** - * @brief setLastRotated Updates last rotated ledger sequence. - * @param session Session with database. - * @param seq New value of last rotated ledger sequence. - */ -void -setLastRotated(soci::session& session, LedgerIndex seq); - -/* DatabaseBody DB */ - -/** - * @brief openDatabaseBodyDb Opens file download DB and returns its descriptor. - * Start new download process or continue existing one. - * @param setup Path to database and other opening parameters. - * @param path Path of new file to download. - * @return Pair of unique pointer to database and current downloaded size - * if download process continues. - */ -std::pair, std::optional> -openDatabaseBodyDb( - DatabaseCon::Setup const& setup, - boost::filesystem::path const& path); - -/** - * @brief databaseBodyDoPut Saves new fragment of downloaded file. - * @param session Session with database. - * @param data Downloaded piece to file data tp save. - * @param path Path of downloading file. - * @param fileSize Size of downloaded piece of file. - * @param part Sequence number of downloaded file part. - * @param maxRowSizePad Maximum size of file part to save. - * @return Number of saved parts. Downloaded piece may be splitted - * into several parts of size not large that maxRowSizePad. - */ -std::uint64_t -databaseBodyDoPut( - soci::session& session, - std::string const& data, - std::string const& path, - std::uint64_t fileSize, - std::uint64_t part, - std::uint16_t maxRowSizePad); - -/** - * @brief databaseBodyFinish Finishes download process and writes file to disk. - * @param session Session with database. - * @param fout Opened file to write downloaded data from database. - */ -void -databaseBodyFinish(soci::session& session, std::ofstream& fout); - -/* Vacuum DB */ - -/** - * @brief doVacuumDB Creates, initialises DB, and performs its cleanup. - * @param setup Path to database and other opening parameters. - * @return True if vacuum process completed successfully. - */ -bool -doVacuumDB(DatabaseCon::Setup const& setup); - -/* PeerFinder DB */ - -/** - * @brief initPeerFinderDB Opens session with peer finder database. - * @param session Structure to open session in. - * @param config Path to database and other opening parameters. - * @param j Journal. - */ -void -initPeerFinderDB( - soci::session& session, - BasicConfig const& config, - beast::Journal j); - -/** - * @brief updatePeerFinderDB Update peer finder DB to new version. - * @param session Session with database. - * @param currentSchemaVersion New version of database. - * @param j Journal. - */ -void -updatePeerFinderDB( - soci::session& session, - int currentSchemaVersion, - beast::Journal j); - -/** - * @brief readPeerFinderDB Read all entries from peer finder DB and call - * given callback for each entry. - * @param session Session with database. - * @param func Callback to call for each entry. - */ -void -readPeerFinderDB( - soci::session& session, - std::function const& func); - -/** - * @brief savePeerFinderDB Save new entry to peer finder DB. - * @param session Session with database. - * @param v Entry to save which contains information about new peer. - */ -void -savePeerFinderDB( - soci::session& session, - std::vector const& v); - -} // namespace ripple - -#endif diff --git a/src/ripple/app/rdb/RelationalDBInterface_postgres.h b/src/ripple/app/rdb/RelationalDBInterface_postgres.h deleted file mode 100644 index f5838813b..000000000 --- a/src/ripple/app/rdb/RelationalDBInterface_postgres.h +++ /dev/null @@ -1,248 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_RELATIONALDBINTERFACE_POSTGRES_H_INCLUDED -#define RIPPLE_CORE_RELATIONALDBINTERFACE_POSTGRES_H_INCLUDED - -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -class PgPool; - -using AccountTxMarker = RelationalDBInterface::AccountTxMarker; -using AccountTxArgs = RelationalDBInterface::AccountTxArgs; -using AccountTxResult = RelationalDBInterface::AccountTxResult; -using AccountTransactionsData = RelationalDBInterface::AccountTransactionsData; - -/** - * @brief getMinLedgerSeq Returns minimum ledger sequence - * from Postgres database - * @param pgPool Link to postgres database - * @param app Application - * @param j Journal - * @return Minimum ledger sequence if any, none if no ledgers - */ -std::optional -getMinLedgerSeq(std::shared_ptr const& pgPool, beast::Journal j); - -/** - * @brief getMaxLedgerSeq Returns maximum ledger sequence - * from Postgres database - * @param pgPool Link to postgres database - * @param app Application - * @return Maximum ledger sequence if any, none if no ledgers - */ -std::optional -getMaxLedgerSeq(std::shared_ptr const& pgPool); - -/** - * @brief getCompleteLedgers Returns string which contains - * list of completed ledgers - * @param pgPool Link to postgres database - * @param app Application - * @return String with completed ledgers - */ -std::string -getCompleteLedgers(std::shared_ptr const& pgPool); - -/** - * @brief getValidatedLedgerAge Returns age of last - * validated ledger - * @param pgPool Link to postgres database - * @param app Application - * @param j Journal - * @return Age of last validated ledger - */ -std::chrono::seconds -getValidatedLedgerAge(std::shared_ptr const& pgPool, beast::Journal j); - -/** - * @brief getNewestLedgerInfo Load latest ledger info from Postgres - * @param pgPool Link to postgres database - * @param app reference to Application - * @return Ledger info - */ -std::optional -getNewestLedgerInfo(std::shared_ptr const& pgPool, Application& app); - -/** - * @brief getLedgerInfoByIndex Load ledger info by index (AKA sequence) - * from Postgres - * @param pgPool Link to postgres database - * @param ledgerIndex the ledger index (or sequence) to load - * @param app reference to Application - * @return Ledger info - */ -std::optional -getLedgerInfoByIndex( - std::shared_ptr const& pgPool, - std::uint32_t ledgerIndex, - Application& app); - -/** - * @brief getLedgerInfoByHash Load ledger info by hash from Postgres - * @param pgPool Link to postgres database - * @param hash Hash of the ledger to load - * @param app reference to Application - * @return Ledger info - */ -std::optional -getLedgerInfoByHash( - std::shared_ptr const& pgPool, - uint256 const& ledgerHash, - Application& app); - -/** - * @brief getHashByIndex Given a ledger sequence, - * return the ledger hash - * @param pgPool Link to postgres database - * @param ledgerIndex Ledger sequence - * @param app Application - * @return Hash of ledger - */ -uint256 -getHashByIndex( - std::shared_ptr const& pgPool, - std::uint32_t ledgerIndex, - Application& app); - -/** - * @brief getHashesByIndex Given a ledger sequence, - * return the ledger hash and the parent hash - * @param pgPool Link to postgres database - * @param ledgerIndex Ledger sequence - * @param[out] ledgerHash Hash of ledger - * @param[out] parentHash Hash of parent ledger - * @param app Application - * @return True if the data was found - */ -bool -getHashesByIndex( - std::shared_ptr const& pgPool, - std::uint32_t ledgerIndex, - uint256& ledgerHash, - uint256& parentHash, - Application& app); - -/** - * @brief getHashesByIndex Given a contiguous range of sequences, - * return a map of sequence -> (hash, parent hash) - * @param pgPool Link to postgres database - * @param minSeq Lower bound of range - * @param maxSeq Upper bound of range - * @param app Application - * @return Mapping of all found ledger sequences to their hash and parent hash - */ -std::map -getHashesByIndex( - std::shared_ptr const& pgPool, - std::uint32_t minSeq, - std::uint32_t maxSeq, - Application& app); - -/** - * @brief getTxHashes Returns vector of tx hashes by given ledger - * sequence - * @param pgPool Link to postgres database - * @param seq Ledger sequence - * @param app Application - * @return Vector of tx hashes - */ -std::vector -getTxHashes( - std::shared_ptr const& pgPool, - LedgerIndex seq, - Application& app); - -/** - * @brief locateTransaction Returns information used to locate - * a transaction. Function is specific to postgres backend. - * @param pgPool Link to postgres database - * @param id Hash of the transaction. - * @param app Application - * @return Information used to locate a transaction. Contains a nodestore - * hash and ledger sequence pair if the transaction was found. - * Otherwise, contains the range of ledgers present in the database - * at the time of search. - */ -Transaction::Locator -locateTransaction( - std::shared_ptr const& pgPool, - uint256 const& id, - Application& app); - -/** - * @brief getTxHistory Returns most recent 20 transactions starting - * from given number or entry. - * @param pgPool Link to postgres database - * @param startIndex First number of returned entry. - * @param app Application - * @param j Journal - * @return Vector of sharded pointers to transactions sorted in - * descending order by ledger sequence. - */ -std::vector> -getTxHistory( - std::shared_ptr const& pgPool, - LedgerIndex startIndex, - Application& app, - beast::Journal j); - -/** - * @brief getAccountTx Get last account transactions specifies by - * passed argumenrs structure. - * @param pgPool Link to postgres database - * @param args Arguments which specify account and whose tx to return. - * @param app Application - * @param j Journal - * @return Vector of account transactions and RPC status of responce. - */ -std::pair -getAccountTx( - std::shared_ptr const& pgPool, - AccountTxArgs const& args, - Application& app, - beast::Journal j); - -/** - * @brief writeLedgerAndTransactions Write new ledger and transaction - * data to Postgres. - * @param pgPool Pool of Postgres connections - * @param info Ledger info to write. - * @param accountTxData Transaction data to write - * @param j Journal (for logging) - * @return True if success, false if failure. - */ -bool -writeLedgerAndTransactions( - std::shared_ptr const& pgPool, - LedgerInfo const& info, - std::vector const& accountTxData, - beast::Journal& j); - -} // namespace ripple - -#endif diff --git a/src/ripple/app/rdb/RelationalDBInterface_shards.h b/src/ripple/app/rdb/RelationalDBInterface_shards.h deleted file mode 100644 index 16ef67d21..000000000 --- a/src/ripple/app/rdb/RelationalDBInterface_shards.h +++ /dev/null @@ -1,257 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_RELATIONALDBINTERFACE_SHARDS_H_INCLUDED -#define RIPPLE_CORE_RELATIONALDBINTERFACE_SHARDS_H_INCLUDED - -#include -#include -#include -#include -#include - -namespace ripple { - -struct DatabasePair -{ - std::unique_ptr ledgerDb; - std::unique_ptr transactionDb; -}; - -/* Shard DB */ - -/** - * @brief makeMetaDBs Opens ledger and transaction 'meta' databases which - * map ledger hashes and transaction IDs to the index of the shard - * that holds the ledger or transaction. - * @param config Config object. - * @param setup Path to database and opening parameters. - * @param checkpointerSetup Database checkpointer setup. - * @return Struct DatabasePair which contains unique pointers to the ledger - * and transaction databases. - */ -DatabasePair -makeMetaDBs( - Config const& config, - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup); - -/** - * @brief saveLedgerMeta Stores (transaction ID -> shard index) and - * (ledger hash -> shard index) mappings in the meta databases. - * @param ledger The ledger. - * @param app Application object. - * @param lgrMetaSession Session to ledger meta database. - * @param txnMetaSession Session to transaction meta database. - * @param shardIndex The index of the shard that contains this ledger. - * @return True on success. - */ -bool -saveLedgerMeta( - std::shared_ptr const& ledger, - Application& app, - soci::session& lgrMetaSession, - soci::session& txnMetaSession, - std::uint32_t shardIndex); - -/** - * @brief getShardIndexforLedger Queries the ledger meta database to - * retrieve the index of the shard that contains this ledger. - * @param session Session to the database. - * @param hash Hash of the ledger. - * @return The index of the shard on success, otherwise an unseated value. - */ -std::optional -getShardIndexforLedger(soci::session& session, LedgerHash const& hash); - -/** - * @brief getShardIndexforTransaction Queries the transaction meta database to - * retrieve the index of the shard that contains this transaction. - * @param session Session to the database. - * @param id ID of the transaction. - * @return The index of the shard on success, otherwise an unseated value. - */ -std::optional -getShardIndexforTransaction(soci::session& session, TxID const& id); - -/** - * @brief makeShardCompleteLedgerDBs Opens shard databases for already - * verified shard and returns its descriptors. - * @param config Config object. - * @param setup Path to database and other opening parameters. - * @return Pair of unique pointers to opened ledger and transaction databases. - */ -DatabasePair -makeShardCompleteLedgerDBs( - Config const& config, - DatabaseCon::Setup const& setup); - -/** - * @brief makeShardIncompleteLedgerDBs Opens shard databases for not - * fully downloaded or verified shard and returns its descriptors. - * @param config Config object. - * @param setup Path to database and other opening parameters. - * @param checkpointerSetup Checkpointer parameters. - * @return Pair of unique pointers to opened ledger and transaction databases. - */ -DatabasePair -makeShardIncompleteLedgerDBs( - Config const& config, - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup); - -/** - * @brief updateLedgerDBs Save given ledger to shard databases. - * @param txdb Session with transaction DB. - * @param lgrdb Sessiob with ledger DB. - * @param ledger Ledger to save. - * @param index Index of the shard which the ledger belonfs to. - * @param stop Link to atomic flag which can stop the process if raised. - * @param j Journal - * @return True if ledger was successfully saved. - */ -bool -updateLedgerDBs( - soci::session& txdb, - soci::session& lgrdb, - std::shared_ptr const& ledger, - std::uint32_t index, - std::atomic& stop, - beast::Journal j); - -/* Shard acquire DB */ - -/** - * @brief makeAcquireDB Opens shard acquire DB and returns its descriptor. - * @param setup Path to DB and other opening parameters. - * @param checkpointerSetup Checkpointer parameters. - * @return Uniqye pointer to opened database. - */ -std::unique_ptr -makeAcquireDB( - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup); - -/** - * @brief insertAcquireDBIndex Adds new shard index to shard acquire DB. - * @param session Session with database. - * @param index Index to add. - */ -void -insertAcquireDBIndex(soci::session& session, std::uint32_t index); - -/** - * @brief selectAcquireDBLedgerSeqs Returns set of acquired ledgers for - * given shard. - * @param session Session with database. - * @param index Shard index. - * @return Pair which contains true if such as index found in database, - * and string which contains set of ledger sequences. - * If set of sequences was not saved than none is returned. - */ -std::pair> -selectAcquireDBLedgerSeqs(soci::session& session, std::uint32_t index); - -struct AcquireShardSeqsHash -{ - std::optional sequences; - std::optional hash; -}; - -/** - * @brief selectAcquireDBLedgerSeqsHash Returns set of acquired ledgers and - * hash for given shard. - * @param session Session with database. - * @param index Shard index. - * @return Pair which contains true of such an index found in database, - * and the AcquireShardSeqsHash structure which contains string - * with ledger sequences set and string with last ledger hash. - * If set of sequences or hash were not saved than none is returned. - */ -std::pair -selectAcquireDBLedgerSeqsHash(soci::session& session, std::uint32_t index); - -/** - * @brief updateAcquireDB Updates information in acquire DB. - * @param session Session with database. - * @param ledger Ledger to save into database. - * @param index Shard index. - * @param lastSeq Last acqyured ledger sequence. - * @param seqs Current set or acquired ledger sequences if it's not empty. - */ -void -updateAcquireDB( - soci::session& session, - std::shared_ptr const& ledger, - std::uint32_t index, - std::uint32_t lastSeq, - std::optional const& seqs); - -/* Archive DB */ - -/** - * @brief makeArchiveDB Opens shard archive DB and returns its descriptor. - * @param dir Path to database to open. - * @param dbName Name of database. - * @return Unique pointer to opened database. - */ -std::unique_ptr -makeArchiveDB(boost::filesystem::path const& dir, std::string const& dbName); - -/** - * @brief readArchiveDB Read entries from shard archive database and calls - * fiven callback for each entry. - * @param db Session with database. - * @param func Callback to call for each entry. - */ -void -readArchiveDB( - DatabaseCon& db, - std::function const& func); - -/** - * @brief insertArchiveDB Adds entry to shard archive database. - * @param db Session with database. - * @param shardIndex Shard index to add. - * @param url Shard download url to add. - */ -void -insertArchiveDB( - DatabaseCon& db, - std::uint32_t shardIndex, - std::string const& url); - -/** - * @brief deleteFromArchiveDB Deletes entry from shard archive DB. - * @param db Session with database. - * @param shardIndex Shard index to remove from DB. - */ -void -deleteFromArchiveDB(DatabaseCon& db, std::uint32_t shardIndex); - -/** - * @brief dropArchiveDB Removes table in shard archive DB. - * @param db Session with database. - */ -void -dropArchiveDB(DatabaseCon& db); - -} // namespace ripple - -#endif diff --git a/src/ripple/app/rdb/RelationalDBInterface.h b/src/ripple/app/rdb/RelationalDatabase.h similarity index 76% rename from src/ripple/app/rdb/RelationalDBInterface.h rename to src/ripple/app/rdb/RelationalDatabase.h index 759261832..a269bf256 100644 --- a/src/ripple/app/rdb/RelationalDBInterface.h +++ b/src/ripple/app/rdb/RelationalDatabase.h @@ -17,8 +17,8 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_RELATIONALDBINTERFACE_H_INCLUDED -#define RIPPLE_CORE_RELATIONALDBINTERFACE_H_INCLUDED +#ifndef RIPPLE_APP_RDB_RELATIONALDATABASE_H_INCLUDED +#define RIPPLE_APP_RDB_RELATIONALDATABASE_H_INCLUDED #include #include @@ -45,7 +45,7 @@ struct LedgerRange uint32_t max; }; -class RelationalDBInterface +class RelationalDatabase { public: struct CountMinMax @@ -135,56 +135,61 @@ public: }; /** - * @brief init Creates and returns appropriate interface based on config. + * @brief init Creates and returns an appropriate RelationalDatabase + * instance based on configuration. * @param app Application object. * @param config Config object. * @param jobQueue JobQueue object. * @return Unique pointer to the interface. */ - static std::unique_ptr + static std::unique_ptr init(Application& app, Config const& config, JobQueue& jobQueue); - virtual ~RelationalDBInterface() = default; + virtual ~RelationalDatabase() = default; /** - * @brief getMinLedgerSeq Returns minimum ledger sequence in Ledgers table. - * @return Ledger sequence or none if no ledgers exist. + * @brief getMinLedgerSeq Returns the minimum ledger sequence in the Ledgers + * table. + * @return Ledger sequence or no value if no ledgers exist. */ virtual std::optional getMinLedgerSeq() = 0; /** - * @brief getMaxLedgerSeq Returns maximum ledger sequence in Ledgers table. + * @brief getMaxLedgerSeq Returns the maximum ledger sequence in the Ledgers + * table. * @return Ledger sequence or none if no ledgers exist. */ virtual std::optional getMaxLedgerSeq() = 0; /** - * @brief getLedgerInfoByIndex Returns ledger by its sequence. + * @brief getLedgerInfoByIndex Returns a ledger by its sequence. * @param ledgerSeq Ledger sequence. - * @return Ledger or none if ledger not found. + * @return The ledger if found, otherwise no value. */ virtual std::optional getLedgerInfoByIndex(LedgerIndex ledgerSeq) = 0; /** - * @brief getNewestLedgerInfo Returns info of newest saved ledger. - * @return Ledger info or none if ledger not found. + * @brief getNewestLedgerInfo Returns the info of the newest saved ledger. + * @return Ledger info if found, otherwise no value. */ virtual std::optional getNewestLedgerInfo() = 0; /** - * @brief getLedgerInfoByHash Returns info of ledger with given hash. + * @brief getLedgerInfoByHash Returns the info of the ledger with given + * hash. * @param ledgerHash Hash of the ledger. - * @return Ledger or none if ledger not found. + * @return Ledger if found, otherwise no value. */ virtual std::optional getLedgerInfoByHash(uint256 const& ledgerHash) = 0; /** - * @brief getHashByIndex Returns hash of ledger with given sequence. + * @brief getHashByIndex Returns the hash of the ledger with the given + * sequence. * @param ledgerIndex Ledger sequence. * @return Hash of the ledger. */ @@ -192,39 +197,40 @@ public: getHashByIndex(LedgerIndex ledgerIndex) = 0; /** - * @brief getHashesByIndex Returns hash of the ledger and hash of parent - * ledger for the ledger of given sequence. + * @brief getHashesByIndex Returns the hashes of the ledger and its parent + * as specified by the ledgerIndex. * @param ledgerIndex Ledger sequence. - * @return Struct LedgerHashPair which contain hashes of the ledger and - * its parent ledger. + * @return Struct LedgerHashPair which contains hashes of the ledger and + * its parent. */ virtual std::optional getHashesByIndex(LedgerIndex ledgerIndex) = 0; /** - * @brief getHashesByIndex Returns hash of the ledger and hash of parent - * ledger for all ledgers with sequences from given minimum limit - * to given maximum limit. + * @brief getHashesByIndex Returns hashes of each ledger and its parent for + * all ledgers within the provided range. * @param minSeq Minimum ledger sequence. * @param maxSeq Maximum ledger sequence. - * @return Map which points sequence number of found ledger to the struct - * LedgerHashPair which contains ledger hash and its parent hash. + * @return Container that maps the sequence number of a found ledger to the + * struct LedgerHashPair which contains the hashes of the ledger and + * its parent. */ virtual std::map getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) = 0; /** - * @brief getTxHistory Returns most recent 20 transactions starting from - * given number or entry. + * @brief getTxHistory Returns the 20 most recent transactions starting from + * the given number. * @param startIndex First number of returned entry. - * @return Vector of sharded pointers to transactions sorted in + * @return Vector of shared pointers to transactions sorted in * descending order by ledger sequence. */ virtual std::vector> getTxHistory(LedgerIndex startIndex) = 0; /** - * @brief ledgerDbHasSpace Checks if ledger database has available space. + * @brief ledgerDbHasSpace Checks if the ledger database has available + * space. * @param config Config object. * @return True if space is available. */ @@ -232,7 +238,7 @@ public: ledgerDbHasSpace(Config const& config) = 0; /** - * @brief transactionDbHasSpace Checks if transaction database has + * @brief transactionDbHasSpace Checks if the transaction database has * available space. * @param config Config object. * @return True if space is available. diff --git a/src/ripple/app/rdb/ShardArchive.h b/src/ripple/app/rdb/ShardArchive.h new file mode 100644 index 000000000..20c4382b0 --- /dev/null +++ b/src/ripple/app/rdb/ShardArchive.h @@ -0,0 +1,78 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_RDB_SHARDARCHIVE_H_INCLUDED +#define RIPPLE_APP_RDB_SHARDARCHIVE_H_INCLUDED + +#include +#include + +namespace ripple { + +/** + * @brief makeArchiveDB Opens the shard archive database and returns its + * descriptor. + * @param dir Path to the database to open. + * @param dbName Name of the database. + * @return Unique pointer to the opened database. + */ +std::unique_ptr +makeArchiveDB(boost::filesystem::path const& dir, std::string const& dbName); + +/** + * @brief readArchiveDB Reads entries from the shard archive database and + * invokes the given callback for each entry. + * @param db Session with the database. + * @param func Callback to invoke for each entry. + */ +void +readArchiveDB( + DatabaseCon& db, + std::function const& func); + +/** + * @brief insertArchiveDB Adds an entry to the shard archive database. + * @param db Session with the database. + * @param shardIndex Shard index to add. + * @param url Shard download url to add. + */ +void +insertArchiveDB( + DatabaseCon& db, + std::uint32_t shardIndex, + std::string const& url); + +/** + * @brief deleteFromArchiveDB Deletes an entry from the shard archive database. + * @param db Session with the database. + * @param shardIndex Shard index to remove from the database. + */ +void +deleteFromArchiveDB(DatabaseCon& db, std::uint32_t shardIndex); + +/** + * @brief dropArchiveDB Removes a table in the shard archive database. + * @param db Session with the database. + */ +void +dropArchiveDB(DatabaseCon& db); + +} // namespace ripple + +#endif diff --git a/src/ripple/app/rdb/State.h b/src/ripple/app/rdb/State.h new file mode 100644 index 000000000..fe74d5f19 --- /dev/null +++ b/src/ripple/app/rdb/State.h @@ -0,0 +1,98 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_RDB_STATE_H_INCLUDED +#define RIPPLE_APP_RDB_STATE_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { + +struct SavedState +{ + std::string writableDb; + std::string archiveDb; + LedgerIndex lastRotated; +}; + +/** + * @brief initStateDB Opens a session with the State database. + * @param session Provides a session with the database. + * @param config Path to the database and other opening parameters. + * @param dbName Name of the database. + */ +void +initStateDB( + soci::session& session, + BasicConfig const& config, + std::string const& dbName); + +/** + * @brief getCanDelete Returns the ledger sequence which can be deleted. + * @param session Session with the database. + * @return Ledger sequence. + */ +LedgerIndex +getCanDelete(soci::session& session); + +/** + * @brief setCanDelete Updates the ledger sequence which can be deleted. + * @param session Session with the database. + * @param canDelete Ledger sequence to save. + * @return Previous value of the ledger sequence which can be deleted. + */ +LedgerIndex +setCanDelete(soci::session& session, LedgerIndex canDelete); + +/** + * @brief getSavedState Returns the saved state. + * @param session Session with the database. + * @return The SavedState structure which contains the names of the writable + * database, the archive database and the last rotated ledger sequence. + */ +SavedState +getSavedState(soci::session& session); + +/** + * @brief setSavedState Saves the given state. + * @param session Session with the database. + * @param state The SavedState structure which contains the names of the + * writable database, the archive database and the last rotated ledger + * sequence. + */ +void +setSavedState(soci::session& session, SavedState const& state); + +/** + * @brief setLastRotated Updates the last rotated ledger sequence. + * @param session Session with the database. + * @param seq New value of the last rotated ledger sequence. + */ +void +setLastRotated(soci::session& session, LedgerIndex seq); + +} // namespace ripple + +#endif diff --git a/src/ripple/app/rdb/UnitaryShard.h b/src/ripple/app/rdb/UnitaryShard.h new file mode 100644 index 000000000..d2ac773db --- /dev/null +++ b/src/ripple/app/rdb/UnitaryShard.h @@ -0,0 +1,155 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_RDB_UNITARYSHARD_H_INCLUDED +#define RIPPLE_APP_RDB_UNITARYSHARD_H_INCLUDED + +#include +#include +#include +#include +#include + +namespace ripple { + +struct DatabasePair +{ + std::unique_ptr ledgerDb; + std::unique_ptr transactionDb; +}; + +/** + * @brief makeShardCompleteLedgerDBs Opens shard databases for verified shards + * and returns their descriptors. + * @param config Config object. + * @param setup Path to the databases and other opening parameters. + * @return Pair of unique pointers to the opened ledger and transaction + * databases. + */ +DatabasePair +makeShardCompleteLedgerDBs( + Config const& config, + DatabaseCon::Setup const& setup); + +/** + * @brief makeShardIncompleteLedgerDBs Opens shard databases for partially + * downloaded or unverified shards and returns their descriptors. + * @param config Config object. + * @param setup Path to the databases and other opening parameters. + * @param checkpointerSetup Checkpointer parameters. + * @return Pair of unique pointers to the opened ledger and transaction + * databases. + */ +DatabasePair +makeShardIncompleteLedgerDBs( + Config const& config, + DatabaseCon::Setup const& setup, + DatabaseCon::CheckpointerSetup const& checkpointerSetup); + +/** + * @brief updateLedgerDBs Saves the given ledger to shard databases. + * @param txdb Session with the transaction databases. + * @param lgrdb Session with the ledger databases. + * @param ledger Ledger to save. + * @param index Index of the shard that owns the ledger. + * @param stop Reference to an atomic flag that can stop the process if raised. + * @param j Journal + * @return True if the ledger was successfully saved. + */ +bool +updateLedgerDBs( + soci::session& txdb, + soci::session& lgrdb, + std::shared_ptr const& ledger, + std::uint32_t index, + std::atomic& stop, + beast::Journal j); + +/** + * @brief makeAcquireDB Opens the shard acquire database and returns its + * descriptor. + * @param setup Path to the database and other opening parameters. + * @param checkpointerSetup Checkpointer parameters. + * @return Unique pointer to the opened database. + */ +std::unique_ptr +makeAcquireDB( + DatabaseCon::Setup const& setup, + DatabaseCon::CheckpointerSetup const& checkpointerSetup); + +/** + * @brief insertAcquireDBIndex Adds a new shard index to the shard acquire + * database. + * @param session Session with the database. + * @param index Index to add. + */ +void +insertAcquireDBIndex(soci::session& session, std::uint32_t index); + +/** + * @brief selectAcquireDBLedgerSeqs Returns the set of acquired ledgers for + * the given shard. + * @param session Session with the database. + * @param index Shard index. + * @return Pair which contains true if such an index was found in the database, + * and a string which contains the set of ledger sequences. + * If no sequences were saved then the optional will have no value. + */ +std::pair> +selectAcquireDBLedgerSeqs(soci::session& session, std::uint32_t index); + +struct AcquireShardSeqsHash +{ + std::optional sequences; + std::optional hash; +}; + +/** + * @brief selectAcquireDBLedgerSeqsHash Returns the set of acquired ledger + * sequences and the last ledger hash for the shard with the provided + * index. + * @param session Session with the database. + * @param index Shard index. + * @return Pair which contains true if such an index was found in the database + * and the AcquireShardSeqsHash structure which contains a string with + * the ledger sequences and a string with last ledger hash. If the set + * of sequences or hash were not saved then no value is returned. + */ +std::pair +selectAcquireDBLedgerSeqsHash(soci::session& session, std::uint32_t index); + +/** + * @brief updateAcquireDB Updates information in the acquire DB. + * @param session Session with the database. + * @param ledger Ledger to save into the database. + * @param index Shard index. + * @param lastSeq Last acquired ledger sequence. + * @param seqs Current set of acquired ledger sequences if it's not empty. + */ +void +updateAcquireDB( + soci::session& session, + std::shared_ptr const& ledger, + std::uint32_t index, + std::uint32_t lastSeq, + std::optional const& seqs); + +} // namespace ripple + +#endif diff --git a/src/ripple/basics/impl/strHex.cpp b/src/ripple/app/rdb/Vacuum.h similarity index 62% rename from src/ripple/basics/impl/strHex.cpp rename to src/ripple/app/rdb/Vacuum.h index 084493af5..3db18da04 100644 --- a/src/ripple/basics/impl/strHex.cpp +++ b/src/ripple/app/rdb/Vacuum.h @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ /* This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. + Copyright (c) 2021 Ripple Labs Inc. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -10,40 +10,28 @@ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== -#include -#include +#ifndef RIPPLE_APP_RDB_VACUUM_H_INCLUDED +#define RIPPLE_APP_RDB_VACUUM_H_INCLUDED + +#include namespace ripple { -int -charUnHex(unsigned char c) -{ - static constexpr std::array const xtab = []() { - std::array t{}; - - for (auto& x : t) - x = -1; - - for (int i = 0; i < 10; ++i) - t['0' + i] = i; - - for (int i = 0; i < 6; ++i) - { - t['A' + i] = 10 + i; - t['a' + i] = 10 + i; - } - - return t; - }(); - - return xtab[c]; -} +/** + * @brief doVacuumDB Creates, initialises, and performs cleanup on a database. + * @param setup Path to the database and other opening parameters. + * @return True if the vacuum process completed successfully. + */ +bool +doVacuumDB(DatabaseCon::Setup const& setup); } // namespace ripple + +#endif diff --git a/src/ripple/app/rdb/Wallet.h b/src/ripple/app/rdb/Wallet.h new file mode 100644 index 000000000..2769e459a --- /dev/null +++ b/src/ripple/app/rdb/Wallet.h @@ -0,0 +1,179 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_RDB_WALLET_H_INCLUDED +#define RIPPLE_APP_RDB_WALLET_H_INCLUDED + +#include +#include +#include +#include +#include +#include + +namespace ripple { + +/** + * @brief makeWalletDB Opens the wallet database and returns it. + * @param setup Path to the database and other opening parameters. + * @return Unique pointer to the database descriptor. + */ +std::unique_ptr +makeWalletDB(DatabaseCon::Setup const& setup); + +/** + * @brief makeTestWalletDB Opens a test wallet database with an arbitrary name. + * @param setup Path to the database and other opening parameters. + * @param dbname Name of the database. + * @return Unique pointer to the database descriptor. + */ +std::unique_ptr +makeTestWalletDB(DatabaseCon::Setup const& setup, std::string const& dbname); + +/** + * @brief getManifests Loads a manifest from the wallet database and stores it + * in the cache. + * @param session Session with the database. + * @param dbTable Name of the database table from which the manifest will be + * extracted. + * @param mCache Cache for storing the manifest. + * @param j Journal. + */ +void +getManifests( + soci::session& session, + std::string const& dbTable, + ManifestCache& mCache, + beast::Journal j); + +/** + * @brief saveManifests Saves all given manifests to the database. + * @param session Session with the database. + * @param dbTable Name of the database table that will store the manifest. + * @param isTrusted Callback that returns true if the key is trusted. + * @param map Maps public keys to manifests. + * @param j Journal. + */ +void +saveManifests( + soci::session& session, + std::string const& dbTable, + std::function const& isTrusted, + hash_map const& map, + beast::Journal j); + +/** + * @brief addValidatorManifest Saves the manifest of a validator to the + * database. + * @param session Session with the database. + * @param serialized Manifest of the validator in raw format. + */ +void +addValidatorManifest(soci::session& session, std::string const& serialized); + +/** Delete any saved public/private key associated with this node. */ +void +clearNodeIdentity(soci::session& session); + +/** Returns a stable public and private key for this node. + + The node's public identity is defined by a secp256k1 keypair + that is (normally) randomly generated. This function will + return such a keypair, securely generating one if needed. + + @param session Session with the database. + + @return Pair of public and private secp256k1 keys. + */ +std::pair +getNodeIdentity(soci::session& session); + +/** + * @brief getPeerReservationTable Returns the peer reservation table. + * @param session Session with the database. + * @param j Journal. + * @return Peer reservation hash table. + */ +std::unordered_set, KeyEqual> +getPeerReservationTable(soci::session& session, beast::Journal j); + +/** + * @brief insertPeerReservation Adds an entry to the peer reservation table. + * @param session Session with the database. + * @param nodeId Public key of the node. + * @param description Description of the node. + */ +void +insertPeerReservation( + soci::session& session, + PublicKey const& nodeId, + std::string const& description); + +/** + * @brief deletePeerReservation Deletes an entry from the peer reservation + * table. + * @param session Session with the database. + * @param nodeId Public key of the node to remove. + */ +void +deletePeerReservation(soci::session& session, PublicKey const& nodeId); + +/** + * @brief createFeatureVotes Creates the FeatureVote table if it does not exist. + * @param session Session with the wallet database. + * @return true if the table already exists + */ +bool +createFeatureVotes(soci::session& session); + +// For historical reasons the up-vote and down-vote integer representations +// are unintuitive. +enum class AmendmentVote : int { up = 0, down = 1 }; + +/** + * @brief readAmendments Reads all amendments from the FeatureVotes table. + * @param session Session with the wallet database. + * @param callback Callback called for each amendment with its hash, name and + * optionally a flag denoting whether the amendment should be vetoed. + */ +void +readAmendments( + soci::session& session, + std::function amendment_hash, + boost::optional amendment_name, + boost::optional vote)> const& callback); + +/** + * @brief voteAmendment Set the veto value for a particular amendment. + * @param session Session with the wallet database. + * @param amendment Hash of the amendment. + * @param name Name of the amendment. + * @param vote Whether to vote in favor of this amendment. + */ +void +voteAmendment( + soci::session& session, + uint256 const& amendment, + std::string const& name, + AmendmentVote vote); + +} // namespace ripple + +#endif diff --git a/src/ripple/app/rdb/backend/RelationalDBInterfacePostgres.h b/src/ripple/app/rdb/backend/PostgresDatabase.h similarity index 53% rename from src/ripple/app/rdb/backend/RelationalDBInterfacePostgres.h rename to src/ripple/app/rdb/backend/PostgresDatabase.h index 7149f475f..e86736112 100644 --- a/src/ripple/app/rdb/backend/RelationalDBInterfacePostgres.h +++ b/src/ripple/app/rdb/backend/PostgresDatabase.h @@ -17,55 +17,47 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_RELATIONALDBINTERFACEPOSTGRES_H_INCLUDED -#define RIPPLE_CORE_RELATIONALDBINTERFACEPOSTGRES_H_INCLUDED +#ifndef RIPPLE_APP_RDB_BACKEND_POSTGRESDATABASE_H_INCLUDED +#define RIPPLE_APP_RDB_BACKEND_POSTGRESDATABASE_H_INCLUDED -#include +#include namespace ripple { -class RelationalDBInterfacePostgres : public RelationalDBInterface +class PostgresDatabase : public RelationalDatabase { public: - /** There is only one implementation of this interface: - * RelationalDBInterfacePostgresImp. It wraps a stoppable object (PgPool) - * that does not follow RAII, and it does not go through the effort of - * following RAII either. The owner of the only object of that type - * (ApplicationImp) holds it by the type of its interface instead of its - * implementation, and thus the lifetime management methods need to be - * part of the interface. - */ virtual void stop() = 0; /** - * @brief sweep Sweep the database. Method is specific for postgres backend. + * @brief sweep Sweeps the database. */ virtual void sweep() = 0; /** - * @brief getCompleteLedgers Returns string which contains list of - * completed ledgers. Method is specific for postgres backend. - * @return String with completed ledger numbers + * @brief getCompleteLedgers Returns a string which contains a list of + * completed ledgers. + * @return String with completed ledger sequences */ virtual std::string getCompleteLedgers() = 0; /** - * @brief getValidatedLedgerAge Returns age of last - * validated ledger. Method is specific for postgres backend. - * @return Age of last validated ledger in seconds + * @brief getValidatedLedgerAge Returns the age of the last validated + * ledger. + * @return Age of the last validated ledger in seconds */ virtual std::chrono::seconds getValidatedLedgerAge() = 0; /** - * @brief writeLedgerAndTransactions Write new ledger and transaction data - * into database. Method is specific for Postgres backend. + * @brief writeLedgerAndTransactions Writes new ledger and transaction data + * into the database. * @param info Ledger info to write. * @param accountTxData Transaction data to write - * @return True if success, false if failure. + * @return True on success, false on failure. */ virtual bool writeLedgerAndTransactions( @@ -73,32 +65,30 @@ public: std::vector const& accountTxData) = 0; /** - * @brief getTxHashes Returns vector of tx hashes by given ledger - * sequence. Method is specific to postgres backend. + * @brief getTxHashes Returns a vector of the hashes of transactions + * belonging to the ledger with the provided sequence. * @param seq Ledger sequence - * @return Vector of tx hashes + * @return Vector of transaction hashes */ virtual std::vector getTxHashes(LedgerIndex seq) = 0; /** - * @brief getAccountTx Get last account transactions specifies by - * passed argumenrs structure. Function if specific to postgres - * backend. - * @param args Arguments which specify account and whose tx to return. - * @param app Application - * @param j Journal - * @return Vector of account transactions and RPC status of responce. + * @brief getAccountTx Get the last account transactions specified by the + * AccountTxArgs struct. + * @param args Arguments which specify the account and which transactions to + * return. + * @return Vector of account transactions and the RPC status response. */ virtual std::pair getAccountTx(AccountTxArgs const& args) = 0; /** * @brief locateTransaction Returns information used to locate - * a transaction. Function is specific to postgres backend. + * a transaction. * @param id Hash of the transaction. * @return Information used to locate a transaction. Contains a nodestore - * hash and ledger sequence pair if the transaction was found. + * hash and a ledger sequence pair if the transaction was found. * Otherwise, contains the range of ledgers present in the database * at the time of search. */ @@ -110,9 +100,9 @@ public: * network * @param[out] reason if the database is not caught up, reason contains a * helpful message describing why - * @return false if the most recently written - * ledger has a close time over 3 minutes ago, or if there are - * no ledgers in the database. true otherwise + * @return false if the most recently written ledger has a close time + * over 3 minutes ago, or if there are no ledgers in the + * database. true otherwise */ virtual bool isCaughtUp(std::string& reason) = 0; diff --git a/src/ripple/app/rdb/backend/RelationalDBInterfacePostgres.cpp b/src/ripple/app/rdb/backend/RelationalDBInterfacePostgres.cpp deleted file mode 100644 index 8b9a96210..000000000 --- a/src/ripple/app/rdb/backend/RelationalDBInterfacePostgres.cpp +++ /dev/null @@ -1,298 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -class RelationalDBInterfacePostgresImp : public RelationalDBInterfacePostgres -{ -public: - RelationalDBInterfacePostgresImp( - Application& app, - Config const& config, - JobQueue& jobQueue) - : app_(app) - , j_(app_.journal("PgPool")) - , pgPool_( -#ifdef RIPPLED_REPORTING - make_PgPool(config.section("ledger_tx_tables"), j_) -#endif - ) - { - assert(config.reporting()); -#ifdef RIPPLED_REPORTING - if (config.reporting() && !config.reportingReadOnly()) // use pg - { - initSchema(pgPool_); - } -#endif - } - - void - stop() override - { -#ifdef RIPPLED_REPORTING - pgPool_->stop(); -#endif - } - - void - sweep() override; - - std::optional - getMinLedgerSeq() override; - - std::optional - getMaxLedgerSeq() override; - - std::string - getCompleteLedgers() override; - - std::chrono::seconds - getValidatedLedgerAge() override; - - bool - writeLedgerAndTransactions( - LedgerInfo const& info, - std::vector const& accountTxData) override; - - std::optional - getLedgerInfoByIndex(LedgerIndex ledgerSeq) override; - - std::optional - getNewestLedgerInfo() override; - - std::optional - getLedgerInfoByHash(uint256 const& ledgerHash) override; - - uint256 - getHashByIndex(LedgerIndex ledgerIndex) override; - - std::optional - getHashesByIndex(LedgerIndex ledgerIndex) override; - - std::map - getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) override; - - std::vector - getTxHashes(LedgerIndex seq) override; - - std::vector> - getTxHistory(LedgerIndex startIndex) override; - - std::pair - getAccountTx(AccountTxArgs const& args) override; - - Transaction::Locator - locateTransaction(uint256 const& id) override; - - bool - ledgerDbHasSpace(Config const& config) override; - - bool - transactionDbHasSpace(Config const& config) override; - - bool - isCaughtUp(std::string& reason) override; - -private: - Application& app_; - beast::Journal j_; - std::shared_ptr pgPool_; - - bool - dbHasSpace(Config const& config); -}; - -void -RelationalDBInterfacePostgresImp::sweep() -{ -#ifdef RIPPLED_REPORTING - pgPool_->idleSweeper(); -#endif -} - -std::optional -RelationalDBInterfacePostgresImp::getMinLedgerSeq() -{ - return ripple::getMinLedgerSeq(pgPool_, j_); -} - -std::optional -RelationalDBInterfacePostgresImp::getMaxLedgerSeq() -{ - return ripple::getMaxLedgerSeq(pgPool_); -} - -std::string -RelationalDBInterfacePostgresImp::getCompleteLedgers() -{ - return ripple::getCompleteLedgers(pgPool_); -} - -std::chrono::seconds -RelationalDBInterfacePostgresImp::getValidatedLedgerAge() -{ - return ripple::getValidatedLedgerAge(pgPool_, j_); -} - -bool -RelationalDBInterfacePostgresImp::writeLedgerAndTransactions( - LedgerInfo const& info, - std::vector const& accountTxData) -{ - return ripple::writeLedgerAndTransactions(pgPool_, info, accountTxData, j_); -} - -std::optional -RelationalDBInterfacePostgresImp::getLedgerInfoByIndex(LedgerIndex ledgerSeq) -{ - return ripple::getLedgerInfoByIndex(pgPool_, ledgerSeq, app_); -} - -std::optional -RelationalDBInterfacePostgresImp::getNewestLedgerInfo() -{ - return ripple::getNewestLedgerInfo(pgPool_, app_); -} - -std::optional -RelationalDBInterfacePostgresImp::getLedgerInfoByHash(uint256 const& ledgerHash) -{ - return ripple::getLedgerInfoByHash(pgPool_, ledgerHash, app_); -} - -uint256 -RelationalDBInterfacePostgresImp::getHashByIndex(LedgerIndex ledgerIndex) -{ - return ripple::getHashByIndex(pgPool_, ledgerIndex, app_); -} - -std::optional -RelationalDBInterfacePostgresImp::getHashesByIndex(LedgerIndex ledgerIndex) -{ - LedgerHashPair p; - if (!ripple::getHashesByIndex( - pgPool_, ledgerIndex, p.ledgerHash, p.parentHash, app_)) - return {}; - return p; -} - -std::map -RelationalDBInterfacePostgresImp::getHashesByIndex( - LedgerIndex minSeq, - LedgerIndex maxSeq) -{ - return ripple::getHashesByIndex(pgPool_, minSeq, maxSeq, app_); -} - -std::vector -RelationalDBInterfacePostgresImp::getTxHashes(LedgerIndex seq) -{ - return ripple::getTxHashes(pgPool_, seq, app_); -} - -std::vector> -RelationalDBInterfacePostgresImp::getTxHistory(LedgerIndex startIndex) -{ - return ripple::getTxHistory(pgPool_, startIndex, app_, j_); -} - -std::pair -RelationalDBInterfacePostgresImp::getAccountTx(AccountTxArgs const& args) -{ - return ripple::getAccountTx(pgPool_, args, app_, j_); -} - -Transaction::Locator -RelationalDBInterfacePostgresImp::locateTransaction(uint256 const& id) -{ - return ripple::locateTransaction(pgPool_, id, app_); -} - -bool -RelationalDBInterfacePostgresImp::dbHasSpace(Config const& config) -{ - /* Postgres server could be running on a different machine. */ - - return true; -} - -bool -RelationalDBInterfacePostgresImp::ledgerDbHasSpace(Config const& config) -{ - return dbHasSpace(config); -} - -bool -RelationalDBInterfacePostgresImp::transactionDbHasSpace(Config const& config) -{ - return dbHasSpace(config); -} - -std::unique_ptr -getRelationalDBInterfacePostgres( - Application& app, - Config const& config, - JobQueue& jobQueue) -{ - return std::make_unique( - app, config, jobQueue); -} -bool -RelationalDBInterfacePostgresImp::isCaughtUp(std::string& reason) -{ -#ifdef RIPPLED_REPORTING - using namespace std::chrono_literals; - auto age = PgQuery(pgPool_)("SELECT age()"); - if (!age || age.isNull()) - { - reason = "No ledgers in database"; - return false; - } - if (std::chrono::seconds{age.asInt()} > 3min) - { - reason = "No recently-published ledger"; - return false; - } -#endif - return true; -} - -} // namespace ripple diff --git a/src/ripple/app/rdb/backend/RelationalDBInterfaceSqlite.h b/src/ripple/app/rdb/backend/RelationalDBInterfaceSqlite.h deleted file mode 100644 index 085f59628..000000000 --- a/src/ripple/app/rdb/backend/RelationalDBInterfaceSqlite.h +++ /dev/null @@ -1,302 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_CORE_RELATIONALDBINTERFACESQLITE_H_INCLUDED -#define RIPPLE_CORE_RELATIONALDBINTERFACESQLITE_H_INCLUDED - -#include - -namespace ripple { - -class RelationalDBInterfaceSqlite : public RelationalDBInterface -{ -public: - /** - * @brief getTransactionsMinLedgerSeq Returns minimum ledger sequence - * among records in the Transactions table. - * @return Ledger sequence or none if no ledgers exist. - */ - virtual std::optional - getTransactionsMinLedgerSeq() = 0; - - /** - * @brief getAccountTransactionsMinLedgerSeq Returns minimum ledger - * sequence among records in the AccountTransactions table. - * @return Ledger sequence or none if no ledgers exist. - */ - virtual std::optional - getAccountTransactionsMinLedgerSeq() = 0; - - /** - * @brief deleteTransactionByLedgerSeq Deletes transactions from ledger - * with given sequence. - * @param ledgerSeq Ledger sequence. - */ - virtual void - deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) = 0; - - /** - * @brief deleteBeforeLedgerSeq Deletes all ledgers with given sequence - * and all sequences below it. - * @param ledgerSeq Ledger sequence. - */ - virtual void - deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) = 0; - - /** - * @brief deleteTransactionsBeforeLedgerSeq Deletes all transactions with - * given ledger sequence and all sequences below it. - * @param ledgerSeq Ledger sequence. - */ - virtual void - deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) = 0; - - /** - * @brief deleteAccountTransactionsBeforeLedgerSeq Deletes all account - * transactions with given ledger sequence and all sequences - * below it. - * @param ledgerSeq Ledger sequence. - */ - virtual void - deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) = 0; - - /** - * @brief getTransactionCount Returns number of transactions. - * @return Number of transactions. - */ - virtual std::size_t - getTransactionCount() = 0; - - /** - * @brief getAccountTransactionCount Returns number of account - * transactions. - * @return Number of account transactions. - */ - virtual std::size_t - getAccountTransactionCount() = 0; - - /** - * @brief getLedgerCountMinMax Returns minumum ledger sequence, - * maximum ledger sequence and total number of saved ledgers. - * @return Struct CountMinMax which contain minimum sequence, - * maximum sequence and number of ledgers. - */ - virtual struct CountMinMax - getLedgerCountMinMax() = 0; - - /** - * @brief saveValidatedLedger Saves ledger into database. - * @param ledger The ledger. - * @param current True if ledger is current. - * @return True is saving was successfull. - */ - virtual bool - saveValidatedLedger( - std::shared_ptr const& ledger, - bool current) = 0; - - /** - * @brief getLimitedOldestLedgerInfo Returns info of oldest ledger - * from ledgers with sequences greater or equal to given. - * @param ledgerFirstIndex Minimum ledger sequence. - * @return Ledger info or none if ledger not found. - */ - virtual std::optional - getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) = 0; - - /** - * @brief getLimitedNewestLedgerInfo Returns info of newest ledger - * from ledgers with sequences greater or equal to given. - * @param ledgerFirstIndex Minimum ledger sequence. - * @return Ledger info or none if ledger not found. - */ - virtual std::optional - getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) = 0; - - /** - * @brief getOldestAccountTxs Returns oldest transactions for given - * account which match given criteria starting from given offset. - * @param options Struct AccountTxOptions which contain criteria to match: - * the account, minimum and maximum ledger numbers to search, - * offset of first entry to return, number of transactions to return, - * flag if this number unlimited. - * @return Vector of pairs of found transactions and their metadata - * sorted in ascending order by account sequence. - */ - virtual AccountTxs - getOldestAccountTxs(AccountTxOptions const& options) = 0; - - /** - * @brief getNewestAccountTxs Returns newest transactions for given - * account which match given criteria starting from given offset. - * @param options Struct AccountTxOptions which contain criteria to match: - * the account, minimum and maximum ledger numbers to search, - * offset of first entry to return, number of transactions to return, - * flag if this number unlimited. - * @return Vector of pairs of found transactions and their metadata - * sorted in descending order by account sequence. - */ - virtual AccountTxs - getNewestAccountTxs(AccountTxOptions const& options) = 0; - - /** - * @brief getOldestAccountTxsB Returns oldest transactions in binary form - * for given account which match given criteria starting from given - * offset. - * @param options Struct AccountTxOptions which contain criteria to match: - * the account, minimum and maximum ledger numbers to search, - * offset of first entry to return, number of transactions to return, - * flag if this number unlimited. - * @return Vector of tuples of found transactions, their metadata and - * account sequences sorted in ascending order by account sequence. - */ - virtual MetaTxsList - getOldestAccountTxsB(AccountTxOptions const& options) = 0; - - /** - * @brief getNewestAccountTxsB Returns newest transactions in binary form - * for given account which match given criteria starting from given - * offset. - * @param options Struct AccountTxOptions which contain criteria to match: - * the account, minimum and maximum ledger numbers to search, - * offset of first entry to return, number of transactions to return, - * flag if this number unlimited. - * @return Vector of tuples of found transactions, their metadata and - * account sequences sorted in descending order by account - * sequence. - */ - virtual MetaTxsList - getNewestAccountTxsB(AccountTxOptions const& options) = 0; - - /** - * @brief oldestAccountTxPage Returns oldest transactions for given - * account which match given criteria starting from given marker. - * @param options Struct AccountTxPageOptions which contain criteria to - * match: the account, minimum and maximum ledger numbers to search, - * marker of first returned entry, number of transactions to return, - * flag if this number unlimited. - * @return Vector of pairs of found transactions and their metadata - * sorted in ascending order by account sequence and marker - * for next search if search not finished. - */ - virtual std::pair> - oldestAccountTxPage(AccountTxPageOptions const& options) = 0; - - /** - * @brief newestAccountTxPage Returns newest transactions for given - * account which match given criteria starting from given marker. - * @param options Struct AccountTxPageOptions which contain criteria to - * match: the account, minimum and maximum ledger numbers to search, - * marker of first returned entry, number of transactions to return, - * flag if this number unlimited. - * @return Vector of pairs of found transactions and their metadata - * sorted in descending order by account sequence and marker - * for next search if search not finished. - */ - virtual std::pair> - newestAccountTxPage(AccountTxPageOptions const& options) = 0; - - /** - * @brief oldestAccountTxPageB Returns oldest transactions in binary form - * for given account which match given criteria starting from given - * marker. - * @param options Struct AccountTxPageOptions which contain criteria to - * match: the account, minimum and maximum ledger numbers to search, - * marker of first returned entry, number of transactions to return, - * flag if this number unlimited. - * @return Vector of tuples of found transactions, their metadata and - * account sequences sorted in ascending order by account - * sequence and marker for next search if search not finished. - */ - virtual std::pair> - oldestAccountTxPageB(AccountTxPageOptions const& options) = 0; - - /** - * @brief newestAccountTxPageB Returns newest transactions in binary form - * for given account which match given criteria starting from given - * marker. - * @param options Struct AccountTxPageOptions which contain criteria to - * match: the account, minimum and maximum ledger numbers to search, - * marker of first returned entry, number of transactions to return, - * flag if this number unlimited. - * @return Vector of tuples of found transactions, their metadata and - * account sequences sorted in descending order by account - * sequence and marker for next search if search not finished. - */ - virtual std::pair> - newestAccountTxPageB(AccountTxPageOptions const& options) = 0; - - /** - * @brief getTransaction Returns transaction with given hash. If not found - * and range given then check if all ledgers from the range are - * present in the database. - * @param id Hash of the transaction. - * @param range Range of ledgers to check, if present. - * @param ec Default value of error code. - * @return Transaction and its metadata if found, TxSearched::all if range - * given and all ledgers from range are present in the database, - * TxSearched::some if range given and not all ledgers are present, - * TxSearched::unknown if range not given or deserializing error - * occured. In the last case error code returned via ec link - * parameter, in other cases default error code not changed. - */ - virtual std::variant - getTransaction( - uint256 const& id, - std::optional> const& range, - error_code_i& ec) = 0; - - /** - * @brief getKBUsedAll Returns space used by all databases. - * @return Space in kilobytes. - */ - virtual uint32_t - getKBUsedAll() = 0; - - /** - * @brief getKBUsedLedger Returns space used by ledger database. - * @return Space in kilobytes. - */ - virtual uint32_t - getKBUsedLedger() = 0; - - /** - * @brief getKBUsedTransaction Returns space used by transaction - * database. - * @return Space in kilobytes. - */ - virtual uint32_t - getKBUsedTransaction() = 0; - - /** - * @brief Closes the ledger database - */ - virtual void - closeLedgerDB() = 0; - - /** - * @brief Closes the transaction database - */ - virtual void - closeTransactionDB() = 0; -}; - -} // namespace ripple - -#endif diff --git a/src/ripple/app/rdb/backend/SQLiteDatabase.h b/src/ripple/app/rdb/backend/SQLiteDatabase.h new file mode 100644 index 000000000..07f9be3e2 --- /dev/null +++ b/src/ripple/app/rdb/backend/SQLiteDatabase.h @@ -0,0 +1,313 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_RDB_BACKEND_SQLITEDATABASE_H_INCLUDED +#define RIPPLE_APP_RDB_BACKEND_SQLITEDATABASE_H_INCLUDED + +#include + +namespace ripple { + +class SQLiteDatabase : public RelationalDatabase +{ +public: + /** + * @brief getTransactionsMinLedgerSeq Returns the minimum ledger sequence + * stored in the Transactions table. + * @return Ledger sequence or no value if no ledgers exist. + */ + virtual std::optional + getTransactionsMinLedgerSeq() = 0; + + /** + * @brief getAccountTransactionsMinLedgerSeq Returns the minimum ledger + * sequence stored in the AccountTransactions table. + * @return Ledger sequence or no value if no ledgers exist. + */ + virtual std::optional + getAccountTransactionsMinLedgerSeq() = 0; + + /** + * @brief deleteTransactionByLedgerSeq Deletes transactions from the ledger + * with the given sequence. + * @param ledgerSeq Ledger sequence. + */ + virtual void + deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) = 0; + + /** + * @brief deleteBeforeLedgerSeq Deletes all ledgers with a sequence number + * less than or equal to the given ledger sequence. + * @param ledgerSeq Ledger sequence. + */ + virtual void + deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) = 0; + + /** + * @brief deleteTransactionsBeforeLedgerSeq Deletes all transactions with + * a sequence number less than or equal to the given ledger + * sequence. + * @param ledgerSeq Ledger sequence. + */ + virtual void + deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) = 0; + + /** + * @brief deleteAccountTransactionsBeforeLedgerSeq Deletes all account + * transactions with a sequence number less than or equal to the + * given ledger sequence. + * @param ledgerSeq Ledger sequence. + */ + virtual void + deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) = 0; + + /** + * @brief getTransactionCount Returns the number of transactions. + * @return Number of transactions. + */ + virtual std::size_t + getTransactionCount() = 0; + + /** + * @brief getAccountTransactionCount Returns the number of account + * transactions. + * @return Number of account transactions. + */ + virtual std::size_t + getAccountTransactionCount() = 0; + + /** + * @brief getLedgerCountMinMax Returns the minimum ledger sequence, + * maximum ledger sequence and total number of saved ledgers. + * @return Struct CountMinMax which contains the minimum sequence, + * maximum sequence and number of ledgers. + */ + virtual struct CountMinMax + getLedgerCountMinMax() = 0; + + /** + * @brief saveValidatedLedger Saves a ledger into the database. + * @param ledger The ledger. + * @param current True if the ledger is current. + * @return True if saving was successful. + */ + virtual bool + saveValidatedLedger( + std::shared_ptr const& ledger, + bool current) = 0; + + /** + * @brief getLimitedOldestLedgerInfo Returns the info of the oldest ledger + * whose sequence number is greater than or equal to the given + * sequence number. + * @param ledgerFirstIndex Minimum ledger sequence. + * @return Ledger info if found, otherwise no value. + */ + virtual std::optional + getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) = 0; + + /** + * @brief getLimitedNewestLedgerInfo Returns the info of the newest ledger + * whose sequence number is greater than or equal to the given + * sequence number. + * @param ledgerFirstIndex Minimum ledger sequence. + * @return Ledger info if found, otherwise no value. + */ + virtual std::optional + getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) = 0; + + /** + * @brief getOldestAccountTxs Returns the oldest transactions for the + * account that matches the given criteria starting from the provided + * offset. + * @param options Struct AccountTxOptions which contains the criteria to + * match: the account, ledger search range, the offset of the first + * entry to return, the number of transactions to return, a flag if + * this number is unlimited. + * @return Vector of pairs of found transactions and their metadata + * sorted in ascending order by account sequence. + */ + virtual AccountTxs + getOldestAccountTxs(AccountTxOptions const& options) = 0; + + /** + * @brief getNewestAccountTxs Returns the newest transactions for the + * account that matches the given criteria starting from the provided + * offset. + * @param options Struct AccountTxOptions which contains the criteria to + * match: the account, the ledger search range, the offset of the + * first entry to return, the number of transactions to return, a + * flag if this number unlimited. + * @return Vector of pairs of found transactions and their metadata + * sorted in descending order by account sequence. + */ + virtual AccountTxs + getNewestAccountTxs(AccountTxOptions const& options) = 0; + + /** + * @brief getOldestAccountTxsB Returns the oldest transactions in binary + * form for the account that matches the given criteria starting from + * the provided offset. + * @param options Struct AccountTxOptions which contains the criteria to + * match: the account, the ledger search range, the offset of the + * first entry to return, the number of transactions to return, a + * flag if this number unlimited. + * @return Vector of tuples of found transactions, their metadata and + * account sequences sorted in ascending order by account sequence. + */ + virtual MetaTxsList + getOldestAccountTxsB(AccountTxOptions const& options) = 0; + + /** + * @brief getNewestAccountTxsB Returns the newest transactions in binary + * form for the account that matches the given criteria starting from + * the provided offset. + * @param options Struct AccountTxOptions which contains the criteria to + * match: the account, the ledger search range, the offset of the + * first entry to return, the number of transactions to return, a + * flag if this number is unlimited. + * @return Vector of tuples of found transactions, their metadata and + * account sequences sorted in descending order by account + * sequence. + */ + virtual MetaTxsList + getNewestAccountTxsB(AccountTxOptions const& options) = 0; + + /** + * @brief oldestAccountTxPage Returns the oldest transactions for the + * account that matches the given criteria starting from the + * provided marker. + * @param options Struct AccountTxPageOptions which contains the criteria to + * match: the account, the ledger search range, the marker of first + * returned entry, the number of transactions to return, a flag if + * this number is unlimited. + * @return Vector of pairs of found transactions and their metadata + * sorted in ascending order by account sequence and a marker + * for the next search if the search was not finished. + */ + virtual std::pair> + oldestAccountTxPage(AccountTxPageOptions const& options) = 0; + + /** + * @brief newestAccountTxPage Returns the newest transactions for the + * account that matches the given criteria starting from the provided + * marker. + * @param options Struct AccountTxPageOptions which contains the criteria to + * match: the account, the ledger search range, the marker of the + * first returned entry, the number of transactions to return, a flag + * if this number unlimited. + * @return Vector of pairs of found transactions and their metadata + * sorted in descending order by account sequence and a marker + * for the next search if the search was not finished. + */ + virtual std::pair> + newestAccountTxPage(AccountTxPageOptions const& options) = 0; + + /** + * @brief oldestAccountTxPageB Returns the oldest transactions in binary + * form for the account that matches the given criteria starting from + * the provided marker. + * @param options Struct AccountTxPageOptions which contains criteria to + * match: the account, the ledger search range, the marker of the + * first returned entry, the number of transactions to return, a flag + * if this number unlimited. + * @return Vector of tuples of found transactions, their metadata and + * account sequences sorted in ascending order by account + * sequence and a marker for the next search if the search was not + * finished. + */ + virtual std::pair> + oldestAccountTxPageB(AccountTxPageOptions const& options) = 0; + + /** + * @brief newestAccountTxPageB Returns the newest transactions in binary + * form for the account that matches the given criteria starting from + * the provided marker. + * @param options Struct AccountTxPageOptions which contains the criteria to + * match: the account, the ledger search range, the marker of the + * first returned entry, the number of transactions to return, a flag + * if this number is unlimited. + * @return Vector of tuples of found transactions, their metadata and + * account sequences sorted in descending order by account + * sequence and a marker for the next search if the search was not + * finished. + */ + virtual std::pair> + newestAccountTxPageB(AccountTxPageOptions const& options) = 0; + + /** + * @brief getTransaction Returns the transaction with the given hash. If a + * range is provided but the transaction is not found, then check if + * all ledgers in the range are present in the database. + * @param id Hash of the transaction. + * @param range Range of ledgers to check, if present. + * @param ec Default error code value. + * @return Transaction and its metadata if found, otherwise TxSearched::all + * if a range is provided and all ledgers from the range are present + * in the database, TxSearched::some if a range is provided and not + * all ledgers are present, TxSearched::unknown if the range is not + * provided or a deserializing error occurred. In the last case the + * error code is returned via the ec parameter, in other cases the + * default error code is not changed. + */ + virtual std::variant + getTransaction( + uint256 const& id, + std::optional> const& range, + error_code_i& ec) = 0; + + /** + * @brief getKBUsedAll Returns the amount of space used by all databases. + * @return Space in kilobytes. + */ + virtual uint32_t + getKBUsedAll() = 0; + + /** + * @brief getKBUsedLedger Returns the amount of space space used by the + * ledger database. + * @return Space in kilobytes. + */ + virtual uint32_t + getKBUsedLedger() = 0; + + /** + * @brief getKBUsedTransaction Returns the amount of space used by the + * transaction database. + * @return Space in kilobytes. + */ + virtual uint32_t + getKBUsedTransaction() = 0; + + /** + * @brief Closes the ledger database + */ + virtual void + closeLedgerDB() = 0; + + /** + * @brief Closes the transaction database + */ + virtual void + closeTransactionDB() = 0; +}; + +} // namespace ripple + +#endif diff --git a/src/ripple/app/rdb/RelationalDBInterface_nodes.h b/src/ripple/app/rdb/backend/detail/Node.h similarity index 93% rename from src/ripple/app/rdb/RelationalDBInterface_nodes.h rename to src/ripple/app/rdb/backend/detail/Node.h index 338e94fb2..ab3a99f5f 100644 --- a/src/ripple/app/rdb/RelationalDBInterface_nodes.h +++ b/src/ripple/app/rdb/backend/detail/Node.h @@ -17,18 +17,19 @@ */ //============================================================================== -#ifndef RIPPLE_CORE_RELATIONALDBINTERFACE_NODES_H_INCLUDED -#define RIPPLE_CORE_RELATIONALDBINTERFACE_NODES_H_INCLUDED +#ifndef RIPPLE_APP_RDB_BACKEND_DETAIL_NODE_H_INCLUDED +#define RIPPLE_APP_RDB_BACKEND_DETAIL_NODE_H_INCLUDED #include #include -#include +#include #include #include #include #include namespace ripple { +namespace detail { /* Need to change TableTypeCount if TableType is modified. */ enum class TableType { Ledgers, Transactions, AccountTransactions }; @@ -109,14 +110,14 @@ std::size_t getRows(soci::session& session, TableType type); /** - * @brief getRowsMinMax Returns minumum ledger sequence, + * @brief getRowsMinMax Returns minimum ledger sequence, * maximum ledger sequence and total number of rows in given table. * @param session Session with database. * @param type Table ID for which the result is returned. * @return Struct CountMinMax which contain minimum sequence, * maximum sequence and number of rows. */ -RelationalDBInterface::CountMinMax +RelationalDatabase::CountMinMax getRowsMinMax(soci::session& session, TableType type); /** @@ -232,7 +233,7 @@ getHashesByIndex( * @param maxSeq Maximum ledger sequence. * @param j Journal. * @return Map which points sequence number of found ledger to the struct - * LedgerHashPair which contauns ledger hash and its parent hash. + * LedgerHashPair which contains ledger hash and its parent hash. */ std::map getHashesByIndex( @@ -283,12 +284,12 @@ getTxHistory( * skipped. We need to skip some quantity of transactions if option * offset is > 0 in the options structure. */ -std::pair +std::pair getOldestAccountTxs( soci::session& session, Application& app, LedgerMaster& ledgerMaster, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, beast::Journal j); @@ -314,12 +315,12 @@ getOldestAccountTxs( * skipped. We need to skip some quantity of transactions if option * offset is > 0 in the options structure. */ -std::pair +std::pair getNewestAccountTxs( soci::session& session, Application& app, LedgerMaster& ledgerMaster, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, beast::Journal j); @@ -345,11 +346,11 @@ getNewestAccountTxs( * skipped. We need to skip some quantity of transactions if option * offset is > 0 in the options structure. */ -std::pair, int> +std::pair, int> getOldestAccountTxsB( soci::session& session, Application& app, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, beast::Journal j); @@ -375,11 +376,11 @@ getOldestAccountTxsB( * skipped. We need to skip some quantity of transactions if option * offset is > 0 in the options structure. */ -std::pair, int> +std::pair, int> getNewestAccountTxsB( soci::session& session, Application& app, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, beast::Journal j); @@ -388,7 +389,6 @@ getNewestAccountTxsB( * account which match given criteria starting from given marker * and calls callback for each found transaction. * @param session Session with database. - * @param idCache Account ID cache. * @param onUnsavedLedger Callback function to call on each found unsaved * ledger within given range. * @param onTransaction Callback function to call on each found transaction. @@ -404,15 +404,14 @@ getNewestAccountTxsB( * sequence and marker for next search if search not finished. * Also number of transactions processed during this call. */ -std::pair, int> +std::pair, int> oldestAccountTxPage( soci::session& session, - AccountIDCache const& idCache, std::function const& onUnsavedLedger, std::function< void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, - RelationalDBInterface::AccountTxPageOptions const& options, + RelationalDatabase::AccountTxPageOptions const& options, int limit_used, std::uint32_t page_length); @@ -421,7 +420,6 @@ oldestAccountTxPage( * account which match given criteria starting from given marker * and calls callback for each found transaction. * @param session Session with database. - * @param idCache Account ID cache. * @param onUnsavedLedger Callback function to call on each found unsaved * ledger within given range. * @param onTransaction Callback function to call on each found transaction. @@ -437,15 +435,14 @@ oldestAccountTxPage( * sequence and marker for next search if search not finished. * Also number of transactions processed during this call. */ -std::pair, int> +std::pair, int> newestAccountTxPage( soci::session& session, - AccountIDCache const& idCache, std::function const& onUnsavedLedger, std::function< void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, - RelationalDBInterface::AccountTxPageOptions const& options, + RelationalDatabase::AccountTxPageOptions const& options, int limit_used, std::uint32_t page_length); @@ -465,7 +462,7 @@ newestAccountTxPage( * occured. In the last case error code modified in ec link * parameter, in other cases default error code remained. */ -std::variant +std::variant getTransaction( soci::session& session, Application& app, @@ -483,6 +480,7 @@ getTransaction( bool dbHasSpace(soci::session& session, Config const& config, beast::Journal j); +} // namespace detail } // namespace ripple #endif diff --git a/src/ripple/app/rdb/backend/detail/Shard.h b/src/ripple/app/rdb/backend/detail/Shard.h new file mode 100644 index 000000000..ac88c24bd --- /dev/null +++ b/src/ripple/app/rdb/backend/detail/Shard.h @@ -0,0 +1,90 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APP_RDB_BACKEND_DETAIL_SHARD_H_INCLUDED +#define RIPPLE_APP_RDB_BACKEND_DETAIL_SHARD_H_INCLUDED + +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace detail { + +/** + * @brief makeMetaDBs Opens ledger and transaction 'meta' databases which + * map ledger hashes and transaction IDs to the index of the shard + * that holds the ledger or transaction. + * @param config Config object. + * @param setup Path to database and opening parameters. + * @param checkpointerSetup Database checkpointer setup. + * @return Struct DatabasePair which contains unique pointers to the ledger + * and transaction databases. + */ +DatabasePair +makeMetaDBs( + Config const& config, + DatabaseCon::Setup const& setup, + DatabaseCon::CheckpointerSetup const& checkpointerSetup); + +/** + * @brief saveLedgerMeta Stores (transaction ID -> shard index) and + * (ledger hash -> shard index) mappings in the meta databases. + * @param ledger The ledger. + * @param app Application object. + * @param lgrMetaSession Session to ledger meta database. + * @param txnMetaSession Session to transaction meta database. + * @param shardIndex The index of the shard that contains this ledger. + * @return True on success. + */ +bool +saveLedgerMeta( + std::shared_ptr const& ledger, + Application& app, + soci::session& lgrMetaSession, + soci::session& txnMetaSession, + std::uint32_t shardIndex); + +/** + * @brief getShardIndexforLedger Queries the ledger meta database to + * retrieve the index of the shard that contains this ledger. + * @param session Session to the database. + * @param hash Hash of the ledger. + * @return The index of the shard on success, otherwise an unseated value. + */ +std::optional +getShardIndexforLedger(soci::session& session, LedgerHash const& hash); + +/** + * @brief getShardIndexforTransaction Queries the transaction meta database to + * retrieve the index of the shard that contains this transaction. + * @param session Session to the database. + * @param id ID of the transaction. + * @return The index of the shard on success, otherwise an unseated value. + */ +std::optional +getShardIndexforTransaction(soci::session& session, TxID const& id); + +} // namespace detail +} // namespace ripple + +#endif diff --git a/src/ripple/app/rdb/impl/RelationalDBInterface_nodes.cpp b/src/ripple/app/rdb/backend/detail/impl/Node.cpp similarity index 85% rename from src/ripple/app/rdb/impl/RelationalDBInterface_nodes.cpp rename to src/ripple/app/rdb/backend/detail/impl/Node.cpp index c067bfe0c..b3b354ebe 100644 --- a/src/ripple/app/rdb/impl/RelationalDBInterface_nodes.cpp +++ b/src/ripple/app/rdb/backend/detail/impl/Node.cpp @@ -23,8 +23,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include @@ -35,10 +35,11 @@ #include namespace ripple { +namespace detail { /** - * @brief to_string Returns name of table by table ID. - * @param type Table ID. + * @brief to_string Returns the name of a table according to its TableType. + * @param type An enum denoting the table's type. * @return Name of the table. */ static std::string @@ -47,6 +48,7 @@ to_string(TableType type) static_assert( TableTypeCount == 3, "Need to modify switch statement if enum is modified"); + switch (type) { case TableType::Ledgers: @@ -56,7 +58,7 @@ to_string(TableType type) case TableType::AccountTransactions: return "AccountTransactions"; default: - assert(0); + assert(false); return "Unknown"; } } @@ -166,10 +168,10 @@ getRows(soci::session& session, TableType type) return rows; } -RelationalDBInterface::CountMinMax +RelationalDatabase::CountMinMax getRowsMinMax(soci::session& session, TableType type) { - RelationalDBInterface::CountMinMax res; + RelationalDatabase::CountMinMax res; session << "SELECT COUNT(*) AS rows, " "MIN(LedgerSeq) AS first, " "MAX(LedgerSeq) AS last " @@ -305,7 +307,7 @@ saveValidatedLedger( sql += txnId; sql += "','"; - sql += app.accountIDCache().toBase58(account); + sql += toBase58(account); sql += "',"; sql += ledgerSeq; sql += ","; @@ -378,12 +380,12 @@ saveValidatedLedger( } /** - * @brief getLedgerInfo Returns info of ledger with special condition - * given as SQL query. - * @param session Session with database. - * @param sqlSuffix Special condition for found the ledger. + * @brief getLedgerInfo Returns the info of the ledger retrieved from the + * database by using the provided SQL query suffix. + * @param session Session with the database. + * @param sqlSuffix SQL string used to specify the sought ledger. * @param j Journal. - * @return Ledger info or none if ledger not found. + * @return Ledger info or no value if the ledger was not found. */ static std::optional getLedgerInfo( @@ -674,21 +676,22 @@ getTxHistory( } /** - * @brief transactionsSQL Returns SQL query to select oldest or newest - * transactions in decoded or binary form for given account which - * match given criteria starting from given offset. + * @brief transactionsSQL Returns a SQL query for selecting the oldest or newest + * transactions in decoded or binary form for the account that matches + * the given criteria starting from the provided offset. * @param app Application object. - * @param selection List of table fields to select from database. - * @param options Struct AccountTxOptions which contain criteria to match: - * the account, minimum and maximum ledger numbers to search, - * offset of first entry to return, number of transactions to return, - * flag if this number unlimited. + * @param selection List of table fields to select from the database. + * @param options Struct AccountTxOptions which contains the criteria to match: + * the account, the ledger search range, the offset of the first entry to + * return, the number of transactions to return, and a flag if this + * number is unlimited. * @param limit_used Number of transactions already returned in calls - * to another shard databases, if shard databases are used. - * None if node database is used. + * to other shard databases, if shard databases are used. + * No value if the node database is used. * @param descending True for descending order, false for ascending. * @param binary True for binary form, false for decoded. - * @param count True for count number of transaction, false for select it. + * @param count True for counting the number of transactions, false for + * selecting them. * @param j Journal. * @return SQL query string. */ @@ -696,7 +699,7 @@ static std::string transactionsSQL( Application& app, std::string selection, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, bool descending, bool binary, @@ -757,8 +760,7 @@ transactionsSQL( sql = boost::str( boost::format("SELECT %s FROM AccountTransactions " "WHERE Account = '%s' %s %s LIMIT %u, %u;") % - selection % app.accountIDCache().toBase58(options.account) % - maxClause % minClause % + selection % toBase58(options.account) % maxClause % minClause % beast::lexicalCastThrow(options.offset) % beast::lexicalCastThrow(numberOfResults)); else @@ -771,9 +773,9 @@ transactionsSQL( "ORDER BY AccountTransactions.LedgerSeq %s, " "AccountTransactions.TxnSeq %s, AccountTransactions.TransID %s " "LIMIT %u, %u;") % - selection % app.accountIDCache().toBase58(options.account) % - maxClause % minClause % (descending ? "DESC" : "ASC") % + selection % toBase58(options.account) % maxClause % minClause % (descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") % + (descending ? "DESC" : "ASC") % beast::lexicalCastThrow(options.offset) % beast::lexicalCastThrow(numberOfResults)); JLOG(j.trace()) << "txSQL query: " << sql; @@ -781,39 +783,40 @@ transactionsSQL( } /** - * @brief getAccountTxs Returns oldest or newest transactions for given - * account which match given criteria starting from given offset. - * @param session Session with database. + * @brief getAccountTxs Returns the oldest or newest transactions for the + * account that matches the given criteria starting from the provided + * offset. + * @param session Session with the database. * @param app Application object. * @param ledgerMaster LedgerMaster object. - * @param options Struct AccountTxOptions which contain criteria to match: - * the account, minimum and maximum ledger numbers to search, - * offset of first entry to return, number of transactions to return, - * flag if this number unlimited. + * @param options Struct AccountTxOptions which contains the criteria to match: + * the account, the ledger search range, the offset of the first entry to + * return, the number of transactions to return, and a flag if this + * number is unlimited. * @param limit_used Number of transactions already returned in calls - * to another shard databases, if shard databases are used. - * None if node database is used. + * to other shard databases, if shard databases are used. + * No value if the node database is used. * @param descending True for descending order, false for ascending. * @param j Journal. - * @return Vector of pairs of found transactions and its metadata - * sorted in given order by account sequence. - * Also the number of transactions processed or skipped. - * If this number is >= 0, then it means number of transactions - * processed, if it is < 0, then -number means number of transactions - * skipped. We need to skip some quantity of transactions if option - * offset is > 0 in the options structure. + * @return Vector of pairs of found transactions and their metadata sorted by + * account sequence in the specified order along with the number of + * transactions processed or skipped. If this number is >= 0, then it + * represents the number of transactions processed, if it is < 0, then + * -number represents the number of transactions skipped. We need to + * skip some number of transactions if option offset is > 0 in the + * options structure. */ -static std::pair +static std::pair getAccountTxs( soci::session& session, Application& app, LedgerMaster& ledgerMaster, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, bool descending, beast::Journal j) { - RelationalDBInterface::AccountTxs ret; + RelationalDatabase::AccountTxs ret; std::string sql = transactionsSQL( app, @@ -883,7 +886,7 @@ getAccountTxs( if (!total && limit_used) { - RelationalDBInterface::AccountTxOptions opt = options; + RelationalDatabase::AccountTxOptions opt = options; opt.offset = 0; std::string sql1 = transactionsSQL( app, "COUNT(*)", opt, limit_used, descending, false, false, j); @@ -897,12 +900,12 @@ getAccountTxs( return {ret, total}; } -std::pair +std::pair getOldestAccountTxs( soci::session& session, Application& app, LedgerMaster& ledgerMaster, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, beast::Journal j) { @@ -910,12 +913,12 @@ getOldestAccountTxs( session, app, ledgerMaster, options, limit_used, false, j); } -std::pair +std::pair getNewestAccountTxs( soci::session& session, Application& app, LedgerMaster& ledgerMaster, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, beast::Journal j) { @@ -924,38 +927,38 @@ getNewestAccountTxs( } /** - * @brief getAccountTxsB Returns oldest or newset transactions in binary - * form for given account which match given criteria starting from - * given offset. - * @param session Session with database. + * @brief getAccountTxsB Returns the oldest or newest transactions in binary + * form for the account that matches given criteria starting from + * the provided offset. + * @param session Session with the database. * @param app Application object. - * @param options Struct AccountTxOptions which contain criteria to match: - * the account, minimum and maximum ledger numbers to search, - * offset of first entry to return, number of transactions to return, - * flag if this number unlimited. - * @param limit_used Number or transactions already returned in calls - * to another shard databases, if shard databases are used. - * None if node database is used. + * @param options Struct AccountTxOptions which contains the criteria to match: + * the account, the ledger search range, the offset of the first entry to + * return, the number of transactions to return, and a flag if this + * number is unlimited. + * @param limit_used Number of transactions already returned in calls to other + * shard databases, if shard databases are used. No value if the node + * database is used. * @param descending True for descending order, false for ascending. * @param j Journal. - * @return Vector of tuples of found transactions, its metadata and - * account sequences sorted in given order by account - * sequence. Also number of transactions processed or skipped. - * If this number is >= 0, then it means number of transactions - * processed, if it is < 0, then -number means number of transactions - * skipped. We need to skip some quantity of transactions if option - * offset is > 0 in the options structure. + * @return Vector of tuples each containing (the found transactions, their + * metadata, and their account sequences) sorted by account sequence in + * the specified order along with the number of transactions processed + * or skipped. If this number is >= 0, then it represents the number of + * transactions processed, if it is < 0, then -number represents the + * number of transactions skipped. We need to skip some number of + * transactions if option offset is > 0 in the options structure. */ -static std::pair, int> +static std::pair, int> getAccountTxsB( soci::session& session, Application& app, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, bool descending, beast::Journal j) { - std::vector ret; + std::vector ret; std::string sql = transactionsSQL( app, @@ -1004,7 +1007,7 @@ getAccountTxsB( if (!total && limit_used) { - RelationalDBInterface::AccountTxOptions opt = options; + RelationalDatabase::AccountTxOptions opt = options; opt.offset = 0; std::string sql1 = transactionsSQL( app, "COUNT(*)", opt, limit_used, descending, true, false, j); @@ -1018,22 +1021,22 @@ getAccountTxsB( return {ret, total}; } -std::pair, int> +std::pair, int> getOldestAccountTxsB( soci::session& session, Application& app, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, beast::Journal j) { return getAccountTxsB(session, app, options, limit_used, false, j); } -std::pair, int> +std::pair, int> getNewestAccountTxsB( soci::session& session, Application& app, - RelationalDBInterface::AccountTxOptions const& options, + RelationalDatabase::AccountTxOptions const& options, std::optional const& limit_used, beast::Journal j) { @@ -1041,36 +1044,34 @@ getNewestAccountTxsB( } /** - * @brief accountTxPage Searches oldest or newest transactions for given - * account which match given criteria starting from given marker - * and calls callback for each found transaction. - * @param session Session with database. - * @param idCache Account ID cache. + * @brief accountTxPage Searches for the oldest or newest transactions for the + * account that matches the given criteria starting from the provided + * marker and invokes the callback parameter for each found transaction. + * @param session Session with the database. * @param onUnsavedLedger Callback function to call on each found unsaved - * ledger within given range. - * @param onTransaction Callback function to call on eahc found transaction. - * @param options Struct AccountTxPageOptions which contain criteria to - * match: the account, minimum and maximum ledger numbers to search, - * marker of first returned entry, number of transactions to return, - * flag if this number unlimited. - * @param limit_used Number or transactions already returned in calls - * to another shard databases. + * ledger within the given range. + * @param onTransaction Callback function to call on each found transaction. + * @param options Struct AccountTxPageOptions which contains the criteria to + * match: the account, the ledger search range, the marker of the first + * returned entry, the number of transactions to return, and a flag if + * this number unlimited. + * @param limit_used Number of transactions already returned in calls + * to other shard databases. * @param page_length Total number of transactions to return. * @param forward True for ascending order, false for descending. - * @return Vector of tuples of found transactions, its metadata and - * account sequences sorted in given order by account - * sequence and marker for next search if search not finished. - * Also number of transactions processed during this call. + * @return Vector of tuples of found transactions, their metadata and account + * sequences sorted in the specified order by account sequence, a marker + * for the next search if the search was not finished and the number of + * transactions processed during this call. */ -static std::pair, int> +static std::pair, int> accountTxPage( soci::session& session, - AccountIDCache const& idCache, std::function const& onUnsavedLedger, std::function< void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, - RelationalDBInterface::AccountTxPageOptions const& options, + RelationalDatabase::AccountTxPageOptions const& options, int limit_used, std::uint32_t page_length, bool forward) @@ -1105,7 +1106,7 @@ accountTxPage( findSeq = options.marker->txnSeq; } - std::optional newmarker; + std::optional newmarker; if (limit_used > 0) newmarker = options.marker; @@ -1131,8 +1132,8 @@ accountTxPage( ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s LIMIT %u;)")) % - idCache.toBase58(options.account) % options.minLedger % - options.maxLedger % order % order % queryLimit); + toBase58(options.account) % options.minLedger % options.maxLedger % + order % order % queryLimit); } else { @@ -1142,7 +1143,7 @@ accountTxPage( const std::uint32_t maxLedger = forward ? options.maxLedger : findLedger - 1; - auto b58acct = idCache.toBase58(options.account); + auto b58acct = toBase58(options.account); sql = boost::str( boost::format(( R"(SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq, @@ -1243,21 +1244,19 @@ accountTxPage( return {newmarker, total}; } -std::pair, int> +std::pair, int> oldestAccountTxPage( soci::session& session, - AccountIDCache const& idCache, std::function const& onUnsavedLedger, std::function< void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, - RelationalDBInterface::AccountTxPageOptions const& options, + RelationalDatabase::AccountTxPageOptions const& options, int limit_used, std::uint32_t page_length) { return accountTxPage( session, - idCache, onUnsavedLedger, onTransaction, options, @@ -1266,21 +1265,19 @@ oldestAccountTxPage( true); } -std::pair, int> +std::pair, int> newestAccountTxPage( soci::session& session, - AccountIDCache const& idCache, std::function const& onUnsavedLedger, std::function< void(std::uint32_t, std::string const&, Blob&&, Blob&&)> const& onTransaction, - RelationalDBInterface::AccountTxPageOptions const& options, + RelationalDatabase::AccountTxPageOptions const& options, int limit_used, std::uint32_t page_length) { return accountTxPage( session, - idCache, onUnsavedLedger, onTransaction, options, @@ -1289,7 +1286,7 @@ newestAccountTxPage( false); } -std::variant +std::variant getTransaction( soci::session& session, Application& app, @@ -1435,4 +1432,5 @@ dbHasSpace(soci::session& session, Config const& config, beast::Journal j) return true; } +} // namespace detail } // namespace ripple diff --git a/src/ripple/app/rdb/backend/detail/impl/Shard.cpp b/src/ripple/app/rdb/backend/detail/impl/Shard.cpp new file mode 100644 index 000000000..f7a0ce457 --- /dev/null +++ b/src/ripple/app/rdb/backend/detail/impl/Shard.cpp @@ -0,0 +1,147 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2020 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace detail { + +DatabasePair +makeMetaDBs( + Config const& config, + DatabaseCon::Setup const& setup, + DatabaseCon::CheckpointerSetup const& checkpointerSetup) +{ + // ledger meta database + auto lgrMetaDB{std::make_unique( + setup, + LgrMetaDBName, + LgrMetaDBPragma, + LgrMetaDBInit, + checkpointerSetup)}; + + if (!config.useTxTables()) + return {std::move(lgrMetaDB), nullptr}; + + // transaction meta database + auto txMetaDB{std::make_unique( + setup, TxMetaDBName, TxMetaDBPragma, TxMetaDBInit, checkpointerSetup)}; + + return {std::move(lgrMetaDB), std::move(txMetaDB)}; +} + +bool +saveLedgerMeta( + std::shared_ptr const& ledger, + Application& app, + soci::session& lgrMetaSession, + soci::session& txnMetaSession, + std::uint32_t const shardIndex) +{ + std::string_view constexpr lgrSQL = + R"sql(INSERT OR REPLACE INTO LedgerMeta VALUES + (:ledgerHash,:shardIndex);)sql"; + + auto const hash = to_string(ledger->info().hash); + lgrMetaSession << lgrSQL, soci::use(hash), soci::use(shardIndex); + + if (!app.config().useTxTables()) + return true; + + auto const aLedger = [&app, ledger]() -> std::shared_ptr { + try + { + auto aLedger = + app.getAcceptedLedgerCache().fetch(ledger->info().hash); + if (!aLedger) + { + aLedger = std::make_shared(ledger, app); + app.getAcceptedLedgerCache().canonicalize_replace_client( + ledger->info().hash, aLedger); + } + + return aLedger; + } + catch (std::exception const&) + { + JLOG(app.journal("Ledger").warn()) + << "An accepted ledger was missing nodes"; + } + + return {}; + }(); + + if (!aLedger) + return false; + + soci::transaction tr(txnMetaSession); + + for (auto const& acceptedLedgerTx : *aLedger) + { + std::string_view constexpr txnSQL = + R"sql(INSERT OR REPLACE INTO TransactionMeta VALUES + (:transactionID,:shardIndex);)sql"; + + auto const transactionID = + to_string(acceptedLedgerTx->getTransactionID()); + + txnMetaSession << txnSQL, soci::use(transactionID), + soci::use(shardIndex); + } + + tr.commit(); + return true; +} + +std::optional +getShardIndexforLedger(soci::session& session, LedgerHash const& hash) +{ + std::uint32_t shardIndex; + session << "SELECT ShardIndex FROM LedgerMeta WHERE LedgerHash = '" << hash + << "';", + soci::into(shardIndex); + + if (!session.got_data()) + return std::nullopt; + + return shardIndex; +} + +std::optional +getShardIndexforTransaction(soci::session& session, TxID const& id) +{ + std::uint32_t shardIndex; + session << "SELECT ShardIndex FROM TransactionMeta WHERE TransID = '" << id + << "';", + soci::into(shardIndex); + + if (!session.got_data()) + return std::nullopt; + + return shardIndex; +} + +} // namespace detail +} // namespace ripple diff --git a/src/ripple/app/rdb/impl/RelationalDBInterface_postgres.cpp b/src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp similarity index 71% rename from src/ripple/app/rdb/impl/RelationalDBInterface_postgres.cpp rename to src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp index d242c8b4b..5ee4ce551 100644 --- a/src/ripple/app/rdb/impl/RelationalDBInterface_postgres.cpp +++ b/src/ripple/app/rdb/backend/impl/PostgresDatabase.cpp @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ /* This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2021 Ripple Labs Inc. + Copyright (c) 2020 Ripple Labs Inc. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -17,73 +17,137 @@ */ //============================================================================== -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include -#include +#include +#include +#include +#include +#include namespace ripple { -using TxnsData = RelationalDBInterface::AccountTxs; -using TxnsDataBinary = RelationalDBInterface::MetaTxsList; +class PgPool; -using LedgerHash = RelationalDBInterface::LedgerHash; -using LedgerSequence = RelationalDBInterface::LedgerSequence; -using LedgerShortcut = RelationalDBInterface::LedgerShortcut; +using AccountTxResult = RelationalDatabase::AccountTxResult; +using TxnsData = RelationalDatabase::AccountTxs; +using TxnsDataBinary = RelationalDatabase::MetaTxsList; -std::optional -getMinLedgerSeq(std::shared_ptr const& pgPool, beast::Journal j) +class PostgresDatabaseImp final : public PostgresDatabase { +public: + PostgresDatabaseImp( + Application& app, + Config const& config, + JobQueue& jobQueue) + : app_(app) + , j_(app_.journal("PgPool")) + , pgPool_( #ifdef RIPPLED_REPORTING - auto seq = PgQuery(pgPool)("SELECT min_ledger()"); - if (!seq) + make_PgPool(config.section("ledger_tx_tables"), j_) +#endif + ) { - JLOG(j.error()) << "Error querying minimum ledger sequence."; + assert(config.reporting()); +#ifdef RIPPLED_REPORTING + if (config.reporting() && !config.reportingReadOnly()) // use pg + { + initSchema(pgPool_); + } +#endif } - else if (!seq.isNull()) - return seq.asInt(); -#endif - return {}; -} -std::optional -getMaxLedgerSeq(std::shared_ptr const& pgPool) -{ + void + stop() override + { #ifdef RIPPLED_REPORTING - auto seq = PgQuery(pgPool)("SELECT max_ledger()"); - if (seq && !seq.isNull()) - return seq.asBigInt(); + pgPool_->stop(); #endif - return {}; -} + } -std::string -getCompleteLedgers(std::shared_ptr const& pgPool) -{ -#ifdef RIPPLED_REPORTING - auto range = PgQuery(pgPool)("SELECT complete_ledgers()"); - if (range) - return range.c_str(); -#endif - return "error"; -} + void + sweep() override; -std::chrono::seconds -getValidatedLedgerAge(std::shared_ptr const& pgPool, beast::Journal j) -{ - using namespace std::chrono_literals; -#ifdef RIPPLED_REPORTING - auto age = PgQuery(pgPool)("SELECT age()"); - if (!age || age.isNull()) - JLOG(j.debug()) << "No ledgers in database"; - else - return std::chrono::seconds{age.asInt()}; -#endif - return weeks{2}; -} + std::optional + getMinLedgerSeq() override; + + std::optional + getMaxLedgerSeq() override; + + std::string + getCompleteLedgers() override; + + std::chrono::seconds + getValidatedLedgerAge() override; + + bool + writeLedgerAndTransactions( + LedgerInfo const& info, + std::vector const& accountTxData) override; + + std::optional + getLedgerInfoByIndex(LedgerIndex ledgerSeq) override; + + std::optional + getNewestLedgerInfo() override; + + std::optional + getLedgerInfoByHash(uint256 const& ledgerHash) override; + + uint256 + getHashByIndex(LedgerIndex ledgerIndex) override; + + std::optional + getHashesByIndex(LedgerIndex ledgerIndex) override; + + std::map + getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) override; + + std::vector + getTxHashes(LedgerIndex seq) override; + + std::vector> + getTxHistory(LedgerIndex startIndex) override; + + std::pair + getAccountTx(AccountTxArgs const& args) override; + + Transaction::Locator + locateTransaction(uint256 const& id) override; + + bool + ledgerDbHasSpace(Config const& config) override; + + bool + transactionDbHasSpace(Config const& config) override; + + bool + isCaughtUp(std::string& reason) override; + +private: + Application& app_; + beast::Journal j_; + std::shared_ptr pgPool_; + + bool + dbHasSpace(Config const& config); +}; /** - * @brief loadLedgerInfos Load the ledger info for the specified + * @brief loadLedgerInfos Loads the ledger info for the specified * ledger/s from the database * @param pgPool Link to postgres database * @param whichLedger Specifies the ledger to load via ledger sequence, @@ -254,152 +318,30 @@ loadLedgerHelper( return infos[0]; } -std::optional -getNewestLedgerInfo(std::shared_ptr const& pgPool, Application& app) -{ - return loadLedgerHelper(pgPool, {}, app); -} - -std::optional -getLedgerInfoByIndex( - std::shared_ptr const& pgPool, - std::uint32_t ledgerIndex, - Application& app) -{ - return loadLedgerHelper(pgPool, uint32_t{ledgerIndex}, app); -} - -std::optional -getLedgerInfoByHash( - std::shared_ptr const& pgPool, - uint256 const& ledgerHash, - Application& app) -{ - return loadLedgerHelper(pgPool, uint256{ledgerHash}, app); -} - -uint256 -getHashByIndex( - std::shared_ptr const& pgPool, - std::uint32_t ledgerIndex, - Application& app) -{ - auto infos = loadLedgerInfos(pgPool, ledgerIndex, app); - assert(infos.size() <= 1); - if (infos.size()) - return infos[0].hash; - return {}; -} - -bool -getHashesByIndex( - std::shared_ptr const& pgPool, - std::uint32_t ledgerIndex, - uint256& ledgerHash, - uint256& parentHash, - Application& app) -{ - auto infos = loadLedgerInfos(pgPool, ledgerIndex, app); - assert(infos.size() <= 1); - if (infos.size()) - { - ledgerHash = infos[0].hash; - parentHash = infos[0].parentHash; - return true; - } - return false; -} - -std::map -getHashesByIndex( - std::shared_ptr const& pgPool, - std::uint32_t minSeq, - std::uint32_t maxSeq, - Application& app) -{ - std::map ret; - auto infos = loadLedgerInfos(pgPool, std::make_pair(minSeq, maxSeq), app); - for (auto& info : infos) - { - ret[info.seq] = {info.hash, info.parentHash}; - } - return ret; -} - -std::vector -getTxHashes( - std::shared_ptr const& pgPool, - LedgerIndex seq, - Application& app) -{ - std::vector nodestoreHashes; - #ifdef RIPPLED_REPORTING - auto log = app.journal("Ledger"); +static bool +writeToLedgersDB(LedgerInfo const& info, PgQuery& pgQuery, beast::Journal& j) +{ + JLOG(j.debug()) << __func__; + auto cmd = boost::format( + R"(INSERT INTO ledgers + VALUES (%u,'\x%s', '\x%s',%u,%u,%u,%u,%u,'\x%s','\x%s'))"); - std::string query = - "SELECT nodestore_hash" - " FROM transactions " - " WHERE ledger_seq = " + - std::to_string(seq); - auto res = PgQuery(pgPool)(query.c_str()); + auto ledgerInsert = boost::str( + cmd % info.seq % strHex(info.hash) % strHex(info.parentHash) % + info.drops.drops() % info.closeTime.time_since_epoch().count() % + info.parentCloseTime.time_since_epoch().count() % + info.closeTimeResolution.count() % info.closeFlags % + strHex(info.accountHash) % strHex(info.txHash)); + JLOG(j.trace()) << __func__ << " : " + << " : " + << "query string = " << ledgerInsert; - if (!res) - { - JLOG(log.error()) << __func__ - << " : Postgres response is null - query = " << query; - assert(false); - return {}; - } - else if (res.status() != PGRES_TUPLES_OK) - { - JLOG(log.error()) << __func__ - << " : Postgres response should have been " - "PGRES_TUPLES_OK but instead was " - << res.status() << " - msg = " << res.msg() - << " - query = " << query; - assert(false); - return {}; - } + auto res = pgQuery(ledgerInsert.data()); - JLOG(log.trace()) << __func__ << " Postgres result msg : " << res.msg(); - - if (res.isNull() || res.ntuples() == 0) - { - JLOG(log.debug()) << __func__ - << " : Ledger not found. query = " << query; - return {}; - } - else if (res.ntuples() > 0) - { - if (res.nfields() != 1) - { - JLOG(log.error()) << __func__ - << " : Wrong number of fields in Postgres " - "response. Expected 1, but got " - << res.nfields() << " . query = " << query; - assert(false); - return {}; - } - } - - JLOG(log.trace()) << __func__ << " : result = " << res.c_str() - << " : query = " << query; - for (size_t i = 0; i < res.ntuples(); ++i) - { - char const* nodestoreHash = res.c_str(i, 0); - uint256 hash; - if (!hash.parseHex(nodestoreHash + 2)) - assert(false); - - nodestoreHashes.push_back(hash); - } -#endif - - return nodestoreHashes; + return res; } -#ifdef RIPPLED_REPORTING enum class DataFormat { binary, expanded }; static std::variant flatFetchTransactions( @@ -448,7 +390,7 @@ flatFetchTransactions( static std::pair processAccountTxStoredProcedureResult( - AccountTxArgs const& args, + RelationalDatabase::AccountTxArgs const& args, Json::Value& result, Application& app, beast::Journal j) @@ -538,12 +480,368 @@ processAccountTxStoredProcedureResult( } #endif +void +PostgresDatabaseImp::sweep() +{ +#ifdef RIPPLED_REPORTING + pgPool_->idleSweeper(); +#endif +} + +std::optional +PostgresDatabaseImp::getMinLedgerSeq() +{ +#ifdef RIPPLED_REPORTING + auto seq = PgQuery(pgPool_)("SELECT min_ledger()"); + if (!seq) + { + JLOG(j_.error()) << "Error querying minimum ledger sequence."; + } + else if (!seq.isNull()) + return seq.asInt(); +#endif + return {}; +} + +std::optional +PostgresDatabaseImp::getMaxLedgerSeq() +{ +#ifdef RIPPLED_REPORTING + auto seq = PgQuery(pgPool_)("SELECT max_ledger()"); + if (seq && !seq.isNull()) + return seq.asBigInt(); +#endif + return {}; +} + +std::string +PostgresDatabaseImp::getCompleteLedgers() +{ +#ifdef RIPPLED_REPORTING + auto range = PgQuery(pgPool_)("SELECT complete_ledgers()"); + if (range) + return range.c_str(); +#endif + return "error"; +} + +std::chrono::seconds +PostgresDatabaseImp::getValidatedLedgerAge() +{ + using namespace std::chrono_literals; +#ifdef RIPPLED_REPORTING + auto age = PgQuery(pgPool_)("SELECT age()"); + if (!age || age.isNull()) + JLOG(j_.debug()) << "No ledgers in database"; + else + return std::chrono::seconds{age.asInt()}; +#endif + return weeks{2}; +} + +bool +PostgresDatabaseImp::writeLedgerAndTransactions( + LedgerInfo const& info, + std::vector const& accountTxData) +{ +#ifdef RIPPLED_REPORTING + JLOG(j_.debug()) << __func__ << " : " + << "Beginning write to Postgres"; + + try + { + // Create a PgQuery object to run multiple commands over the same + // connection in a single transaction block. + PgQuery pg(pgPool_); + auto res = pg("BEGIN"); + if (!res || res.status() != PGRES_COMMAND_OK) + { + std::stringstream msg; + msg << "bulkWriteToTable : Postgres insert error: " << res.msg(); + Throw(msg.str()); + } + + // Writing to the ledgers db fails if the ledger already exists in the + // db. In this situation, the ETL process has detected there is another + // writer, and falls back to only publishing + if (!writeToLedgersDB(info, pg, j_)) + { + JLOG(j_.warn()) << __func__ << " : " + << "Failed to write to ledgers database."; + return false; + } + + std::stringstream transactionsCopyBuffer; + std::stringstream accountTransactionsCopyBuffer; + for (auto const& data : accountTxData) + { + std::string txHash = strHex(data.txHash); + std::string nodestoreHash = strHex(data.nodestoreHash); + auto idx = data.transactionIndex; + auto ledgerSeq = data.ledgerSequence; + + transactionsCopyBuffer << std::to_string(ledgerSeq) << '\t' + << std::to_string(idx) << '\t' << "\\\\x" + << txHash << '\t' << "\\\\x" << nodestoreHash + << '\n'; + + for (auto const& a : data.accounts) + { + std::string acct = strHex(a); + accountTransactionsCopyBuffer + << "\\\\x" << acct << '\t' << std::to_string(ledgerSeq) + << '\t' << std::to_string(idx) << '\n'; + } + } + + pg.bulkInsert("transactions", transactionsCopyBuffer.str()); + pg.bulkInsert( + "account_transactions", accountTransactionsCopyBuffer.str()); + + res = pg("COMMIT"); + if (!res || res.status() != PGRES_COMMAND_OK) + { + std::stringstream msg; + msg << "bulkWriteToTable : Postgres insert error: " << res.msg(); + assert(false); + Throw(msg.str()); + } + + JLOG(j_.info()) << __func__ << " : " + << "Successfully wrote to Postgres"; + return true; + } + catch (std::exception& e) + { + JLOG(j_.error()) << __func__ + << "Caught exception writing to Postgres : " + << e.what(); + assert(false); + return false; + } +#else + return false; +#endif +} + +std::optional +PostgresDatabaseImp::getLedgerInfoByIndex(LedgerIndex ledgerSeq) +{ + return loadLedgerHelper(pgPool_, ledgerSeq, app_); +} + +std::optional +PostgresDatabaseImp::getNewestLedgerInfo() +{ + return loadLedgerHelper(pgPool_, {}, app_); +} + +std::optional +PostgresDatabaseImp::getLedgerInfoByHash(uint256 const& ledgerHash) +{ + return loadLedgerHelper(pgPool_, ledgerHash, app_); +} + +uint256 +PostgresDatabaseImp::getHashByIndex(LedgerIndex ledgerIndex) +{ + auto infos = loadLedgerInfos(pgPool_, ledgerIndex, app_); + assert(infos.size() <= 1); + if (infos.size()) + return infos[0].hash; + return {}; +} + +std::optional +PostgresDatabaseImp::getHashesByIndex(LedgerIndex ledgerIndex) +{ + LedgerHashPair p; + auto infos = loadLedgerInfos(pgPool_, ledgerIndex, app_); + assert(infos.size() <= 1); + if (infos.size()) + { + p.ledgerHash = infos[0].hash; + p.parentHash = infos[0].parentHash; + return p; + } + return {}; +} + +std::map +PostgresDatabaseImp::getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) +{ + std::map ret; + auto infos = loadLedgerInfos(pgPool_, std::make_pair(minSeq, maxSeq), app_); + for (auto& info : infos) + { + ret[info.seq] = {info.hash, info.parentHash}; + } + return ret; +} + +std::vector +PostgresDatabaseImp::getTxHashes(LedgerIndex seq) +{ + std::vector nodestoreHashes; + +#ifdef RIPPLED_REPORTING + auto log = app_.journal("Ledger"); + + std::string query = + "SELECT nodestore_hash" + " FROM transactions " + " WHERE ledger_seq = " + + std::to_string(seq); + auto res = PgQuery(pgPool_)(query.c_str()); + + if (!res) + { + JLOG(log.error()) << __func__ + << " : Postgres response is null - query = " << query; + assert(false); + return {}; + } + else if (res.status() != PGRES_TUPLES_OK) + { + JLOG(log.error()) << __func__ + << " : Postgres response should have been " + "PGRES_TUPLES_OK but instead was " + << res.status() << " - msg = " << res.msg() + << " - query = " << query; + assert(false); + return {}; + } + + JLOG(log.trace()) << __func__ << " Postgres result msg : " << res.msg(); + + if (res.isNull() || res.ntuples() == 0) + { + JLOG(log.debug()) << __func__ + << " : Ledger not found. query = " << query; + return {}; + } + else if (res.ntuples() > 0) + { + if (res.nfields() != 1) + { + JLOG(log.error()) << __func__ + << " : Wrong number of fields in Postgres " + "response. Expected 1, but got " + << res.nfields() << " . query = " << query; + assert(false); + return {}; + } + } + + JLOG(log.trace()) << __func__ << " : result = " << res.c_str() + << " : query = " << query; + for (size_t i = 0; i < res.ntuples(); ++i) + { + char const* nodestoreHash = res.c_str(i, 0); + uint256 hash; + if (!hash.parseHex(nodestoreHash + 2)) + assert(false); + + nodestoreHashes.push_back(hash); + } +#endif + + return nodestoreHashes; +} + +std::vector> +PostgresDatabaseImp::getTxHistory(LedgerIndex startIndex) +{ + std::vector> ret; + +#ifdef RIPPLED_REPORTING + if (!app_.config().reporting()) + { + assert(false); + Throw( + "called getTxHistory but not in reporting mode"); + } + + std::string sql = boost::str( + boost::format("SELECT nodestore_hash, ledger_seq " + " FROM transactions" + " ORDER BY ledger_seq DESC LIMIT 20 " + "OFFSET %u;") % + startIndex); + + auto res = PgQuery(pgPool_)(sql.data()); + + if (!res) + { + JLOG(j_.error()) << __func__ + << " : Postgres response is null - sql = " << sql; + assert(false); + return {}; + } + else if (res.status() != PGRES_TUPLES_OK) + { + JLOG(j_.error()) << __func__ + << " : Postgres response should have been " + "PGRES_TUPLES_OK but instead was " + << res.status() << " - msg = " << res.msg() + << " - sql = " << sql; + assert(false); + return {}; + } + + JLOG(j_.trace()) << __func__ << " Postgres result msg : " << res.msg(); + + if (res.isNull() || res.ntuples() == 0) + { + JLOG(j_.debug()) << __func__ << " : Empty postgres response"; + assert(false); + return {}; + } + else if (res.ntuples() > 0) + { + if (res.nfields() != 2) + { + JLOG(j_.error()) << __func__ + << " : Wrong number of fields in Postgres " + "response. Expected 1, but got " + << res.nfields() << " . sql = " << sql; + assert(false); + return {}; + } + } + + JLOG(j_.trace()) << __func__ << " : Postgres result = " << res.c_str(); + + std::vector nodestoreHashes; + std::vector ledgerSequences; + for (size_t i = 0; i < res.ntuples(); ++i) + { + uint256 hash; + if (!hash.parseHex(res.c_str(i, 0) + 2)) + assert(false); + nodestoreHashes.push_back(hash); + ledgerSequences.push_back(res.asBigInt(i, 1)); + } + + auto txns = flatFetchTransactions(app_, nodestoreHashes); + for (size_t i = 0; i < txns.size(); ++i) + { + auto const& [sttx, meta] = txns[i]; + assert(sttx); + + std::string reason; + auto txn = std::make_shared(sttx, reason, app_); + txn->setLedger(ledgerSequences[i]); + txn->setStatus(COMMITTED); + ret.push_back(txn); + } + +#endif + return ret; +} + std::pair -getAccountTx( - std::shared_ptr const& pgPool, - AccountTxArgs const& args, - Application& app, - beast::Journal j) +PostgresDatabaseImp::getAccountTx(AccountTxArgs const& args) { #ifdef RIPPLED_REPORTING pg_params dbParams; @@ -587,8 +885,8 @@ getAccountTx( } else { - JLOG(j.error()) << "doAccountTxStoredProcedure - " - << "Error parsing ledger args"; + JLOG(j_.error()) << "doAccountTxStoredProcedure - " + << "Error parsing ledger args"; return {}; } } @@ -600,52 +898,52 @@ getAccountTx( } for (size_t i = 0; i < values.size(); ++i) { - JLOG(j.trace()) << "value " << std::to_string(i) << " = " - << (values[i] ? values[i].value() : "null"); + JLOG(j_.trace()) << "value " << std::to_string(i) << " = " + << (values[i] ? values[i].value() : "null"); } - auto res = PgQuery(pgPool)(dbParams); + auto res = PgQuery(pgPool_)(dbParams); if (!res) { - JLOG(j.error()) << __func__ - << " : Postgres response is null - account = " - << strHex(args.account); + JLOG(j_.error()) << __func__ + << " : Postgres response is null - account = " + << strHex(args.account); assert(false); return {{}, {rpcINTERNAL, "Postgres error"}}; } else if (res.status() != PGRES_TUPLES_OK) { - JLOG(j.error()) << __func__ - << " : Postgres response should have been " - "PGRES_TUPLES_OK but instead was " - << res.status() << " - msg = " << res.msg() - << " - account = " << strHex(args.account); + JLOG(j_.error()) << __func__ + << " : Postgres response should have been " + "PGRES_TUPLES_OK but instead was " + << res.status() << " - msg = " << res.msg() + << " - account = " << strHex(args.account); assert(false); return {{}, {rpcINTERNAL, "Postgres error"}}; } - JLOG(j.trace()) << __func__ << " Postgres result msg : " << res.msg(); + JLOG(j_.trace()) << __func__ << " Postgres result msg : " << res.msg(); if (res.isNull() || res.ntuples() == 0) { - JLOG(j.debug()) << __func__ - << " : No data returned from Postgres : account = " - << strHex(args.account); + JLOG(j_.debug()) << __func__ + << " : No data returned from Postgres : account = " + << strHex(args.account); assert(false); return {{}, {rpcINTERNAL, "Postgres error"}}; } char const* resultStr = res.c_str(); - JLOG(j.trace()) << __func__ << " : " - << "postgres result = " << resultStr - << " : account = " << strHex(args.account); + JLOG(j_.trace()) << __func__ << " : " + << "postgres result = " << resultStr + << " : account = " << strHex(args.account); Json::Value v; Json::Reader reader; bool success = reader.parse(resultStr, resultStr + strlen(resultStr), v); if (success) { - return processAccountTxStoredProcedureResult(args, v, app, j); + return processAccountTxStoredProcedureResult(args, v, app_, j_); } #endif // This shouldn't happen. Postgres should return a parseable error @@ -654,10 +952,7 @@ getAccountTx( } Transaction::Locator -locateTransaction( - std::shared_ptr const& pgPool, - uint256 const& id, - Application& app) +PostgresDatabaseImp::locateTransaction(uint256 const& id) { #ifdef RIPPLED_REPORTING auto baseCmd = boost::format(R"(SELECT tx('%s');)"); @@ -665,11 +960,11 @@ locateTransaction( std::string txHash = "\\x" + strHex(id); std::string sql = boost::str(baseCmd % txHash); - auto res = PgQuery(pgPool)(sql.data()); + auto res = PgQuery(pgPool_)(sql.data()); if (!res) { - JLOG(app.journal("Transaction").error()) + JLOG(app_.journal("Transaction").error()) << __func__ << " : Postgres response is null - tx ID = " << strHex(id); assert(false); @@ -677,7 +972,7 @@ locateTransaction( } else if (res.status() != PGRES_TUPLES_OK) { - JLOG(app.journal("Transaction").error()) + JLOG(app_.journal("Transaction").error()) << __func__ << " : Postgres response should have been " "PGRES_TUPLES_OK but instead was " @@ -687,11 +982,11 @@ locateTransaction( return {}; } - JLOG(app.journal("Transaction").trace()) + JLOG(app_.journal("Transaction").trace()) << __func__ << " Postgres result msg : " << res.msg(); if (res.isNull() || res.ntuples() == 0) { - JLOG(app.journal("Transaction").debug()) + JLOG(app_.journal("Transaction").debug()) << __func__ << " : No data returned from Postgres : tx ID = " << strHex(id); // This shouldn't happen @@ -700,7 +995,7 @@ locateTransaction( } char const* resultStr = res.c_str(); - JLOG(app.journal("Transaction").debug()) + JLOG(app_.journal("Transaction").debug()) << "postgres result = " << resultStr; Json::Value v; @@ -733,210 +1028,50 @@ locateTransaction( return {}; } -#ifdef RIPPLED_REPORTING -static bool -writeToLedgersDB(LedgerInfo const& info, PgQuery& pgQuery, beast::Journal& j) +bool +PostgresDatabaseImp::dbHasSpace(Config const& config) { - JLOG(j.debug()) << __func__; - auto cmd = boost::format( - R"(INSERT INTO ledgers - VALUES (%u,'\x%s', '\x%s',%u,%u,%u,%u,%u,'\x%s','\x%s'))"); + /* Postgres server could be running on a different machine. */ - auto ledgerInsert = boost::str( - cmd % info.seq % strHex(info.hash) % strHex(info.parentHash) % - info.drops.drops() % info.closeTime.time_since_epoch().count() % - info.parentCloseTime.time_since_epoch().count() % - info.closeTimeResolution.count() % info.closeFlags % - strHex(info.accountHash) % strHex(info.txHash)); - JLOG(j.trace()) << __func__ << " : " - << " : " - << "query string = " << ledgerInsert; - - auto res = pgQuery(ledgerInsert.data()); - - return res; + return true; } -#endif bool -writeLedgerAndTransactions( - std::shared_ptr const& pgPool, - LedgerInfo const& info, - std::vector const& accountTxData, - beast::Journal& j) +PostgresDatabaseImp::ledgerDbHasSpace(Config const& config) { -#ifdef RIPPLED_REPORTING - JLOG(j.debug()) << __func__ << " : " - << "Beginning write to Postgres"; - - try - { - // Create a PgQuery object to run multiple commands over the same - // connection in a single transaction block. - PgQuery pg(pgPool); - auto res = pg("BEGIN"); - if (!res || res.status() != PGRES_COMMAND_OK) - { - std::stringstream msg; - msg << "bulkWriteToTable : Postgres insert error: " << res.msg(); - Throw(msg.str()); - } - - // Writing to the ledgers db fails if the ledger already exists in the - // db. In this situation, the ETL process has detected there is another - // writer, and falls back to only publishing - if (!writeToLedgersDB(info, pg, j)) - { - JLOG(j.warn()) << __func__ << " : " - << "Failed to write to ledgers database."; - return false; - } - - std::stringstream transactionsCopyBuffer; - std::stringstream accountTransactionsCopyBuffer; - for (auto const& data : accountTxData) - { - std::string txHash = strHex(data.txHash); - std::string nodestoreHash = strHex(data.nodestoreHash); - auto idx = data.transactionIndex; - auto ledgerSeq = data.ledgerSequence; - - transactionsCopyBuffer << std::to_string(ledgerSeq) << '\t' - << std::to_string(idx) << '\t' << "\\\\x" - << txHash << '\t' << "\\\\x" << nodestoreHash - << '\n'; - - for (auto const& a : data.accounts) - { - std::string acct = strHex(a); - accountTransactionsCopyBuffer - << "\\\\x" << acct << '\t' << std::to_string(ledgerSeq) - << '\t' << std::to_string(idx) << '\n'; - } - } - - pg.bulkInsert("transactions", transactionsCopyBuffer.str()); - pg.bulkInsert( - "account_transactions", accountTransactionsCopyBuffer.str()); - - res = pg("COMMIT"); - if (!res || res.status() != PGRES_COMMAND_OK) - { - std::stringstream msg; - msg << "bulkWriteToTable : Postgres insert error: " << res.msg(); - assert(false); - Throw(msg.str()); - } - - JLOG(j.info()) << __func__ << " : " - << "Successfully wrote to Postgres"; - return true; - } - catch (std::exception& e) - { - JLOG(j.error()) << __func__ << "Caught exception writing to Postgres : " - << e.what(); - assert(false); - return false; - } -#else - return false; -#endif + return dbHasSpace(config); } -std::vector> -getTxHistory( - std::shared_ptr const& pgPool, - LedgerIndex startIndex, - Application& app, - beast::Journal j) +bool +PostgresDatabaseImp::transactionDbHasSpace(Config const& config) { - std::vector> ret; + return dbHasSpace(config); +} +std::unique_ptr +getPostgresDatabase(Application& app, Config const& config, JobQueue& jobQueue) +{ + return std::make_unique(app, config, jobQueue); +} + +bool +PostgresDatabaseImp::isCaughtUp(std::string& reason) +{ #ifdef RIPPLED_REPORTING - if (!app.config().reporting()) + using namespace std::chrono_literals; + auto age = PgQuery(pgPool_)("SELECT age()"); + if (!age || age.isNull()) { - assert(false); - Throw( - "called getTxHistory but not in reporting mode"); + reason = "No ledgers in database"; + return false; } - - std::string sql = boost::str( - boost::format("SELECT nodestore_hash, ledger_seq " - " FROM transactions" - " ORDER BY ledger_seq DESC LIMIT 20 " - "OFFSET %u;") % - startIndex); - - auto res = PgQuery(pgPool)(sql.data()); - - if (!res) + if (std::chrono::seconds{age.asInt()} > 3min) { - JLOG(j.error()) << __func__ - << " : Postgres response is null - sql = " << sql; - assert(false); - return {}; + reason = "No recently-published ledger"; + return false; } - else if (res.status() != PGRES_TUPLES_OK) - { - JLOG(j.error()) << __func__ - << " : Postgres response should have been " - "PGRES_TUPLES_OK but instead was " - << res.status() << " - msg = " << res.msg() - << " - sql = " << sql; - assert(false); - return {}; - } - - JLOG(j.trace()) << __func__ << " Postgres result msg : " << res.msg(); - - if (res.isNull() || res.ntuples() == 0) - { - JLOG(j.debug()) << __func__ << " : Empty postgres response"; - assert(false); - return {}; - } - else if (res.ntuples() > 0) - { - if (res.nfields() != 2) - { - JLOG(j.error()) << __func__ - << " : Wrong number of fields in Postgres " - "response. Expected 1, but got " - << res.nfields() << " . sql = " << sql; - assert(false); - return {}; - } - } - - JLOG(j.trace()) << __func__ << " : Postgres result = " << res.c_str(); - - std::vector nodestoreHashes; - std::vector ledgerSequences; - for (size_t i = 0; i < res.ntuples(); ++i) - { - uint256 hash; - if (!hash.parseHex(res.c_str(i, 0) + 2)) - assert(false); - nodestoreHashes.push_back(hash); - ledgerSequences.push_back(res.asBigInt(i, 1)); - } - - auto txns = flatFetchTransactions(app, nodestoreHashes); - for (size_t i = 0; i < txns.size(); ++i) - { - auto const& [sttx, meta] = txns[i]; - assert(sttx); - - std::string reason; - auto txn = std::make_shared(sttx, reason, app); - txn->setLedger(ledgerSequences[i]); - txn->setStatus(COMMITTED); - ret.push_back(txn); - } - #endif - return ret; + return true; } } // namespace ripple diff --git a/src/ripple/app/rdb/backend/RelationalDBInterfaceSqlite.cpp b/src/ripple/app/rdb/backend/impl/SQLiteDatabase.cpp similarity index 72% rename from src/ripple/app/rdb/backend/RelationalDBInterfaceSqlite.cpp rename to src/ripple/app/rdb/backend/impl/SQLiteDatabase.cpp index cd5277fb9..547ab843b 100644 --- a/src/ripple/app/rdb/backend/RelationalDBInterfaceSqlite.cpp +++ b/src/ripple/app/rdb/backend/impl/SQLiteDatabase.cpp @@ -23,43 +23,41 @@ #include #include #include -#include -#include -#include -#include +#include +#include +#include #include #include #include #include #include #include -#include -#include #include namespace ripple { -class RelationalDBInterfaceSqliteImp : public RelationalDBInterfaceSqlite +class SQLiteDatabaseImp final : public SQLiteDatabase { public: - RelationalDBInterfaceSqliteImp( + SQLiteDatabaseImp( Application& app, Config const& config, JobQueue& jobQueue) : app_(app) , useTxTables_(config.useTxTables()) - , j_(app_.journal("Ledger")) + , j_(app_.journal("SQLiteDatabaseImp")) { - DatabaseCon::Setup setup = setup_DatabaseCon(config, j_); + DatabaseCon::Setup const setup = setup_DatabaseCon(config, j_); if (!makeLedgerDBs( config, setup, DatabaseCon::CheckpointerSetup{&jobQueue, &app_.logs()})) { - JLOG(app_.journal("RelationalDBInterfaceSqlite").fatal()) - << "AccountTransactions database should not have a primary key"; - Throw( - "AccountTransactions database initialization failed."); + std::string_view constexpr error = + "Failed to create ledger databases"; + + JLOG(j_.fatal()) << error; + Throw(error.data()); } if (app.getShardStore() && @@ -68,10 +66,11 @@ public: setup, DatabaseCon::CheckpointerSetup{&jobQueue, &app_.logs()})) { - JLOG(app_.journal("RelationalDBInterfaceSqlite").fatal()) - << "Error during meta DB init"; - Throw( - "Shard meta database initialization failed."); + std::string_view constexpr error = + "Failed to create metadata databases"; + + JLOG(j_.fatal()) << error; + Throw(error.data()); } } @@ -105,7 +104,7 @@ public: std::size_t getAccountTransactionCount() override; - RelationalDBInterface::CountMinMax + RelationalDatabase::CountMinMax getLedgerCountMinMax() override; bool @@ -199,12 +198,12 @@ private: std::unique_ptr lgrMetaDB_, txMetaDB_; /** - * @brief makeLedgerDBs Opens node ledger and transaction databases, - * and saves its descriptors into internal variables. + * @brief makeLedgerDBs Opens ledger and transaction databases for the node + * store, and stores their descriptors in private member variables. * @param config Config object. - * @param setup Path to database and other opening parameters. + * @param setup Path to the databases and other opening parameters. * @param checkpointerSetup Checkpointer parameters. - * @return True if node databases opened succsessfully. + * @return True if node databases opened successfully. */ bool makeLedgerDBs( @@ -213,10 +212,10 @@ private: DatabaseCon::CheckpointerSetup const& checkpointerSetup); /** - * @brief makeMetaDBs Opens shard index lookup databases, and saves - * their descriptors into internal variables. + * @brief makeMetaDBs Opens shard index lookup databases, and stores + * their descriptors in private member variables. * @param config Config object. - * @param setup Path to database and other opening parameters. + * @param setup Path to the databases and other opening parameters. * @param checkpointerSetup Checkpointer parameters. * @return True if node databases opened successfully. */ @@ -227,7 +226,8 @@ private: DatabaseCon::CheckpointerSetup const& checkpointerSetup); /** - * @brief seqToShardIndex Converts ledgers sequence to shard index. + * @brief seqToShardIndex Provides the index of the shard that stores the + * ledger with the given sequence. * @param ledgerSeq Ledger sequence. * @return Shard index. */ @@ -238,7 +238,8 @@ private: } /** - * @brief firstLedgerSeq Returns first ledger sequence for given shard. + * @brief firstLedgerSeq Returns the sequence of the first ledger stored in + * the shard specified by the shard index parameter. * @param shardIndex Shard Index. * @return First ledger sequence. */ @@ -249,7 +250,8 @@ private: } /** - * @brief lastLedgerSeq Returns last ledger sequence for given shard. + * @brief lastLedgerSeq Returns the sequence of the last ledger stored in + * the shard specified by the shard index parameter. * @param shardIndex Shard Index. * @return Last ledger sequence. */ @@ -260,8 +262,8 @@ private: } /** - * @brief existsLedger Checks if node ledger DB exists. - * @return True if node ledger DB exists. + * @brief existsLedger Checks if the node store ledger database exists. + * @return True if the node store ledger database exists. */ bool existsLedger() @@ -270,8 +272,9 @@ private: } /** - * @brief existsTransaction Checks if node transaction DB exists. - * @return True if node transaction DB exists. + * @brief existsTransaction Checks if the node store transaction database + * exists. + * @return True if the node store transaction database exists. */ bool existsTransaction() @@ -290,8 +293,9 @@ private: } /** - * @brief checkoutTransaction Checks out and returns node ledger DB. - * @return Session to node ledger DB. + * @brief checkoutTransaction Checks out and returns node store ledger + * database. + * @return Session to the node store ledger database. */ auto checkoutLedger() @@ -300,8 +304,9 @@ private: } /** - * @brief checkoutTransaction Checks out and returns node transaction DB. - * @return Session to node transaction DB. + * @brief checkoutTransaction Checks out and returns the node store + * transaction database. + * @return Session to the node store transaction database. */ auto checkoutTransaction() @@ -310,9 +315,9 @@ private: } /** - * @brief doLedger Checks out ledger database for shard - * containing given ledger and calls given callback function passing - * shard index and session with the database to it. + * @brief doLedger Checks out the ledger database owned by the shard + * containing the given ledger, and invokes the provided callback + * with a session to that database. * @param ledgerSeq Ledger sequence. * @param callback Callback function to call. * @return Value returned by callback function. @@ -327,9 +332,9 @@ private: } /** - * @brief doTransaction Checks out transaction database for shard - * containing given ledger and calls given callback function passing - * shard index and session with the database to it. + * @brief doTransaction Checks out the transaction database owned by the + * shard containing the given ledger, and invokes the provided + * callback with a session to that database. * @param ledgerSeq Ledger sequence. * @param callback Callback function to call. * @return Value returned by callback function. @@ -344,12 +349,12 @@ private: } /** - * @brief iterateLedgerForward Checks out ledger databases for - * all shards in ascending order starting from given shard index - * until shard with the largest index visited or callback returned - * false. For each visited shard calls given callback function - * passing shard index and session with the database to it. - * @param firstIndex Start shard index to visit or none if all shards + * @brief iterateLedgerForward Checks out ledger databases for all shards in + * ascending order starting from the given shard index, until all + * shards in range have been visited or the callback returns false. + * For each visited shard, we invoke the provided callback with a + * session to the database and the current shard index. + * @param firstIndex First shard index to visit or no value if all shards * should be visited. * @param callback Callback function to call. * @return True if each callback function returned true, false otherwise. @@ -366,12 +371,13 @@ private: } /** - * @brief iterateTransactionForward Checks out transaction databases for - * all shards in ascending order starting from given shard index - * until shard with the largest index visited or callback returned - * false. For each visited shard calls given callback function - * passing shard index and session with the database to it. - * @param firstIndex Start shard index to visit or none if all shards + * @brief iterateTransactionForward Checks out transaction databases for all + * shards in ascending order starting from the given shard index, + * until all shards in range have been visited or the callback + * returns false. For each visited shard, we invoke the provided + * callback with a session to the database and the current shard + * index. + * @param firstIndex First shard index to visit or no value if all shards * should be visited. * @param callback Callback function to call. * @return True if each callback function returned true, false otherwise. @@ -388,12 +394,13 @@ private: } /** - * @brief iterateLedgerBack Checks out ledger databases for - * all shards in descending order starting from given shard index - * until shard with the smallest index visited or callback returned - * false. For each visited shard calls given callback function - * passing shard index and session with the database to it. - * @param firstIndex Start shard index to visit or none if all shards + * @brief iterateLedgerBack Checks out ledger databases for all + * shards in descending order starting from the given shard index, + * until all shards in range have been visited or the callback + * returns false. For each visited shard, we invoke the provided + * callback with a session to the database and the current shard + * index. + * @param firstIndex First shard index to visit or no value if all shards * should be visited. * @param callback Callback function to call. * @return True if each callback function returned true, false otherwise. @@ -410,12 +417,13 @@ private: } /** - * @brief iterateTransactionForward Checks out transaction databases for - * all shards in descending order starting from given shard index - * until shard with the smallest index visited or callback returned - * false. For each visited shard calls given callback function - * passing shard index and session with the database to it. - * @param firstIndex Start shard index to visit or none if all shards + * @brief iterateTransactionBack Checks out transaction databases for all + * shards in descending order starting from the given shard index, + * until all shards in range have been visited or the callback + * returns false. For each visited shard, we invoke the provided + * callback with a session to the database and the current shard + * index. + * @param firstIndex First shard index to visit or no value if all shards * should be visited. * @param callback Callback function to call. * @return True if each callback function returned true, false otherwise. @@ -433,26 +441,26 @@ private: }; bool -RelationalDBInterfaceSqliteImp::makeLedgerDBs( +SQLiteDatabaseImp::makeLedgerDBs( Config const& config, DatabaseCon::Setup const& setup, DatabaseCon::CheckpointerSetup const& checkpointerSetup) { auto [lgr, tx, res] = - ripple::makeLedgerDBs(config, setup, checkpointerSetup); + detail::makeLedgerDBs(config, setup, checkpointerSetup); txdb_ = std::move(tx); lgrdb_ = std::move(lgr); return res; } bool -RelationalDBInterfaceSqliteImp::makeMetaDBs( +SQLiteDatabaseImp::makeMetaDBs( Config const& config, DatabaseCon::Setup const& setup, DatabaseCon::CheckpointerSetup const& checkpointerSetup) { auto [lgrMetaDB, txMetaDB] = - ripple::makeMetaDBs(config, setup, checkpointerSetup); + detail::makeMetaDBs(config, setup, checkpointerSetup); txMetaDB_ = std::move(txMetaDB); lgrMetaDB_ = std::move(lgrMetaDB); @@ -461,13 +469,13 @@ RelationalDBInterfaceSqliteImp::makeMetaDBs( } std::optional -RelationalDBInterfaceSqliteImp::getMinLedgerSeq() +SQLiteDatabaseImp::getMinLedgerSeq() { /* if databases exists, use it */ if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getMinLedgerSeq(*db, TableType::Ledgers); + return detail::getMinLedgerSeq(*db, detail::TableType::Ledgers); } /* else use shard databases, if available */ @@ -476,7 +484,8 @@ RelationalDBInterfaceSqliteImp::getMinLedgerSeq() std::optional res; iterateLedgerForward( {}, [&](soci::session& session, std::uint32_t shardIndex) { - res = ripple::getMinLedgerSeq(session, TableType::Ledgers); + res = detail::getMinLedgerSeq( + session, detail::TableType::Ledgers); return !res; }); return res; @@ -487,7 +496,7 @@ RelationalDBInterfaceSqliteImp::getMinLedgerSeq() } std::optional -RelationalDBInterfaceSqliteImp::getTransactionsMinLedgerSeq() +SQLiteDatabaseImp::getTransactionsMinLedgerSeq() { if (!useTxTables_) return {}; @@ -495,7 +504,7 @@ RelationalDBInterfaceSqliteImp::getTransactionsMinLedgerSeq() if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::getMinLedgerSeq(*db, TableType::Transactions); + return detail::getMinLedgerSeq(*db, detail::TableType::Transactions); } if (shardStoreExists()) @@ -503,7 +512,8 @@ RelationalDBInterfaceSqliteImp::getTransactionsMinLedgerSeq() std::optional res; iterateTransactionForward( {}, [&](soci::session& session, std::uint32_t shardIndex) { - res = ripple::getMinLedgerSeq(session, TableType::Transactions); + res = detail::getMinLedgerSeq( + session, detail::TableType::Transactions); return !res; }); return res; @@ -513,7 +523,7 @@ RelationalDBInterfaceSqliteImp::getTransactionsMinLedgerSeq() } std::optional -RelationalDBInterfaceSqliteImp::getAccountTransactionsMinLedgerSeq() +SQLiteDatabaseImp::getAccountTransactionsMinLedgerSeq() { if (!useTxTables_) return {}; @@ -521,7 +531,8 @@ RelationalDBInterfaceSqliteImp::getAccountTransactionsMinLedgerSeq() if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::getMinLedgerSeq(*db, TableType::AccountTransactions); + return detail::getMinLedgerSeq( + *db, detail::TableType::AccountTransactions); } if (shardStoreExists()) @@ -529,8 +540,8 @@ RelationalDBInterfaceSqliteImp::getAccountTransactionsMinLedgerSeq() std::optional res; iterateTransactionForward( {}, [&](soci::session& session, std::uint32_t shardIndex) { - res = ripple::getMinLedgerSeq( - session, TableType::AccountTransactions); + res = detail::getMinLedgerSeq( + session, detail::TableType::AccountTransactions); return !res; }); return res; @@ -540,12 +551,12 @@ RelationalDBInterfaceSqliteImp::getAccountTransactionsMinLedgerSeq() } std::optional -RelationalDBInterfaceSqliteImp::getMaxLedgerSeq() +SQLiteDatabaseImp::getMaxLedgerSeq() { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getMaxLedgerSeq(*db, TableType::Ledgers); + return detail::getMaxLedgerSeq(*db, detail::TableType::Ledgers); } if (shardStoreExists()) @@ -553,7 +564,8 @@ RelationalDBInterfaceSqliteImp::getMaxLedgerSeq() std::optional res; iterateLedgerBack( {}, [&](soci::session& session, std::uint32_t shardIndex) { - res = ripple::getMaxLedgerSeq(session, TableType::Ledgers); + res = detail::getMaxLedgerSeq( + session, detail::TableType::Ledgers); return !res; }); return res; @@ -563,8 +575,7 @@ RelationalDBInterfaceSqliteImp::getMaxLedgerSeq() } void -RelationalDBInterfaceSqliteImp::deleteTransactionByLedgerSeq( - LedgerIndex ledgerSeq) +SQLiteDatabaseImp::deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) { if (!useTxTables_) return; @@ -572,27 +583,29 @@ RelationalDBInterfaceSqliteImp::deleteTransactionByLedgerSeq( if (existsTransaction()) { auto db = checkoutTransaction(); - ripple::deleteByLedgerSeq(*db, TableType::Transactions, ledgerSeq); + detail::deleteByLedgerSeq( + *db, detail::TableType::Transactions, ledgerSeq); return; } if (shardStoreExists()) { doTransaction(ledgerSeq, [&](soci::session& session) { - ripple::deleteByLedgerSeq( - session, TableType::Transactions, ledgerSeq); + detail::deleteByLedgerSeq( + session, detail::TableType::Transactions, ledgerSeq); return true; }); } } void -RelationalDBInterfaceSqliteImp::deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) +SQLiteDatabaseImp::deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) { if (existsLedger()) { auto db = checkoutLedger(); - ripple::deleteBeforeLedgerSeq(*db, TableType::Ledgers, ledgerSeq); + detail::deleteBeforeLedgerSeq( + *db, detail::TableType::Ledgers, ledgerSeq); return; } @@ -601,16 +614,15 @@ RelationalDBInterfaceSqliteImp::deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) iterateLedgerBack( seqToShardIndex(ledgerSeq), [&](soci::session& session, std::uint32_t shardIndex) { - ripple::deleteBeforeLedgerSeq( - session, TableType::Ledgers, ledgerSeq); + detail::deleteBeforeLedgerSeq( + session, detail::TableType::Ledgers, ledgerSeq); return true; }); } } void -RelationalDBInterfaceSqliteImp::deleteTransactionsBeforeLedgerSeq( - LedgerIndex ledgerSeq) +SQLiteDatabaseImp::deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) { if (!useTxTables_) return; @@ -618,7 +630,8 @@ RelationalDBInterfaceSqliteImp::deleteTransactionsBeforeLedgerSeq( if (existsTransaction()) { auto db = checkoutTransaction(); - ripple::deleteBeforeLedgerSeq(*db, TableType::Transactions, ledgerSeq); + detail::deleteBeforeLedgerSeq( + *db, detail::TableType::Transactions, ledgerSeq); return; } @@ -627,15 +640,15 @@ RelationalDBInterfaceSqliteImp::deleteTransactionsBeforeLedgerSeq( iterateTransactionBack( seqToShardIndex(ledgerSeq), [&](soci::session& session, std::uint32_t shardIndex) { - ripple::deleteBeforeLedgerSeq( - session, TableType::Transactions, ledgerSeq); + detail::deleteBeforeLedgerSeq( + session, detail::TableType::Transactions, ledgerSeq); return true; }); } } void -RelationalDBInterfaceSqliteImp::deleteAccountTransactionsBeforeLedgerSeq( +SQLiteDatabaseImp::deleteAccountTransactionsBeforeLedgerSeq( LedgerIndex ledgerSeq) { if (!useTxTables_) @@ -644,8 +657,8 @@ RelationalDBInterfaceSqliteImp::deleteAccountTransactionsBeforeLedgerSeq( if (existsTransaction()) { auto db = checkoutTransaction(); - ripple::deleteBeforeLedgerSeq( - *db, TableType::AccountTransactions, ledgerSeq); + detail::deleteBeforeLedgerSeq( + *db, detail::TableType::AccountTransactions, ledgerSeq); return; } @@ -654,15 +667,15 @@ RelationalDBInterfaceSqliteImp::deleteAccountTransactionsBeforeLedgerSeq( iterateTransactionBack( seqToShardIndex(ledgerSeq), [&](soci::session& session, std::uint32_t shardIndex) { - ripple::deleteBeforeLedgerSeq( - session, TableType::AccountTransactions, ledgerSeq); + detail::deleteBeforeLedgerSeq( + session, detail::TableType::AccountTransactions, ledgerSeq); return true; }); } } std::size_t -RelationalDBInterfaceSqliteImp::getTransactionCount() +SQLiteDatabaseImp::getTransactionCount() { if (!useTxTables_) return 0; @@ -670,33 +683,7 @@ RelationalDBInterfaceSqliteImp::getTransactionCount() if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::getRows(*db, TableType::Transactions); - } - - if (shardStoreExists()) - { - std::size_t rows = 0; - iterateTransactionForward( - {}, [&](soci::session& session, std::uint32_t shardIndex) { - rows += ripple::getRows(session, TableType::Transactions); - return true; - }); - return rows; - } - - return 0; -} - -std::size_t -RelationalDBInterfaceSqliteImp::getAccountTransactionCount() -{ - if (!useTxTables_) - return 0; - - if (existsTransaction()) - { - auto db = checkoutTransaction(); - return ripple::getRows(*db, TableType::AccountTransactions); + return detail::getRows(*db, detail::TableType::Transactions); } if (shardStoreExists()) @@ -705,7 +692,7 @@ RelationalDBInterfaceSqliteImp::getAccountTransactionCount() iterateTransactionForward( {}, [&](soci::session& session, std::uint32_t shardIndex) { rows += - ripple::getRows(session, TableType::AccountTransactions); + detail::getRows(session, detail::TableType::Transactions); return true; }); return rows; @@ -714,13 +701,40 @@ RelationalDBInterfaceSqliteImp::getAccountTransactionCount() return 0; } -RelationalDBInterface::CountMinMax -RelationalDBInterfaceSqliteImp::getLedgerCountMinMax() +std::size_t +SQLiteDatabaseImp::getAccountTransactionCount() +{ + if (!useTxTables_) + return 0; + + if (existsTransaction()) + { + auto db = checkoutTransaction(); + return detail::getRows(*db, detail::TableType::AccountTransactions); + } + + if (shardStoreExists()) + { + std::size_t rows = 0; + iterateTransactionForward( + {}, [&](soci::session& session, std::uint32_t shardIndex) { + rows += detail::getRows( + session, detail::TableType::AccountTransactions); + return true; + }); + return rows; + } + + return 0; +} + +RelationalDatabase::CountMinMax +SQLiteDatabaseImp::getLedgerCountMinMax() { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getRowsMinMax(*db, TableType::Ledgers); + return detail::getRowsMinMax(*db, detail::TableType::Ledgers); } if (shardStoreExists()) @@ -728,7 +742,8 @@ RelationalDBInterfaceSqliteImp::getLedgerCountMinMax() CountMinMax res{0, 0, 0}; iterateLedgerForward( {}, [&](soci::session& session, std::uint32_t shardIndex) { - auto r = ripple::getRowsMinMax(session, TableType::Ledgers); + auto r = + detail::getRowsMinMax(session, detail::TableType::Ledgers); if (r.numberOfRows) { res.numberOfRows += r.numberOfRows; @@ -745,13 +760,13 @@ RelationalDBInterfaceSqliteImp::getLedgerCountMinMax() } bool -RelationalDBInterfaceSqliteImp::saveValidatedLedger( +SQLiteDatabaseImp::saveValidatedLedger( std::shared_ptr const& ledger, bool current) { if (existsLedger()) { - if (!ripple::saveValidatedLedger( + if (!detail::saveValidatedLedger( *lgrdb_, *txdb_, app_, ledger, current)) return false; } @@ -769,7 +784,7 @@ RelationalDBInterfaceSqliteImp::saveValidatedLedger( auto lgrMetaSession = lgrMetaDB_->checkoutDb(); auto txMetaSession = txMetaDB_->checkoutDb(); - return ripple::saveLedgerMeta( + return detail::saveLedgerMeta( ledger, app_, *lgrMetaSession, @@ -781,19 +796,22 @@ RelationalDBInterfaceSqliteImp::saveValidatedLedger( } std::optional -RelationalDBInterfaceSqliteImp::getLedgerInfoByIndex(LedgerIndex ledgerSeq) +SQLiteDatabaseImp::getLedgerInfoByIndex(LedgerIndex ledgerSeq) { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getLedgerInfoByIndex(*db, ledgerSeq, j_); + auto const res = detail::getLedgerInfoByIndex(*db, ledgerSeq, j_); + + if (res.has_value()) + return res; } if (shardStoreExists()) { std::optional res; doLedger(ledgerSeq, [&](soci::session& session) { - res = ripple::getLedgerInfoByIndex(session, ledgerSeq, j_); + res = detail::getLedgerInfoByIndex(session, ledgerSeq, j_); return true; }); return res; @@ -803,12 +821,15 @@ RelationalDBInterfaceSqliteImp::getLedgerInfoByIndex(LedgerIndex ledgerSeq) } std::optional -RelationalDBInterfaceSqliteImp::getNewestLedgerInfo() +SQLiteDatabaseImp::getNewestLedgerInfo() { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getNewestLedgerInfo(*db, j_); + auto const res = detail::getNewestLedgerInfo(*db, j_); + + if (res.has_value()) + return res; } if (shardStoreExists()) @@ -816,7 +837,7 @@ RelationalDBInterfaceSqliteImp::getNewestLedgerInfo() std::optional res; iterateLedgerBack( {}, [&](soci::session& session, std::uint32_t shardIndex) { - if (auto info = ripple::getNewestLedgerInfo(session, j_)) + if (auto info = detail::getNewestLedgerInfo(session, j_)) { res = info; return false; @@ -831,13 +852,16 @@ RelationalDBInterfaceSqliteImp::getNewestLedgerInfo() } std::optional -RelationalDBInterfaceSqliteImp::getLimitedOldestLedgerInfo( - LedgerIndex ledgerFirstIndex) +SQLiteDatabaseImp::getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getLimitedOldestLedgerInfo(*db, ledgerFirstIndex, j_); + auto const res = + detail::getLimitedOldestLedgerInfo(*db, ledgerFirstIndex, j_); + + if (res.has_value()) + return res; } if (shardStoreExists()) @@ -846,7 +870,7 @@ RelationalDBInterfaceSqliteImp::getLimitedOldestLedgerInfo( iterateLedgerForward( seqToShardIndex(ledgerFirstIndex), [&](soci::session& session, std::uint32_t shardIndex) { - if (auto info = ripple::getLimitedOldestLedgerInfo( + if (auto info = detail::getLimitedOldestLedgerInfo( session, ledgerFirstIndex, j_)) { res = info; @@ -862,13 +886,16 @@ RelationalDBInterfaceSqliteImp::getLimitedOldestLedgerInfo( } std::optional -RelationalDBInterfaceSqliteImp::getLimitedNewestLedgerInfo( - LedgerIndex ledgerFirstIndex) +SQLiteDatabaseImp::getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getLimitedNewestLedgerInfo(*db, ledgerFirstIndex, j_); + auto const res = + detail::getLimitedNewestLedgerInfo(*db, ledgerFirstIndex, j_); + + if (res.has_value()) + return res; } if (shardStoreExists()) @@ -876,7 +903,7 @@ RelationalDBInterfaceSqliteImp::getLimitedNewestLedgerInfo( std::optional res; iterateLedgerBack( {}, [&](soci::session& session, std::uint32_t shardIndex) { - if (auto info = ripple::getLimitedNewestLedgerInfo( + if (auto info = detail::getLimitedNewestLedgerInfo( session, ledgerFirstIndex, j_)) { res = info; @@ -892,12 +919,15 @@ RelationalDBInterfaceSqliteImp::getLimitedNewestLedgerInfo( } std::optional -RelationalDBInterfaceSqliteImp::getLedgerInfoByHash(uint256 const& ledgerHash) +SQLiteDatabaseImp::getLedgerInfoByHash(uint256 const& ledgerHash) { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getLedgerInfoByHash(*db, ledgerHash, j_); + auto const res = detail::getLedgerInfoByHash(*db, ledgerHash, j_); + + if (res.has_value()) + return res; } if (auto shardStore = app_.getShardStore()) @@ -906,11 +936,11 @@ RelationalDBInterfaceSqliteImp::getLedgerInfoByHash(uint256 const& ledgerHash) auto lgrMetaSession = lgrMetaDB_->checkoutDb(); if (auto const shardIndex = - ripple::getShardIndexforLedger(*lgrMetaSession, ledgerHash)) + detail::getShardIndexforLedger(*lgrMetaSession, ledgerHash)) { shardStore->callForLedgerSQLByShardIndex( *shardIndex, [&](soci::session& session) { - res = ripple::getLedgerInfoByHash(session, ledgerHash, j_); + res = detail::getLedgerInfoByHash(session, ledgerHash, j_); return false; // unused }); } @@ -922,19 +952,22 @@ RelationalDBInterfaceSqliteImp::getLedgerInfoByHash(uint256 const& ledgerHash) } uint256 -RelationalDBInterfaceSqliteImp::getHashByIndex(LedgerIndex ledgerIndex) +SQLiteDatabaseImp::getHashByIndex(LedgerIndex ledgerIndex) { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getHashByIndex(*db, ledgerIndex); + auto const res = detail::getHashByIndex(*db, ledgerIndex); + + if (res.isNonZero()) + return res; } if (shardStoreExists()) { uint256 hash; doLedger(ledgerIndex, [&](soci::session& session) { - hash = ripple::getHashByIndex(session, ledgerIndex); + hash = detail::getHashByIndex(session, ledgerIndex); return true; }); return hash; @@ -944,19 +977,22 @@ RelationalDBInterfaceSqliteImp::getHashByIndex(LedgerIndex ledgerIndex) } std::optional -RelationalDBInterfaceSqliteImp::getHashesByIndex(LedgerIndex ledgerIndex) +SQLiteDatabaseImp::getHashesByIndex(LedgerIndex ledgerIndex) { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getHashesByIndex(*db, ledgerIndex, j_); + auto const res = detail::getHashesByIndex(*db, ledgerIndex, j_); + + if (res.has_value()) + return res; } if (shardStoreExists()) { std::optional res; doLedger(ledgerIndex, [&](soci::session& session) { - res = ripple::getHashesByIndex(session, ledgerIndex, j_); + res = detail::getHashesByIndex(session, ledgerIndex, j_); return true; }); return res; @@ -966,14 +1002,15 @@ RelationalDBInterfaceSqliteImp::getHashesByIndex(LedgerIndex ledgerIndex) } std::map -RelationalDBInterfaceSqliteImp::getHashesByIndex( - LedgerIndex minSeq, - LedgerIndex maxSeq) +SQLiteDatabaseImp::getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::getHashesByIndex(*db, minSeq, maxSeq, j_); + auto const res = detail::getHashesByIndex(*db, minSeq, maxSeq, j_); + + if (!res.empty()) + return res; } if (shardStoreExists()) @@ -986,7 +1023,7 @@ RelationalDBInterfaceSqliteImp::getHashesByIndex( shardMaxSeq = maxSeq; doLedger(minSeq, [&](soci::session& session) { auto r = - ripple::getHashesByIndex(session, minSeq, shardMaxSeq, j_); + detail::getHashesByIndex(session, minSeq, shardMaxSeq, j_); res.insert(r.begin(), r.end()); return true; }); @@ -1000,7 +1037,7 @@ RelationalDBInterfaceSqliteImp::getHashesByIndex( } std::vector> -RelationalDBInterfaceSqliteImp::getTxHistory(LedgerIndex startIndex) +SQLiteDatabaseImp::getTxHistory(LedgerIndex startIndex) { if (!useTxTables_) return {}; @@ -1008,7 +1045,11 @@ RelationalDBInterfaceSqliteImp::getTxHistory(LedgerIndex startIndex) if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::getTxHistory(*db, app_, startIndex, 20, false).first; + auto const res = + detail::getTxHistory(*db, app_, startIndex, 20, false).first; + + if (!res.empty()) + return res; } if (shardStoreExists()) @@ -1017,7 +1058,7 @@ RelationalDBInterfaceSqliteImp::getTxHistory(LedgerIndex startIndex) int quantity = 20; iterateTransactionBack( {}, [&](soci::session& session, std::uint32_t shardIndex) { - auto [tx, total] = ripple::getTxHistory( + auto [tx, total] = detail::getTxHistory( session, app_, startIndex, quantity, true); txs.insert(txs.end(), tx.begin(), tx.end()); if (total > 0) @@ -1040,9 +1081,8 @@ RelationalDBInterfaceSqliteImp::getTxHistory(LedgerIndex startIndex) return {}; } -RelationalDBInterface::AccountTxs -RelationalDBInterfaceSqliteImp::getOldestAccountTxs( - AccountTxOptions const& options) +RelationalDatabase::AccountTxs +SQLiteDatabaseImp::getOldestAccountTxs(AccountTxOptions const& options) { if (!useTxTables_) return {}; @@ -1052,7 +1092,7 @@ RelationalDBInterfaceSqliteImp::getOldestAccountTxs( if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::getOldestAccountTxs( + return detail::getOldestAccountTxs( *db, app_, ledgerMaster, options, {}, j_) .first; } @@ -1069,7 +1109,7 @@ RelationalDBInterfaceSqliteImp::getOldestAccountTxs( if (opt.maxLedger && shardIndex > seqToShardIndex(opt.maxLedger)) return false; - auto [r, total] = ripple::getOldestAccountTxs( + auto [r, total] = detail::getOldestAccountTxs( session, app_, ledgerMaster, opt, limit_used, j_); ret.insert(ret.end(), r.begin(), r.end()); if (!total) @@ -1101,9 +1141,8 @@ RelationalDBInterfaceSqliteImp::getOldestAccountTxs( return {}; } -RelationalDBInterface::AccountTxs -RelationalDBInterfaceSqliteImp::getNewestAccountTxs( - AccountTxOptions const& options) +RelationalDatabase::AccountTxs +SQLiteDatabaseImp::getNewestAccountTxs(AccountTxOptions const& options) { if (!useTxTables_) return {}; @@ -1113,7 +1152,7 @@ RelationalDBInterfaceSqliteImp::getNewestAccountTxs( if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::getNewestAccountTxs( + return detail::getNewestAccountTxs( *db, app_, ledgerMaster, options, {}, j_) .first; } @@ -1130,7 +1169,7 @@ RelationalDBInterfaceSqliteImp::getNewestAccountTxs( if (opt.minLedger && shardIndex < seqToShardIndex(opt.minLedger)) return false; - auto [r, total] = ripple::getNewestAccountTxs( + auto [r, total] = detail::getNewestAccountTxs( session, app_, ledgerMaster, opt, limit_used, j_); ret.insert(ret.end(), r.begin(), r.end()); if (!total) @@ -1162,9 +1201,8 @@ RelationalDBInterfaceSqliteImp::getNewestAccountTxs( return {}; } -RelationalDBInterface::MetaTxsList -RelationalDBInterfaceSqliteImp::getOldestAccountTxsB( - AccountTxOptions const& options) +RelationalDatabase::MetaTxsList +SQLiteDatabaseImp::getOldestAccountTxsB(AccountTxOptions const& options) { if (!useTxTables_) return {}; @@ -1172,7 +1210,7 @@ RelationalDBInterfaceSqliteImp::getOldestAccountTxsB( if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::getOldestAccountTxsB(*db, app_, options, {}, j_).first; + return detail::getOldestAccountTxsB(*db, app_, options, {}, j_).first; } if (shardStoreExists()) @@ -1187,7 +1225,7 @@ RelationalDBInterfaceSqliteImp::getOldestAccountTxsB( if (opt.maxLedger && shardIndex > seqToShardIndex(opt.maxLedger)) return false; - auto [r, total] = ripple::getOldestAccountTxsB( + auto [r, total] = detail::getOldestAccountTxsB( session, app_, opt, limit_used, j_); ret.insert(ret.end(), r.begin(), r.end()); if (!total) @@ -1219,9 +1257,8 @@ RelationalDBInterfaceSqliteImp::getOldestAccountTxsB( return {}; } -RelationalDBInterface::MetaTxsList -RelationalDBInterfaceSqliteImp::getNewestAccountTxsB( - AccountTxOptions const& options) +RelationalDatabase::MetaTxsList +SQLiteDatabaseImp::getNewestAccountTxsB(AccountTxOptions const& options) { if (!useTxTables_) return {}; @@ -1229,7 +1266,7 @@ RelationalDBInterfaceSqliteImp::getNewestAccountTxsB( if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::getNewestAccountTxsB(*db, app_, options, {}, j_).first; + return detail::getNewestAccountTxsB(*db, app_, options, {}, j_).first; } if (shardStoreExists()) @@ -1244,7 +1281,7 @@ RelationalDBInterfaceSqliteImp::getNewestAccountTxsB( if (opt.minLedger && shardIndex < seqToShardIndex(opt.minLedger)) return false; - auto [r, total] = ripple::getNewestAccountTxsB( + auto [r, total] = detail::getNewestAccountTxsB( session, app_, opt, limit_used, j_); ret.insert(ret.end(), r.begin(), r.end()); if (!total) @@ -1277,16 +1314,14 @@ RelationalDBInterfaceSqliteImp::getNewestAccountTxsB( } std::pair< - RelationalDBInterface::AccountTxs, - std::optional> -RelationalDBInterfaceSqliteImp::oldestAccountTxPage( - AccountTxPageOptions const& options) + RelationalDatabase::AccountTxs, + std::optional> +SQLiteDatabaseImp::oldestAccountTxPage(AccountTxPageOptions const& options) { if (!useTxTables_) return {}; static std::uint32_t const page_length(200); - auto& idCache = app_.accountIDCache(); auto onUnsavedLedger = std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); AccountTxs ret; @@ -1302,15 +1337,10 @@ RelationalDBInterfaceSqliteImp::oldestAccountTxPage( if (existsTransaction()) { auto db = checkoutTransaction(); - auto newmarker = ripple::oldestAccountTxPage( - *db, - idCache, - onUnsavedLedger, - onTransaction, - options, - 0, - page_length) - .first; + auto newmarker = + detail::oldestAccountTxPage( + *db, onUnsavedLedger, onTransaction, options, 0, page_length) + .first; return {ret, newmarker}; } @@ -1325,9 +1355,8 @@ RelationalDBInterfaceSqliteImp::oldestAccountTxPage( if (opt.maxLedger != UINT32_MAX && shardIndex > seqToShardIndex(opt.minLedger)) return false; - auto [marker, total] = ripple::oldestAccountTxPage( + auto [marker, total] = detail::oldestAccountTxPage( session, - idCache, onUnsavedLedger, onTransaction, opt, @@ -1347,16 +1376,14 @@ RelationalDBInterfaceSqliteImp::oldestAccountTxPage( } std::pair< - RelationalDBInterface::AccountTxs, - std::optional> -RelationalDBInterfaceSqliteImp::newestAccountTxPage( - AccountTxPageOptions const& options) + RelationalDatabase::AccountTxs, + std::optional> +SQLiteDatabaseImp::newestAccountTxPage(AccountTxPageOptions const& options) { if (!useTxTables_) return {}; static std::uint32_t const page_length(200); - auto& idCache = app_.accountIDCache(); auto onUnsavedLedger = std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); AccountTxs ret; @@ -1372,15 +1399,10 @@ RelationalDBInterfaceSqliteImp::newestAccountTxPage( if (existsTransaction()) { auto db = checkoutTransaction(); - auto newmarker = ripple::newestAccountTxPage( - *db, - idCache, - onUnsavedLedger, - onTransaction, - options, - 0, - page_length) - .first; + auto newmarker = + detail::newestAccountTxPage( + *db, onUnsavedLedger, onTransaction, options, 0, page_length) + .first; return {ret, newmarker}; } @@ -1395,9 +1417,8 @@ RelationalDBInterfaceSqliteImp::newestAccountTxPage( if (opt.minLedger && shardIndex < seqToShardIndex(opt.minLedger)) return false; - auto [marker, total] = ripple::newestAccountTxPage( + auto [marker, total] = detail::newestAccountTxPage( session, - idCache, onUnsavedLedger, onTransaction, opt, @@ -1417,16 +1438,14 @@ RelationalDBInterfaceSqliteImp::newestAccountTxPage( } std::pair< - RelationalDBInterface::MetaTxsList, - std::optional> -RelationalDBInterfaceSqliteImp::oldestAccountTxPageB( - AccountTxPageOptions const& options) + RelationalDatabase::MetaTxsList, + std::optional> +SQLiteDatabaseImp::oldestAccountTxPageB(AccountTxPageOptions const& options) { if (!useTxTables_) return {}; static std::uint32_t const page_length(500); - auto& idCache = app_.accountIDCache(); auto onUnsavedLedger = std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); MetaTxsList ret; @@ -1441,15 +1460,10 @@ RelationalDBInterfaceSqliteImp::oldestAccountTxPageB( if (existsTransaction()) { auto db = checkoutTransaction(); - auto newmarker = ripple::oldestAccountTxPage( - *db, - idCache, - onUnsavedLedger, - onTransaction, - options, - 0, - page_length) - .first; + auto newmarker = + detail::oldestAccountTxPage( + *db, onUnsavedLedger, onTransaction, options, 0, page_length) + .first; return {ret, newmarker}; } @@ -1464,9 +1478,8 @@ RelationalDBInterfaceSqliteImp::oldestAccountTxPageB( if (opt.maxLedger != UINT32_MAX && shardIndex > seqToShardIndex(opt.minLedger)) return false; - auto [marker, total] = ripple::oldestAccountTxPage( + auto [marker, total] = detail::oldestAccountTxPage( session, - idCache, onUnsavedLedger, onTransaction, opt, @@ -1486,16 +1499,14 @@ RelationalDBInterfaceSqliteImp::oldestAccountTxPageB( } std::pair< - RelationalDBInterface::MetaTxsList, - std::optional> -RelationalDBInterfaceSqliteImp::newestAccountTxPageB( - AccountTxPageOptions const& options) + RelationalDatabase::MetaTxsList, + std::optional> +SQLiteDatabaseImp::newestAccountTxPageB(AccountTxPageOptions const& options) { if (!useTxTables_) return {}; static std::uint32_t const page_length(500); - auto& idCache = app_.accountIDCache(); auto onUnsavedLedger = std::bind(saveLedgerAsync, std::ref(app_), std::placeholders::_1); MetaTxsList ret; @@ -1510,15 +1521,10 @@ RelationalDBInterfaceSqliteImp::newestAccountTxPageB( if (existsTransaction()) { auto db = checkoutTransaction(); - auto newmarker = ripple::newestAccountTxPage( - *db, - idCache, - onUnsavedLedger, - onTransaction, - options, - 0, - page_length) - .first; + auto newmarker = + detail::newestAccountTxPage( + *db, onUnsavedLedger, onTransaction, options, 0, page_length) + .first; return {ret, newmarker}; } @@ -1533,9 +1539,8 @@ RelationalDBInterfaceSqliteImp::newestAccountTxPageB( if (opt.minLedger && shardIndex < seqToShardIndex(opt.minLedger)) return false; - auto [marker, total] = ripple::newestAccountTxPage( + auto [marker, total] = detail::newestAccountTxPage( session, - idCache, onUnsavedLedger, onTransaction, opt, @@ -1554,8 +1559,8 @@ RelationalDBInterfaceSqliteImp::newestAccountTxPageB( return {}; } -std::variant -RelationalDBInterfaceSqliteImp::getTransaction( +std::variant +SQLiteDatabaseImp::getTransaction( uint256 const& id, std::optional> const& range, error_code_i& ec) @@ -1566,7 +1571,7 @@ RelationalDBInterfaceSqliteImp::getTransaction( if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::getTransaction(*db, app_, id, range, ec); + return detail::getTransaction(*db, app_, id, range, ec); } if (auto shardStore = app_.getShardStore(); shardStore) @@ -1575,7 +1580,7 @@ RelationalDBInterfaceSqliteImp::getTransaction( auto txMetaSession = txMetaDB_->checkoutDb(); if (auto const shardIndex = - ripple::getShardIndexforTransaction(*txMetaSession, id)) + detail::getShardIndexforTransaction(*txMetaSession, id)) { shardStore->callForTransactionSQLByShardIndex( *shardIndex, [&](soci::session& session) { @@ -1589,7 +1594,7 @@ RelationalDBInterfaceSqliteImp::getTransaction( if (low <= high) range1 = ClosedInterval(low, high); } - res = ripple::getTransaction(session, app_, id, range1, ec); + res = detail::getTransaction(session, app_, id, range1, ec); return res.index() == 1 && std::get(res) != @@ -1604,19 +1609,19 @@ RelationalDBInterfaceSqliteImp::getTransaction( } bool -RelationalDBInterfaceSqliteImp::ledgerDbHasSpace(Config const& config) +SQLiteDatabaseImp::ledgerDbHasSpace(Config const& config) { if (existsLedger()) { auto db = checkoutLedger(); - return ripple::dbHasSpace(*db, config, j_); + return detail::dbHasSpace(*db, config, j_); } if (shardStoreExists()) { return iterateLedgerBack( {}, [&](soci::session& session, std::uint32_t shardIndex) { - return ripple::dbHasSpace(session, config, j_); + return detail::dbHasSpace(session, config, j_); }); } @@ -1624,7 +1629,7 @@ RelationalDBInterfaceSqliteImp::ledgerDbHasSpace(Config const& config) } bool -RelationalDBInterfaceSqliteImp::transactionDbHasSpace(Config const& config) +SQLiteDatabaseImp::transactionDbHasSpace(Config const& config) { if (!useTxTables_) return true; @@ -1632,14 +1637,14 @@ RelationalDBInterfaceSqliteImp::transactionDbHasSpace(Config const& config) if (existsTransaction()) { auto db = checkoutTransaction(); - return ripple::dbHasSpace(*db, config, j_); + return detail::dbHasSpace(*db, config, j_); } if (shardStoreExists()) { return iterateTransactionBack( {}, [&](soci::session& session, std::uint32_t shardIndex) { - return ripple::dbHasSpace(session, config, j_); + return detail::dbHasSpace(session, config, j_); }); } @@ -1647,7 +1652,7 @@ RelationalDBInterfaceSqliteImp::transactionDbHasSpace(Config const& config) } std::uint32_t -RelationalDBInterfaceSqliteImp::getKBUsedAll() +SQLiteDatabaseImp::getKBUsedAll() { if (existsLedger()) { @@ -1669,7 +1674,7 @@ RelationalDBInterfaceSqliteImp::getKBUsedAll() } std::uint32_t -RelationalDBInterfaceSqliteImp::getKBUsedLedger() +SQLiteDatabaseImp::getKBUsedLedger() { if (existsLedger()) { @@ -1691,7 +1696,7 @@ RelationalDBInterfaceSqliteImp::getKBUsedLedger() } std::uint32_t -RelationalDBInterfaceSqliteImp::getKBUsedTransaction() +SQLiteDatabaseImp::getKBUsedTransaction() { if (!useTxTables_) return 0; @@ -1716,25 +1721,21 @@ RelationalDBInterfaceSqliteImp::getKBUsedTransaction() } void -RelationalDBInterfaceSqliteImp::closeLedgerDB() +SQLiteDatabaseImp::closeLedgerDB() { lgrdb_.reset(); } void -RelationalDBInterfaceSqliteImp::closeTransactionDB() +SQLiteDatabaseImp::closeTransactionDB() { txdb_.reset(); } -std::unique_ptr -getRelationalDBInterfaceSqlite( - Application& app, - Config const& config, - JobQueue& jobQueue) +std::unique_ptr +getSQLiteDatabase(Application& app, Config const& config, JobQueue& jobQueue) { - return std::make_unique( - app, config, jobQueue); + return std::make_unique(app, config, jobQueue); } } // namespace ripple diff --git a/src/ripple/app/rdb/impl/Download.cpp b/src/ripple/app/rdb/impl/Download.cpp new file mode 100644 index 000000000..0905ee577 --- /dev/null +++ b/src/ripple/app/rdb/impl/Download.cpp @@ -0,0 +1,152 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include + +namespace ripple { + +std::pair, std::optional> +openDatabaseBodyDb( + DatabaseCon::Setup const& setup, + boost::filesystem::path const& path) +{ + // SOCI requires boost::optional (not std::optional) as the parameter. + boost::optional pathFromDb; + boost::optional size; + + auto conn = std::make_unique( + setup, "Download", DownloaderDBPragma, DatabaseBodyDBInit); + + auto& session = *conn->checkoutDb(); + + session << "SELECT Path FROM Download WHERE Part=0;", + soci::into(pathFromDb); + + // Try to reuse preexisting + // database. + if (pathFromDb) + { + // Can't resuse - database was + // from a different file download. + if (pathFromDb != path.string()) + { + session << "DROP TABLE Download;"; + } + + // Continuing a file download. + else + { + session << "SELECT SUM(LENGTH(Data)) FROM Download;", + soci::into(size); + } + } + + return {std::move(conn), (size ? *size : std::optional())}; +} + +std::uint64_t +databaseBodyDoPut( + soci::session& session, + std::string const& data, + std::string const& path, + std::uint64_t fileSize, + std::uint64_t part, + std::uint16_t maxRowSizePad) +{ + std::uint64_t rowSize = 0; + soci::indicator rti; + + std::uint64_t remainingInRow = 0; + + auto be = + dynamic_cast(session.get_backend()); + BOOST_ASSERT(be); + + // This limits how large we can make the blob + // in each row. Also subtract a pad value to + // account for the other values in the row. + auto const blobMaxSize = + sqlite_api::sqlite3_limit(be->conn_, SQLITE_LIMIT_LENGTH, -1) - + maxRowSizePad; + + std::string newpath; + + auto rowInit = [&] { + session << "INSERT INTO Download VALUES (:path, zeroblob(0), 0, :part)", + soci::use(newpath), soci::use(part); + + remainingInRow = blobMaxSize; + rowSize = 0; + }; + + session << "SELECT Path,Size,Part FROM Download ORDER BY Part DESC " + "LIMIT 1", + soci::into(newpath), soci::into(rowSize), soci::into(part, rti); + + if (!session.got_data()) + { + newpath = path; + rowInit(); + } + else + remainingInRow = blobMaxSize - rowSize; + + auto insert = [&session, &rowSize, &part, &fs = fileSize]( + auto const& data) { + std::uint64_t updatedSize = rowSize + data.size(); + + session << "UPDATE Download SET Data = CAST(Data || :data AS blob), " + "Size = :size WHERE Part = :part;", + soci::use(data), soci::use(updatedSize), soci::use(part); + + fs += data.size(); + }; + + size_t currentBase = 0; + + while (currentBase + remainingInRow < data.size()) + { + if (remainingInRow) + { + insert(data.substr(currentBase, remainingInRow)); + currentBase += remainingInRow; + } + + ++part; + rowInit(); + } + + insert(data.substr(currentBase)); + + return part; +} + +void +databaseBodyFinish(soci::session& session, std::ofstream& fout) +{ + soci::rowset rs = + (session.prepare << "SELECT Data FROM Download ORDER BY PART ASC;"); + + // iteration through the resultset: + for (auto it = rs.begin(); it != rs.end(); ++it) + fout.write(it->data(), it->size()); +} + +} // namespace ripple diff --git a/src/ripple/app/rdb/impl/PeerFinder.cpp b/src/ripple/app/rdb/impl/PeerFinder.cpp new file mode 100644 index 000000000..46dca3760 --- /dev/null +++ b/src/ripple/app/rdb/impl/PeerFinder.cpp @@ -0,0 +1,271 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +namespace ripple { + +void +initPeerFinderDB( + soci::session& session, + BasicConfig const& config, + beast::Journal j) +{ + DBConfig m_sociConfig(config, "peerfinder"); + m_sociConfig.open(session); + + JLOG(j.info()) << "Opening database at '" << m_sociConfig.connectionString() + << "'"; + + soci::transaction tr(session); + session << "PRAGMA encoding=\"UTF-8\";"; + + session << "CREATE TABLE IF NOT EXISTS SchemaVersion ( " + " name TEXT PRIMARY KEY, " + " version INTEGER" + ");"; + + session << "CREATE TABLE IF NOT EXISTS PeerFinder_BootstrapCache ( " + " id INTEGER PRIMARY KEY AUTOINCREMENT, " + " address TEXT UNIQUE NOT NULL, " + " valence INTEGER" + ");"; + + session << "CREATE INDEX IF NOT EXISTS " + " PeerFinder_BootstrapCache_Index ON " + "PeerFinder_BootstrapCache " + " ( " + " address " + " ); "; + + tr.commit(); +} + +void +updatePeerFinderDB( + soci::session& session, + int currentSchemaVersion, + beast::Journal j) +{ + soci::transaction tr(session); + // get version + int version(0); + { + // SOCI requires a boost::optional (not std::optional) parameter. + boost::optional vO; + session << "SELECT " + " version " + "FROM SchemaVersion WHERE " + " name = 'PeerFinder';", + soci::into(vO); + + version = vO.value_or(0); + + JLOG(j.info()) << "Opened version " << version << " database"; + } + + { + if (version < currentSchemaVersion) + { + JLOG(j.info()) << "Updating database to version " + << currentSchemaVersion; + } + else if (version > currentSchemaVersion) + { + Throw( + "The PeerFinder database version is higher than expected"); + } + } + + if (version < 4) + { + // + // Remove the "uptime" column from the bootstrap table + // + + session << "CREATE TABLE IF NOT EXISTS " + "PeerFinder_BootstrapCache_Next ( " + " id INTEGER PRIMARY KEY AUTOINCREMENT, " + " address TEXT UNIQUE NOT NULL, " + " valence INTEGER" + ");"; + + session << "CREATE INDEX IF NOT EXISTS " + " PeerFinder_BootstrapCache_Next_Index ON " + " PeerFinder_BootstrapCache_Next " + " ( address ); "; + + std::size_t count; + session << "SELECT COUNT(*) FROM PeerFinder_BootstrapCache;", + soci::into(count); + + std::vector list; + + { + list.reserve(count); + std::string s; + int valence; + soci::statement st = + (session.prepare << "SELECT " + " address, " + " valence " + "FROM PeerFinder_BootstrapCache;", + soci::into(s), + soci::into(valence)); + + st.execute(); + while (st.fetch()) + { + PeerFinder::Store::Entry entry; + entry.endpoint = beast::IP::Endpoint::from_string(s); + if (!is_unspecified(entry.endpoint)) + { + entry.valence = valence; + list.push_back(entry); + } + else + { + JLOG(j.error()) << "Bad address string '" << s + << "' in Bootcache table"; + } + } + } + + if (!list.empty()) + { + std::vector s; + std::vector valence; + s.reserve(list.size()); + valence.reserve(list.size()); + + for (auto iter(list.cbegin()); iter != list.cend(); ++iter) + { + s.emplace_back(to_string(iter->endpoint)); + valence.emplace_back(iter->valence); + } + + session << "INSERT INTO PeerFinder_BootstrapCache_Next ( " + " address, " + " valence " + ") VALUES ( " + " :s, :valence" + ");", + soci::use(s), soci::use(valence); + } + + session << "DROP TABLE IF EXISTS PeerFinder_BootstrapCache;"; + + session << "DROP INDEX IF EXISTS PeerFinder_BootstrapCache_Index;"; + + session << "ALTER TABLE PeerFinder_BootstrapCache_Next " + " RENAME TO PeerFinder_BootstrapCache;"; + + session << "CREATE INDEX IF NOT EXISTS " + " PeerFinder_BootstrapCache_Index ON " + "PeerFinder_BootstrapCache " + " ( " + " address " + " ); "; + } + + if (version < 3) + { + // + // Remove legacy endpoints from the schema + // + + session << "DROP TABLE IF EXISTS LegacyEndpoints;"; + + session << "DROP TABLE IF EXISTS PeerFinderLegacyEndpoints;"; + + session << "DROP TABLE IF EXISTS PeerFinder_LegacyEndpoints;"; + + session << "DROP TABLE IF EXISTS PeerFinder_LegacyEndpoints_Index;"; + } + + { + int const v(currentSchemaVersion); + session << "INSERT OR REPLACE INTO SchemaVersion (" + " name " + " ,version " + ") VALUES ( " + " 'PeerFinder', :version " + ");", + soci::use(v); + } + + tr.commit(); +} + +void +readPeerFinderDB( + soci::session& session, + std::function const& func) +{ + std::string s; + int valence; + soci::statement st = + (session.prepare << "SELECT " + " address, " + " valence " + "FROM PeerFinder_BootstrapCache;", + soci::into(s), + soci::into(valence)); + + st.execute(); + while (st.fetch()) + { + func(s, valence); + } +} + +void +savePeerFinderDB( + soci::session& session, + std::vector const& v) +{ + soci::transaction tr(session); + session << "DELETE FROM PeerFinder_BootstrapCache;"; + + if (!v.empty()) + { + std::vector s; + std::vector valence; + s.reserve(v.size()); + valence.reserve(v.size()); + + for (auto const& e : v) + { + s.emplace_back(to_string(e.endpoint)); + valence.emplace_back(e.valence); + } + + session << "INSERT INTO PeerFinder_BootstrapCache ( " + " address, " + " valence " + ") VALUES ( " + " :s, :valence " + ");", + soci::use(s), soci::use(valence); + } + + tr.commit(); +} + +} // namespace ripple diff --git a/src/ripple/app/rdb/impl/RelationalDBInterface_global.cpp b/src/ripple/app/rdb/impl/RelationalDBInterface_global.cpp deleted file mode 100644 index 17b86f0ca..000000000 --- a/src/ripple/app/rdb/impl/RelationalDBInterface_global.cpp +++ /dev/null @@ -1,836 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -/* Wallet DB */ - -std::unique_ptr -makeWalletDB(DatabaseCon::Setup const& setup) -{ - // wallet database - return std::make_unique( - setup, WalletDBName, std::array(), WalletDBInit); -} - -std::unique_ptr -makeTestWalletDB(DatabaseCon::Setup const& setup, std::string const& dbname) -{ - // wallet database - return std::make_unique( - setup, dbname.data(), std::array(), WalletDBInit); -} - -void -getManifests( - soci::session& session, - std::string const& dbTable, - ManifestCache& mCache, - beast::Journal j) -{ - // Load manifests stored in database - std::string const sql = "SELECT RawData FROM " + dbTable + ";"; - soci::blob sociRawData(session); - soci::statement st = (session.prepare << sql, soci::into(sociRawData)); - st.execute(); - while (st.fetch()) - { - std::string serialized; - convert(sociRawData, serialized); - if (auto mo = deserializeManifest(serialized)) - { - if (!mo->verify()) - { - JLOG(j.warn()) << "Unverifiable manifest in db"; - continue; - } - - mCache.applyManifest(std::move(*mo)); - } - else - { - JLOG(j.warn()) << "Malformed manifest in database"; - } - } -} - -static void -saveManifest( - soci::session& session, - std::string const& dbTable, - std::string const& serialized) -{ - // soci does not support bulk insertion of blob data - // Do not reuse blob because manifest ecdsa signatures vary in length - // but blob write length is expected to be >= the last write - soci::blob rawData(session); - convert(serialized, rawData); - session << "INSERT INTO " << dbTable << " (RawData) VALUES (:rawData);", - soci::use(rawData); -} - -void -saveManifests( - soci::session& session, - std::string const& dbTable, - std::function const& isTrusted, - hash_map const& map, - beast::Journal j) -{ - soci::transaction tr(session); - session << "DELETE FROM " << dbTable; - for (auto const& v : map) - { - // Save all revocation manifests, - // but only save trusted non-revocation manifests. - if (!v.second.revoked() && !isTrusted(v.second.masterKey)) - { - JLOG(j.info()) << "Untrusted manifest in cache not saved to db"; - continue; - } - - saveManifest(session, dbTable, v.second.serialized); - } - tr.commit(); -} - -void -addValidatorManifest(soci::session& session, std::string const& serialized) -{ - soci::transaction tr(session); - saveManifest(session, "ValidatorManifests", serialized); - tr.commit(); -} - -std::pair -getNodeIdentity(soci::session& session) -{ - { - // SOCI requires boost::optional (not std::optional) as the parameter. - boost::optional pubKO, priKO; - soci::statement st = - (session.prepare - << "SELECT PublicKey, PrivateKey FROM NodeIdentity;", - soci::into(pubKO), - soci::into(priKO)); - st.execute(); - while (st.fetch()) - { - auto const sk = parseBase58( - TokenType::NodePrivate, priKO.value_or("")); - auto const pk = parseBase58( - TokenType::NodePublic, pubKO.value_or("")); - - // Only use if the public and secret keys are a pair - if (sk && pk && (*pk == derivePublicKey(KeyType::secp256k1, *sk))) - return {*pk, *sk}; - } - } - - // If a valid identity wasn't found, we randomly generate a new one: - auto [newpublicKey, newsecretKey] = randomKeyPair(KeyType::secp256k1); - - session << str( - boost::format("INSERT INTO NodeIdentity (PublicKey,PrivateKey) " - "VALUES ('%s','%s');") % - toBase58(TokenType::NodePublic, newpublicKey) % - toBase58(TokenType::NodePrivate, newsecretKey)); - - return {newpublicKey, newsecretKey}; -} - -std::unordered_set, KeyEqual> -getPeerReservationTable(soci::session& session, beast::Journal j) -{ - std::unordered_set, KeyEqual> table; - // These values must be boost::optionals (not std) because SOCI expects - // boost::optionals. - boost::optional valPubKey, valDesc; - // We should really abstract the table and column names into constants, - // but no one else does. Because it is too tedious? It would be easy if we - // had a jOOQ for C++. - soci::statement st = - (session.prepare - << "SELECT PublicKey, Description FROM PeerReservations;", - soci::into(valPubKey), - soci::into(valDesc)); - st.execute(); - while (st.fetch()) - { - if (!valPubKey || !valDesc) - { - // This represents a `NULL` in a `NOT NULL` column. It should be - // unreachable. - continue; - } - auto const optNodeId = - parseBase58(TokenType::NodePublic, *valPubKey); - if (!optNodeId) - { - JLOG(j.warn()) << "load: not a public key: " << valPubKey; - continue; - } - table.insert(PeerReservation{*optNodeId, *valDesc}); - } - - return table; -} - -void -insertPeerReservation( - soci::session& session, - PublicKey const& nodeId, - std::string const& description) -{ - session << "INSERT INTO PeerReservations (PublicKey, Description) " - "VALUES (:nodeId, :desc) " - "ON CONFLICT (PublicKey) DO UPDATE SET " - "Description=excluded.Description", - soci::use(toBase58(TokenType::NodePublic, nodeId)), - soci::use(description); -} - -void -deletePeerReservation(soci::session& session, PublicKey const& nodeId) -{ - session << "DELETE FROM PeerReservations WHERE PublicKey = :nodeId", - soci::use(toBase58(TokenType::NodePublic, nodeId)); -} - -bool -createFeatureVotes(soci::session& session) -{ - soci::transaction tr(session); - std::string sql = - "SELECT count(*) FROM sqlite_master " - "WHERE type='table' AND name='FeatureVotes'"; - // SOCI requires boost::optional (not std::optional) as the parameter. - boost::optional featureVotesCount; - session << sql, soci::into(featureVotesCount); - bool exists = static_cast(*featureVotesCount); - - // Create FeatureVotes table in WalletDB if it doesn't exist - if (!exists) - { - session << "CREATE TABLE FeatureVotes ( " - "AmendmentHash CHARACTER(64) NOT NULL, " - "AmendmentName TEXT, " - "Veto INTEGER NOT NULL );"; - tr.commit(); - } - return exists; -} - -void -readAmendments( - soci::session& session, - std::function amendment_hash, - boost::optional amendment_name, - boost::optional vote)> const& callback) -{ - // lambda that converts the internally stored int to an AmendmentVote. - auto intToVote = [](boost::optional const& dbVote) - -> boost::optional { - return safe_cast(dbVote.value_or(1)); - }; - - soci::transaction tr(session); - std::string sql = - "SELECT AmendmentHash, AmendmentName, Veto FROM FeatureVotes"; - // SOCI requires boost::optional (not std::optional) as parameters. - boost::optional amendment_hash; - boost::optional amendment_name; - boost::optional vote_to_veto; - soci::statement st = - (session.prepare << sql, - soci::into(amendment_hash), - soci::into(amendment_name), - soci::into(vote_to_veto)); - st.execute(); - while (st.fetch()) - { - callback(amendment_hash, amendment_name, intToVote(vote_to_veto)); - } -} - -void -voteAmendment( - soci::session& session, - uint256 const& amendment, - std::string const& name, - AmendmentVote vote) -{ - soci::transaction tr(session); - std::string sql = - "INSERT INTO FeatureVotes (AmendmentHash, AmendmentName, Veto) VALUES " - "('"; - sql += to_string(amendment); - sql += "', '" + name; - sql += "', '" + std::to_string(safe_cast(vote)) + "');"; - session << sql; - tr.commit(); -} - -/* State DB */ - -void -initStateDB( - soci::session& session, - BasicConfig const& config, - std::string const& dbName) -{ - open(session, config, dbName); - - session << "PRAGMA synchronous=FULL;"; - - session << "CREATE TABLE IF NOT EXISTS DbState (" - " Key INTEGER PRIMARY KEY," - " WritableDb TEXT," - " ArchiveDb TEXT," - " LastRotatedLedger INTEGER" - ");"; - - session << "CREATE TABLE IF NOT EXISTS CanDelete (" - " Key INTEGER PRIMARY KEY," - " CanDeleteSeq INTEGER" - ");"; - - std::int64_t count = 0; - { - // SOCI requires boost::optional (not std::optional) as the parameter. - boost::optional countO; - session << "SELECT COUNT(Key) FROM DbState WHERE Key = 1;", - soci::into(countO); - if (!countO) - Throw( - "Failed to fetch Key Count from DbState."); - count = *countO; - } - - if (!count) - { - session << "INSERT INTO DbState VALUES (1, '', '', 0);"; - } - - { - // SOCI requires boost::optional (not std::optional) as the parameter. - boost::optional countO; - session << "SELECT COUNT(Key) FROM CanDelete WHERE Key = 1;", - soci::into(countO); - if (!countO) - Throw( - "Failed to fetch Key Count from CanDelete."); - count = *countO; - } - - if (!count) - { - session << "INSERT INTO CanDelete VALUES (1, 0);"; - } -} - -LedgerIndex -getCanDelete(soci::session& session) -{ - LedgerIndex seq; - session << "SELECT CanDeleteSeq FROM CanDelete WHERE Key = 1;", - soci::into(seq); - ; - return seq; -} - -LedgerIndex -setCanDelete(soci::session& session, LedgerIndex canDelete) -{ - session << "UPDATE CanDelete SET CanDeleteSeq = :canDelete WHERE Key = 1;", - soci::use(canDelete); - return canDelete; -} - -SavedState -getSavedState(soci::session& session) -{ - SavedState state; - session << "SELECT WritableDb, ArchiveDb, LastRotatedLedger" - " FROM DbState WHERE Key = 1;", - soci::into(state.writableDb), soci::into(state.archiveDb), - soci::into(state.lastRotated); - - return state; -} - -void -setSavedState(soci::session& session, SavedState const& state) -{ - session << "UPDATE DbState" - " SET WritableDb = :writableDb," - " ArchiveDb = :archiveDb," - " LastRotatedLedger = :lastRotated" - " WHERE Key = 1;", - soci::use(state.writableDb), soci::use(state.archiveDb), - soci::use(state.lastRotated); -} - -void -setLastRotated(soci::session& session, LedgerIndex seq) -{ - session << "UPDATE DbState SET LastRotatedLedger = :seq" - " WHERE Key = 1;", - soci::use(seq); -} - -/* DatabaseBody DB */ - -std::pair, std::optional> -openDatabaseBodyDb( - DatabaseCon::Setup const& setup, - boost::filesystem::path const& path) -{ - // SOCI requires boost::optional (not std::optional) as the parameter. - boost::optional pathFromDb; - boost::optional size; - - auto conn = std::make_unique( - setup, "Download", DownloaderDBPragma, DatabaseBodyDBInit); - - auto& session = *conn->checkoutDb(); - - session << "SELECT Path FROM Download WHERE Part=0;", - soci::into(pathFromDb); - - // Try to reuse preexisting - // database. - if (pathFromDb) - { - // Can't resuse - database was - // from a different file download. - if (pathFromDb != path.string()) - { - session << "DROP TABLE Download;"; - } - - // Continuing a file download. - else - { - session << "SELECT SUM(LENGTH(Data)) FROM Download;", - soci::into(size); - } - } - - return {std::move(conn), (size ? *size : std::optional())}; -} - -std::uint64_t -databaseBodyDoPut( - soci::session& session, - std::string const& data, - std::string const& path, - std::uint64_t fileSize, - std::uint64_t part, - std::uint16_t maxRowSizePad) -{ - std::uint64_t rowSize = 0; - soci::indicator rti; - - std::uint64_t remainingInRow = 0; - - auto be = - dynamic_cast(session.get_backend()); - BOOST_ASSERT(be); - - // This limits how large we can make the blob - // in each row. Also subtract a pad value to - // account for the other values in the row. - auto const blobMaxSize = - sqlite_api::sqlite3_limit(be->conn_, SQLITE_LIMIT_LENGTH, -1) - - maxRowSizePad; - - std::string newpath; - - auto rowInit = [&] { - session << "INSERT INTO Download VALUES (:path, zeroblob(0), 0, :part)", - soci::use(newpath), soci::use(part); - - remainingInRow = blobMaxSize; - rowSize = 0; - }; - - session << "SELECT Path,Size,Part FROM Download ORDER BY Part DESC " - "LIMIT 1", - soci::into(newpath), soci::into(rowSize), soci::into(part, rti); - - if (!session.got_data()) - { - newpath = path; - rowInit(); - } - else - remainingInRow = blobMaxSize - rowSize; - - auto insert = [&session, &rowSize, &part, &fs = fileSize]( - auto const& data) { - std::uint64_t updatedSize = rowSize + data.size(); - - session << "UPDATE Download SET Data = CAST(Data || :data AS blob), " - "Size = :size WHERE Part = :part;", - soci::use(data), soci::use(updatedSize), soci::use(part); - - fs += data.size(); - }; - - size_t currentBase = 0; - - while (currentBase + remainingInRow < data.size()) - { - if (remainingInRow) - { - insert(data.substr(currentBase, remainingInRow)); - currentBase += remainingInRow; - } - - ++part; - rowInit(); - } - - insert(data.substr(currentBase)); - - return part; -} - -void -databaseBodyFinish(soci::session& session, std::ofstream& fout) -{ - soci::rowset rs = - (session.prepare << "SELECT Data FROM Download ORDER BY PART ASC;"); - - // iteration through the resultset: - for (auto it = rs.begin(); it != rs.end(); ++it) - fout.write(it->data(), it->size()); -} - -/* Vacuum DB */ - -bool -doVacuumDB(DatabaseCon::Setup const& setup) -{ - boost::filesystem::path dbPath = setup.dataDir / TxDBName; - - uintmax_t const dbSize = file_size(dbPath); - assert(dbSize != static_cast(-1)); - - if (auto available = space(dbPath.parent_path()).available; - available < dbSize) - { - std::cerr << "The database filesystem must have at least as " - "much free space as the size of " - << dbPath.string() << ", which is " << dbSize - << " bytes. Only " << available << " bytes are available.\n"; - return false; - } - - auto txnDB = - std::make_unique(setup, TxDBName, TxDBPragma, TxDBInit); - auto& session = txnDB->getSession(); - std::uint32_t pageSize; - - // Only the most trivial databases will fit in memory on typical - // (recommended) software. Force temp files to be written to disk - // regardless of the config settings. - session << boost::format(CommonDBPragmaTemp) % "file"; - session << "PRAGMA page_size;", soci::into(pageSize); - - std::cout << "VACUUM beginning. page_size: " << pageSize << std::endl; - - session << "VACUUM;"; - assert(setup.globalPragma); - for (auto const& p : *setup.globalPragma) - session << p; - session << "PRAGMA page_size;", soci::into(pageSize); - - std::cout << "VACUUM finished. page_size: " << pageSize << std::endl; - - return true; -} - -/* PeerFinder DB */ - -void -initPeerFinderDB( - soci::session& session, - BasicConfig const& config, - beast::Journal j) -{ - DBConfig m_sociConfig(config, "peerfinder"); - m_sociConfig.open(session); - - JLOG(j.info()) << "Opening database at '" << m_sociConfig.connectionString() - << "'"; - - soci::transaction tr(session); - session << "PRAGMA encoding=\"UTF-8\";"; - - session << "CREATE TABLE IF NOT EXISTS SchemaVersion ( " - " name TEXT PRIMARY KEY, " - " version INTEGER" - ");"; - - session << "CREATE TABLE IF NOT EXISTS PeerFinder_BootstrapCache ( " - " id INTEGER PRIMARY KEY AUTOINCREMENT, " - " address TEXT UNIQUE NOT NULL, " - " valence INTEGER" - ");"; - - session << "CREATE INDEX IF NOT EXISTS " - " PeerFinder_BootstrapCache_Index ON " - "PeerFinder_BootstrapCache " - " ( " - " address " - " ); "; - - tr.commit(); -} - -void -updatePeerFinderDB( - soci::session& session, - int currentSchemaVersion, - beast::Journal j) -{ - soci::transaction tr(session); - // get version - int version(0); - { - // SOCI requires a boost::optional (not std::optional) parameter. - boost::optional vO; - session << "SELECT " - " version " - "FROM SchemaVersion WHERE " - " name = 'PeerFinder';", - soci::into(vO); - - version = vO.value_or(0); - - JLOG(j.info()) << "Opened version " << version << " database"; - } - - { - if (version < currentSchemaVersion) - { - JLOG(j.info()) << "Updating database to version " - << currentSchemaVersion; - } - else if (version > currentSchemaVersion) - { - Throw( - "The PeerFinder database version is higher than expected"); - } - } - - if (version < 4) - { - // - // Remove the "uptime" column from the bootstrap table - // - - session << "CREATE TABLE IF NOT EXISTS " - "PeerFinder_BootstrapCache_Next ( " - " id INTEGER PRIMARY KEY AUTOINCREMENT, " - " address TEXT UNIQUE NOT NULL, " - " valence INTEGER" - ");"; - - session << "CREATE INDEX IF NOT EXISTS " - " PeerFinder_BootstrapCache_Next_Index ON " - " PeerFinder_BootstrapCache_Next " - " ( address ); "; - - std::size_t count; - session << "SELECT COUNT(*) FROM PeerFinder_BootstrapCache;", - soci::into(count); - - std::vector list; - - { - list.reserve(count); - std::string s; - int valence; - soci::statement st = - (session.prepare << "SELECT " - " address, " - " valence " - "FROM PeerFinder_BootstrapCache;", - soci::into(s), - soci::into(valence)); - - st.execute(); - while (st.fetch()) - { - PeerFinder::Store::Entry entry; - entry.endpoint = beast::IP::Endpoint::from_string(s); - if (!is_unspecified(entry.endpoint)) - { - entry.valence = valence; - list.push_back(entry); - } - else - { - JLOG(j.error()) << "Bad address string '" << s - << "' in Bootcache table"; - } - } - } - - if (!list.empty()) - { - std::vector s; - std::vector valence; - s.reserve(list.size()); - valence.reserve(list.size()); - - for (auto iter(list.cbegin()); iter != list.cend(); ++iter) - { - s.emplace_back(to_string(iter->endpoint)); - valence.emplace_back(iter->valence); - } - - session << "INSERT INTO PeerFinder_BootstrapCache_Next ( " - " address, " - " valence " - ") VALUES ( " - " :s, :valence" - ");", - soci::use(s), soci::use(valence); - } - - session << "DROP TABLE IF EXISTS PeerFinder_BootstrapCache;"; - - session << "DROP INDEX IF EXISTS PeerFinder_BootstrapCache_Index;"; - - session << "ALTER TABLE PeerFinder_BootstrapCache_Next " - " RENAME TO PeerFinder_BootstrapCache;"; - - session << "CREATE INDEX IF NOT EXISTS " - " PeerFinder_BootstrapCache_Index ON " - "PeerFinder_BootstrapCache " - " ( " - " address " - " ); "; - } - - if (version < 3) - { - // - // Remove legacy endpoints from the schema - // - - session << "DROP TABLE IF EXISTS LegacyEndpoints;"; - - session << "DROP TABLE IF EXISTS PeerFinderLegacyEndpoints;"; - - session << "DROP TABLE IF EXISTS PeerFinder_LegacyEndpoints;"; - - session << "DROP TABLE IF EXISTS PeerFinder_LegacyEndpoints_Index;"; - } - - { - int const v(currentSchemaVersion); - session << "INSERT OR REPLACE INTO SchemaVersion (" - " name " - " ,version " - ") VALUES ( " - " 'PeerFinder', :version " - ");", - soci::use(v); - } - - tr.commit(); -} - -void -readPeerFinderDB( - soci::session& session, - std::function const& func) -{ - std::string s; - int valence; - soci::statement st = - (session.prepare << "SELECT " - " address, " - " valence " - "FROM PeerFinder_BootstrapCache;", - soci::into(s), - soci::into(valence)); - - st.execute(); - while (st.fetch()) - { - func(s, valence); - } -} - -void -savePeerFinderDB( - soci::session& session, - std::vector const& v) -{ - soci::transaction tr(session); - session << "DELETE FROM PeerFinder_BootstrapCache;"; - - if (!v.empty()) - { - std::vector s; - std::vector valence; - s.reserve(v.size()); - valence.reserve(v.size()); - - for (auto const& e : v) - { - s.emplace_back(to_string(e.endpoint)); - valence.emplace_back(e.valence); - } - - session << "INSERT INTO PeerFinder_BootstrapCache ( " - " address, " - " valence " - ") VALUES ( " - " :s, :valence " - ");", - soci::use(s), soci::use(valence); - } - - tr.commit(); -} - -} // namespace ripple diff --git a/src/ripple/app/rdb/impl/RelationalDBInterface.cpp b/src/ripple/app/rdb/impl/RelationalDatabase.cpp similarity index 76% rename from src/ripple/app/rdb/impl/RelationalDBInterface.cpp rename to src/ripple/app/rdb/impl/RelationalDatabase.cpp index 1ef456bcb..8a3ce5b01 100644 --- a/src/ripple/app/rdb/impl/RelationalDBInterface.cpp +++ b/src/ripple/app/rdb/impl/RelationalDatabase.cpp @@ -18,26 +18,20 @@ //============================================================================== #include -#include +#include #include #include namespace ripple { -extern std::unique_ptr -getRelationalDBInterfaceSqlite( - Application& app, - Config const& config, - JobQueue& jobQueue); +extern std::unique_ptr +getSQLiteDatabase(Application& app, Config const& config, JobQueue& jobQueue); -extern std::unique_ptr -getRelationalDBInterfacePostgres( - Application& app, - Config const& config, - JobQueue& jobQueue); +extern std::unique_ptr +getPostgresDatabase(Application& app, Config const& config, JobQueue& jobQueue); -std::unique_ptr -RelationalDBInterface::init( +std::unique_ptr +RelationalDatabase::init( Application& app, Config const& config, JobQueue& jobQueue) @@ -73,14 +67,14 @@ RelationalDBInterface::init( if (use_sqlite) { - return getRelationalDBInterfaceSqlite(app, config, jobQueue); + return getSQLiteDatabase(app, config, jobQueue); } else if (use_postgres) { - return getRelationalDBInterfacePostgres(app, config, jobQueue); + return getPostgresDatabase(app, config, jobQueue); } - return std::unique_ptr(); + return std::unique_ptr(); } } // namespace ripple diff --git a/src/ripple/app/rdb/impl/ShardArchive.cpp b/src/ripple/app/rdb/impl/ShardArchive.cpp new file mode 100644 index 000000000..6880aa001 --- /dev/null +++ b/src/ripple/app/rdb/impl/ShardArchive.cpp @@ -0,0 +1,68 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +namespace ripple { + +std::unique_ptr +makeArchiveDB(boost::filesystem::path const& dir, std::string const& dbName) +{ + return std::make_unique( + dir, dbName, DownloaderDBPragma, ShardArchiveHandlerDBInit); +} + +void +readArchiveDB( + DatabaseCon& db, + std::function const& func) +{ + soci::rowset rs = + (db.getSession().prepare << "SELECT * FROM State;"); + + for (auto it = rs.begin(); it != rs.end(); ++it) + { + func(it->get(1), it->get(0)); + } +} + +void +insertArchiveDB( + DatabaseCon& db, + std::uint32_t shardIndex, + std::string const& url) +{ + db.getSession() << "INSERT INTO State VALUES (:index, :url);", + soci::use(shardIndex), soci::use(url); +} + +void +deleteFromArchiveDB(DatabaseCon& db, std::uint32_t shardIndex) +{ + db.getSession() << "DELETE FROM State WHERE ShardIndex = :index;", + soci::use(shardIndex); +} + +void +dropArchiveDB(DatabaseCon& db) +{ + db.getSession() << "DROP TABLE State;"; +} + +} // namespace ripple diff --git a/src/ripple/app/rdb/impl/State.cpp b/src/ripple/app/rdb/impl/State.cpp new file mode 100644 index 000000000..8f8beb0c7 --- /dev/null +++ b/src/ripple/app/rdb/impl/State.cpp @@ -0,0 +1,130 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +namespace ripple { + +void +initStateDB( + soci::session& session, + BasicConfig const& config, + std::string const& dbName) +{ + open(session, config, dbName); + + session << "PRAGMA synchronous=FULL;"; + + session << "CREATE TABLE IF NOT EXISTS DbState (" + " Key INTEGER PRIMARY KEY," + " WritableDb TEXT," + " ArchiveDb TEXT," + " LastRotatedLedger INTEGER" + ");"; + + session << "CREATE TABLE IF NOT EXISTS CanDelete (" + " Key INTEGER PRIMARY KEY," + " CanDeleteSeq INTEGER" + ");"; + + std::int64_t count = 0; + { + // SOCI requires boost::optional (not std::optional) as the parameter. + boost::optional countO; + session << "SELECT COUNT(Key) FROM DbState WHERE Key = 1;", + soci::into(countO); + if (!countO) + Throw( + "Failed to fetch Key Count from DbState."); + count = *countO; + } + + if (!count) + { + session << "INSERT INTO DbState VALUES (1, '', '', 0);"; + } + + { + // SOCI requires boost::optional (not std::optional) as the parameter. + boost::optional countO; + session << "SELECT COUNT(Key) FROM CanDelete WHERE Key = 1;", + soci::into(countO); + if (!countO) + Throw( + "Failed to fetch Key Count from CanDelete."); + count = *countO; + } + + if (!count) + { + session << "INSERT INTO CanDelete VALUES (1, 0);"; + } +} + +LedgerIndex +getCanDelete(soci::session& session) +{ + LedgerIndex seq; + session << "SELECT CanDeleteSeq FROM CanDelete WHERE Key = 1;", + soci::into(seq); + ; + return seq; +} + +LedgerIndex +setCanDelete(soci::session& session, LedgerIndex canDelete) +{ + session << "UPDATE CanDelete SET CanDeleteSeq = :canDelete WHERE Key = 1;", + soci::use(canDelete); + return canDelete; +} + +SavedState +getSavedState(soci::session& session) +{ + SavedState state; + session << "SELECT WritableDb, ArchiveDb, LastRotatedLedger" + " FROM DbState WHERE Key = 1;", + soci::into(state.writableDb), soci::into(state.archiveDb), + soci::into(state.lastRotated); + + return state; +} + +void +setSavedState(soci::session& session, SavedState const& state) +{ + session << "UPDATE DbState" + " SET WritableDb = :writableDb," + " ArchiveDb = :archiveDb," + " LastRotatedLedger = :lastRotated" + " WHERE Key = 1;", + soci::use(state.writableDb), soci::use(state.archiveDb), + soci::use(state.lastRotated); +} + +void +setLastRotated(soci::session& session, LedgerIndex seq) +{ + session << "UPDATE DbState SET LastRotatedLedger = :seq" + " WHERE Key = 1;", + soci::use(seq); +} + +} // namespace ripple diff --git a/src/ripple/app/rdb/impl/RelationalDBInterface_shards.cpp b/src/ripple/app/rdb/impl/UnitaryShard.cpp similarity index 68% rename from src/ripple/app/rdb/impl/RelationalDBInterface_shards.cpp rename to src/ripple/app/rdb/impl/UnitaryShard.cpp index 32dcfc251..72441d0b7 100644 --- a/src/ripple/app/rdb/impl/RelationalDBInterface_shards.cpp +++ b/src/ripple/app/rdb/impl/UnitaryShard.cpp @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ /* This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. + Copyright (c) 2021 Ripple Labs Inc. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -10,151 +10,20 @@ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== -#include -#include -#include -#include -#include -#include +#include #include -#include -#include -#include -#include +#include #include namespace ripple { -DatabasePair -makeMetaDBs( - Config const& config, - DatabaseCon::Setup const& setup, - DatabaseCon::CheckpointerSetup const& checkpointerSetup) -{ - // ledger meta database - auto lgrMetaDB{std::make_unique( - setup, - LgrMetaDBName, - LgrMetaDBPragma, - LgrMetaDBInit, - checkpointerSetup)}; - - if (config.useTxTables()) - { - // transaction meta database - auto txMetaDB{std::make_unique( - setup, - TxMetaDBName, - TxMetaDBPragma, - TxMetaDBInit, - checkpointerSetup)}; - - return {std::move(lgrMetaDB), std::move(txMetaDB)}; - } - - return {std::move(lgrMetaDB), nullptr}; -} - -bool -saveLedgerMeta( - std::shared_ptr const& ledger, - Application& app, - soci::session& lgrMetaSession, - soci::session& txnMetaSession, - std::uint32_t const shardIndex) -{ - std::string_view constexpr lgrSQL = - R"sql(INSERT OR REPLACE INTO LedgerMeta VALUES - (:ledgerHash,:shardIndex);)sql"; - - auto const hash = to_string(ledger->info().hash); - lgrMetaSession << lgrSQL, soci::use(hash), soci::use(shardIndex); - - if (app.config().useTxTables()) - { - auto const aLedger = [&app, - ledger]() -> std::shared_ptr { - try - { - auto aLedger = - app.getAcceptedLedgerCache().fetch(ledger->info().hash); - if (!aLedger) - { - aLedger = std::make_shared(ledger, app); - app.getAcceptedLedgerCache().canonicalize_replace_client( - ledger->info().hash, aLedger); - } - - return aLedger; - } - catch (std::exception const&) - { - JLOG(app.journal("Ledger").warn()) - << "An accepted ledger was missing nodes"; - } - - return {}; - }(); - - if (!aLedger) - return false; - - soci::transaction tr(txnMetaSession); - - for (auto const& acceptedLedgerTx : *aLedger) - { - std::string_view constexpr txnSQL = - R"sql(INSERT OR REPLACE INTO TransactionMeta VALUES - (:transactionID,:shardIndex);)sql"; - - auto const transactionID = - to_string(acceptedLedgerTx->getTransactionID()); - - txnMetaSession << txnSQL, soci::use(transactionID), - soci::use(shardIndex); - } - - tr.commit(); - } - - return true; -} - -std::optional -getShardIndexforLedger(soci::session& session, LedgerHash const& hash) -{ - std::uint32_t shardIndex; - session << "SELECT ShardIndex FROM LedgerMeta WHERE LedgerHash = '" << hash - << "';", - soci::into(shardIndex); - - if (!session.got_data()) - return std::nullopt; - - return shardIndex; -} - -std::optional -getShardIndexforTransaction(soci::session& session, TxID const& id) -{ - std::uint32_t shardIndex; - session << "SELECT ShardIndex FROM TransactionMeta WHERE TransID = '" << id - << "';", - soci::into(shardIndex); - - if (!session.got_data()) - return std::nullopt; - - return shardIndex; -} - DatabasePair makeShardCompleteLedgerDBs( Config const& config, @@ -333,8 +202,6 @@ updateLedgerDBs( return true; } -/* Shard acquire db */ - std::unique_ptr makeAcquireDB( DatabaseCon::Setup const& setup, @@ -446,50 +313,4 @@ updateAcquireDB( } } -/* Archive DB */ - -std::unique_ptr -makeArchiveDB(boost::filesystem::path const& dir, std::string const& dbName) -{ - return std::make_unique( - dir, dbName, DownloaderDBPragma, ShardArchiveHandlerDBInit); -} - -void -readArchiveDB( - DatabaseCon& db, - std::function const& func) -{ - soci::rowset rs = - (db.getSession().prepare << "SELECT * FROM State;"); - - for (auto it = rs.begin(); it != rs.end(); ++it) - { - func(it->get(1), it->get(0)); - } -} - -void -insertArchiveDB( - DatabaseCon& db, - std::uint32_t shardIndex, - std::string const& url) -{ - db.getSession() << "INSERT INTO State VALUES (:index, :url);", - soci::use(shardIndex), soci::use(url); -} - -void -deleteFromArchiveDB(DatabaseCon& db, std::uint32_t shardIndex) -{ - db.getSession() << "DELETE FROM State WHERE ShardIndex = :index;", - soci::use(shardIndex); -} - -void -dropArchiveDB(DatabaseCon& db) -{ - db.getSession() << "DROP TABLE State;"; -} - } // namespace ripple diff --git a/src/ripple/app/rdb/impl/Vacuum.cpp b/src/ripple/app/rdb/impl/Vacuum.cpp new file mode 100644 index 000000000..aad456cc5 --- /dev/null +++ b/src/ripple/app/rdb/impl/Vacuum.cpp @@ -0,0 +1,67 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include + +namespace ripple { + +bool +doVacuumDB(DatabaseCon::Setup const& setup) +{ + boost::filesystem::path dbPath = setup.dataDir / TxDBName; + + uintmax_t const dbSize = file_size(dbPath); + assert(dbSize != static_cast(-1)); + + if (auto available = space(dbPath.parent_path()).available; + available < dbSize) + { + std::cerr << "The database filesystem must have at least as " + "much free space as the size of " + << dbPath.string() << ", which is " << dbSize + << " bytes. Only " << available << " bytes are available.\n"; + return false; + } + + auto txnDB = + std::make_unique(setup, TxDBName, TxDBPragma, TxDBInit); + auto& session = txnDB->getSession(); + std::uint32_t pageSize; + + // Only the most trivial databases will fit in memory on typical + // (recommended) hardware. Force temp files to be written to disk + // regardless of the config settings. + session << boost::format(CommonDBPragmaTemp) % "file"; + session << "PRAGMA page_size;", soci::into(pageSize); + + std::cout << "VACUUM beginning. page_size: " << pageSize << std::endl; + + session << "VACUUM;"; + assert(setup.globalPragma); + for (auto const& p : *setup.globalPragma) + session << p; + session << "PRAGMA page_size;", soci::into(pageSize); + + std::cout << "VACUUM finished. page_size: " << pageSize << std::endl; + + return true; +} + +} // namespace ripple diff --git a/src/ripple/app/rdb/impl/Wallet.cpp b/src/ripple/app/rdb/impl/Wallet.cpp new file mode 100644 index 000000000..25a06bbd9 --- /dev/null +++ b/src/ripple/app/rdb/impl/Wallet.cpp @@ -0,0 +1,301 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include + +namespace ripple { + +std::unique_ptr +makeWalletDB(DatabaseCon::Setup const& setup) +{ + // wallet database + return std::make_unique( + setup, WalletDBName, std::array(), WalletDBInit); +} + +std::unique_ptr +makeTestWalletDB(DatabaseCon::Setup const& setup, std::string const& dbname) +{ + // wallet database + return std::make_unique( + setup, dbname.data(), std::array(), WalletDBInit); +} + +void +getManifests( + soci::session& session, + std::string const& dbTable, + ManifestCache& mCache, + beast::Journal j) +{ + // Load manifests stored in database + std::string const sql = "SELECT RawData FROM " + dbTable + ";"; + soci::blob sociRawData(session); + soci::statement st = (session.prepare << sql, soci::into(sociRawData)); + st.execute(); + while (st.fetch()) + { + std::string serialized; + convert(sociRawData, serialized); + if (auto mo = deserializeManifest(serialized)) + { + if (!mo->verify()) + { + JLOG(j.warn()) << "Unverifiable manifest in db"; + continue; + } + + mCache.applyManifest(std::move(*mo)); + } + else + { + JLOG(j.warn()) << "Malformed manifest in database"; + } + } +} + +static void +saveManifest( + soci::session& session, + std::string const& dbTable, + std::string const& serialized) +{ + // soci does not support bulk insertion of blob data + // Do not reuse blob because manifest ecdsa signatures vary in length + // but blob write length is expected to be >= the last write + soci::blob rawData(session); + convert(serialized, rawData); + session << "INSERT INTO " << dbTable << " (RawData) VALUES (:rawData);", + soci::use(rawData); +} + +void +saveManifests( + soci::session& session, + std::string const& dbTable, + std::function const& isTrusted, + hash_map const& map, + beast::Journal j) +{ + soci::transaction tr(session); + session << "DELETE FROM " << dbTable; + for (auto const& v : map) + { + // Save all revocation manifests, + // but only save trusted non-revocation manifests. + if (!v.second.revoked() && !isTrusted(v.second.masterKey)) + { + JLOG(j.info()) << "Untrusted manifest in cache not saved to db"; + continue; + } + + saveManifest(session, dbTable, v.second.serialized); + } + tr.commit(); +} + +void +addValidatorManifest(soci::session& session, std::string const& serialized) +{ + soci::transaction tr(session); + saveManifest(session, "ValidatorManifests", serialized); + tr.commit(); +} + +void +clearNodeIdentity(soci::session& session) +{ + session << "DELETE FROM NodeIdentity;"; +} + +std::pair +getNodeIdentity(soci::session& session) +{ + { + // SOCI requires boost::optional (not std::optional) as the parameter. + boost::optional pubKO, priKO; + soci::statement st = + (session.prepare + << "SELECT PublicKey, PrivateKey FROM NodeIdentity;", + soci::into(pubKO), + soci::into(priKO)); + st.execute(); + while (st.fetch()) + { + auto const sk = parseBase58( + TokenType::NodePrivate, priKO.value_or("")); + auto const pk = parseBase58( + TokenType::NodePublic, pubKO.value_or("")); + + // Only use if the public and secret keys are a pair + if (sk && pk && (*pk == derivePublicKey(KeyType::secp256k1, *sk))) + return {*pk, *sk}; + } + } + + // If a valid identity wasn't found, we randomly generate a new one: + auto [newpublicKey, newsecretKey] = randomKeyPair(KeyType::secp256k1); + + session << str( + boost::format("INSERT INTO NodeIdentity (PublicKey,PrivateKey) " + "VALUES ('%s','%s');") % + toBase58(TokenType::NodePublic, newpublicKey) % + toBase58(TokenType::NodePrivate, newsecretKey)); + + return {newpublicKey, newsecretKey}; +} + +std::unordered_set, KeyEqual> +getPeerReservationTable(soci::session& session, beast::Journal j) +{ + std::unordered_set, KeyEqual> table; + // These values must be boost::optionals (not std) because SOCI expects + // boost::optionals. + boost::optional valPubKey, valDesc; + // We should really abstract the table and column names into constants, + // but no one else does. Because it is too tedious? It would be easy if we + // had a jOOQ for C++. + soci::statement st = + (session.prepare + << "SELECT PublicKey, Description FROM PeerReservations;", + soci::into(valPubKey), + soci::into(valDesc)); + st.execute(); + while (st.fetch()) + { + if (!valPubKey || !valDesc) + { + // This represents a `NULL` in a `NOT NULL` column. It should be + // unreachable. + continue; + } + auto const optNodeId = + parseBase58(TokenType::NodePublic, *valPubKey); + if (!optNodeId) + { + JLOG(j.warn()) << "load: not a public key: " << valPubKey; + continue; + } + table.insert(PeerReservation{*optNodeId, *valDesc}); + } + + return table; +} + +void +insertPeerReservation( + soci::session& session, + PublicKey const& nodeId, + std::string const& description) +{ + session << "INSERT INTO PeerReservations (PublicKey, Description) " + "VALUES (:nodeId, :desc) " + "ON CONFLICT (PublicKey) DO UPDATE SET " + "Description=excluded.Description", + soci::use(toBase58(TokenType::NodePublic, nodeId)), + soci::use(description); +} + +void +deletePeerReservation(soci::session& session, PublicKey const& nodeId) +{ + session << "DELETE FROM PeerReservations WHERE PublicKey = :nodeId", + soci::use(toBase58(TokenType::NodePublic, nodeId)); +} + +bool +createFeatureVotes(soci::session& session) +{ + soci::transaction tr(session); + std::string sql = + "SELECT count(*) FROM sqlite_master " + "WHERE type='table' AND name='FeatureVotes'"; + // SOCI requires boost::optional (not std::optional) as the parameter. + boost::optional featureVotesCount; + session << sql, soci::into(featureVotesCount); + bool exists = static_cast(*featureVotesCount); + + // Create FeatureVotes table in WalletDB if it doesn't exist + if (!exists) + { + session << "CREATE TABLE FeatureVotes ( " + "AmendmentHash CHARACTER(64) NOT NULL, " + "AmendmentName TEXT, " + "Veto INTEGER NOT NULL );"; + tr.commit(); + } + return exists; +} + +void +readAmendments( + soci::session& session, + std::function amendment_hash, + boost::optional amendment_name, + boost::optional vote)> const& callback) +{ + // lambda that converts the internally stored int to an AmendmentVote. + auto intToVote = [](boost::optional const& dbVote) + -> boost::optional { + return safe_cast(dbVote.value_or(1)); + }; + + soci::transaction tr(session); + std::string sql = + "SELECT AmendmentHash, AmendmentName, Veto FROM " + "( SELECT AmendmentHash, AmendmentName, Veto, RANK() OVER " + "( PARTITION BY AmendmentHash ORDER BY ROWID DESC ) " + "as rnk FROM FeatureVotes ) WHERE rnk = 1"; + // SOCI requires boost::optional (not std::optional) as parameters. + boost::optional amendment_hash; + boost::optional amendment_name; + boost::optional vote_to_veto; + soci::statement st = + (session.prepare << sql, + soci::into(amendment_hash), + soci::into(amendment_name), + soci::into(vote_to_veto)); + st.execute(); + while (st.fetch()) + { + callback(amendment_hash, amendment_name, intToVote(vote_to_veto)); + } +} + +void +voteAmendment( + soci::session& session, + uint256 const& amendment, + std::string const& name, + AmendmentVote vote) +{ + soci::transaction tr(session); + std::string sql = + "INSERT INTO FeatureVotes (AmendmentHash, AmendmentName, Veto) VALUES " + "('"; + sql += to_string(amendment); + sql += "', '" + name; + sql += "', '" + std::to_string(safe_cast(vote)) + "');"; + session << sql; + tr.commit(); +} + +} // namespace ripple diff --git a/src/ripple/app/reporting/P2pProxy.cpp b/src/ripple/app/reporting/P2pProxy.cpp index 8e4fc3a97..ee04b68e6 100644 --- a/src/ripple/app/reporting/P2pProxy.cpp +++ b/src/ripple/app/reporting/P2pProxy.cpp @@ -72,7 +72,7 @@ shouldForwardToP2p(RPC::JsonContext& context) if (params.isMember(jss::ledger_index)) { auto indexValue = params[jss::ledger_index]; - if (!indexValue.isNumeric()) + if (indexValue.isString()) { auto index = indexValue.asString(); return index == "current" || index == "closed"; diff --git a/src/ripple/app/reporting/P2pProxy.h b/src/ripple/app/reporting/P2pProxy.h index a3984018e..92cc508a1 100644 --- a/src/ripple/app/reporting/P2pProxy.h +++ b/src/ripple/app/reporting/P2pProxy.h @@ -48,8 +48,6 @@ needCurrentOrClosed(Request& request) { // These are the only gRPC requests that specify a ledger if constexpr ( - std::is_same:: - value || std::is_same::value || std::is_same:: value || diff --git a/src/ripple/app/reporting/ReportingETL.cpp b/src/ripple/app/reporting/ReportingETL.cpp index 15eda7b27..7e15d242a 100644 --- a/src/ripple/app/reporting/ReportingETL.cpp +++ b/src/ripple/app/reporting/ReportingETL.cpp @@ -17,7 +17,7 @@ */ //============================================================================== -#include +#include #include #include @@ -167,8 +167,7 @@ ReportingETL::loadInitialLedger(uint32_t startingSequence) if (app_.config().reporting()) { #ifdef RIPPLED_REPORTING - dynamic_cast( - &app_.getRelationalDBInterface()) + dynamic_cast(&app_.getRelationalDatabase()) ->writeLedgerAndTransactions(ledger->info(), accountTxData); #endif } @@ -595,69 +594,69 @@ ReportingETL::runETLPipeline(uint32_t startSequence) loadQueue.push({}); }}; - std::thread loader{ - [this, &lastPublishedSequence, &loadQueue, &writeConflict]() { - beast::setCurrentThreadName("rippled: ReportingETL load"); - size_t totalTransactions = 0; - double totalTime = 0; - while (!writeConflict) - { - std::optional, - std::vector>> - result{loadQueue.pop()}; - // if result is an empty optional, the transformer thread has - // stopped and the loader should stop as well - if (!result) - break; - if (isStopping()) - continue; + std::thread loader{[this, + &lastPublishedSequence, + &loadQueue, + &writeConflict]() { + beast::setCurrentThreadName("rippled: ReportingETL load"); + size_t totalTransactions = 0; + double totalTime = 0; + while (!writeConflict) + { + std::optional, + std::vector>> + result{loadQueue.pop()}; + // if result is an empty optional, the transformer thread has + // stopped and the loader should stop as well + if (!result) + break; + if (isStopping()) + continue; - auto& ledger = result->first; - auto& accountTxData = result->second; + auto& ledger = result->first; + auto& accountTxData = result->second; - auto start = std::chrono::system_clock::now(); - // write to the key-value store - flushLedger(ledger); + auto start = std::chrono::system_clock::now(); + // write to the key-value store + flushLedger(ledger); - auto mid = std::chrono::system_clock::now(); + auto mid = std::chrono::system_clock::now(); // write to RDBMS // if there is a write conflict, some other process has already // written this ledger and has taken over as the ETL writer #ifdef RIPPLED_REPORTING - if (!dynamic_cast( - &app_.getRelationalDBInterface()) - ->writeLedgerAndTransactions( - ledger->info(), accountTxData)) - writeConflict = true; + if (!dynamic_cast(&app_.getRelationalDatabase()) + ->writeLedgerAndTransactions( + ledger->info(), accountTxData)) + writeConflict = true; #endif - auto end = std::chrono::system_clock::now(); + auto end = std::chrono::system_clock::now(); - if (!writeConflict) - { - publishLedger(ledger); - lastPublishedSequence = ledger->info().seq; - } - // print some performance numbers - auto kvTime = ((mid - start).count()) / 1000000000.0; - auto relationalTime = ((end - mid).count()) / 1000000000.0; - - size_t numTxns = accountTxData.size(); - totalTime += kvTime; - totalTransactions += numTxns; - JLOG(journal_.info()) - << "Load phase of etl : " - << "Successfully published ledger! Ledger info: " - << detail::toString(ledger->info()) - << ". txn count = " << numTxns - << ". key-value write time = " << kvTime - << ". relational write time = " << relationalTime - << ". key-value tps = " << numTxns / kvTime - << ". relational tps = " << numTxns / relationalTime - << ". total key-value tps = " - << totalTransactions / totalTime; + if (!writeConflict) + { + publishLedger(ledger); + lastPublishedSequence = ledger->info().seq; } - }}; + // print some performance numbers + auto kvTime = ((mid - start).count()) / 1000000000.0; + auto relationalTime = ((end - mid).count()) / 1000000000.0; + + size_t numTxns = accountTxData.size(); + totalTime += kvTime; + totalTransactions += numTxns; + JLOG(journal_.info()) + << "Load phase of etl : " + << "Successfully published ledger! Ledger info: " + << detail::toString(ledger->info()) + << ". txn count = " << numTxns + << ". key-value write time = " << kvTime + << ". relational write time = " << relationalTime + << ". key-value tps = " << numTxns / kvTime + << ". relational tps = " << numTxns / relationalTime + << ". total key-value tps = " << totalTransactions / totalTime; + } + }}; // wait for all of the threads to stop loader.join(); diff --git a/src/ripple/app/reporting/ReportingETL.h b/src/ripple/app/reporting/ReportingETL.h index 540cc5bfd..71e08adf1 100644 --- a/src/ripple/app/reporting/ReportingETL.h +++ b/src/ripple/app/reporting/ReportingETL.h @@ -21,7 +21,7 @@ #define RIPPLE_APP_REPORTING_REPORTINGETL_H_INCLUDED #include -#include +#include #include #include #include @@ -50,7 +50,7 @@ #include namespace ripple { -using AccountTransactionsData = RelationalDBInterface::AccountTransactionsData; +using AccountTransactionsData = RelationalDatabase::AccountTransactionsData; /** * This class is responsible for continuously extracting data from a diff --git a/src/ripple/app/tx/impl/Change.cpp b/src/ripple/app/tx/impl/Change.cpp index bd66d7d58..93ed1a04f 100644 --- a/src/ripple/app/tx/impl/Change.cpp +++ b/src/ripple/app/tx/impl/Change.cpp @@ -23,9 +23,11 @@ #include #include #include +#include #include #include #include +#include namespace ripple { @@ -120,6 +122,88 @@ Change::preCompute() assert(account_ == beast::zero); } +void +Change::activateTrustLinesToSelfFix() +{ + JLOG(j_.warn()) << "fixTrustLinesToSelf amendment activation code starting"; + + auto removeTrustLineToSelf = [this](Sandbox& sb, uint256 id) { + auto tl = sb.peek(keylet::child(id)); + + if (tl == nullptr) + { + JLOG(j_.warn()) << id << ": Unable to locate trustline"; + return true; + } + + if (tl->getType() != ltRIPPLE_STATE) + { + JLOG(j_.warn()) << id << ": Unexpected type " + << static_cast(tl->getType()); + return true; + } + + auto const& lo = tl->getFieldAmount(sfLowLimit); + auto const& hi = tl->getFieldAmount(sfHighLimit); + + if (lo != hi) + { + JLOG(j_.warn()) << id << ": Trustline doesn't meet requirements"; + return true; + } + + if (auto const page = tl->getFieldU64(sfLowNode); !sb.dirRemove( + keylet::ownerDir(lo.getIssuer()), page, tl->key(), false)) + { + JLOG(j_.error()) << id << ": failed to remove low entry from " + << toBase58(lo.getIssuer()) << ":" << page + << " owner directory"; + return false; + } + + if (auto const page = tl->getFieldU64(sfHighNode); !sb.dirRemove( + keylet::ownerDir(hi.getIssuer()), page, tl->key(), false)) + { + JLOG(j_.error()) << id << ": failed to remove high entry from " + << toBase58(hi.getIssuer()) << ":" << page + << " owner directory"; + return false; + } + + if (tl->getFlags() & lsfLowReserve) + adjustOwnerCount( + sb, sb.peek(keylet::account(lo.getIssuer())), -1, j_); + + if (tl->getFlags() & lsfHighReserve) + adjustOwnerCount( + sb, sb.peek(keylet::account(hi.getIssuer())), -1, j_); + + sb.erase(tl); + + JLOG(j_.warn()) << "Successfully deleted trustline " << id; + + return true; + }; + + using namespace std::literals; + + Sandbox sb(&view()); + + if (removeTrustLineToSelf( + sb, + uint256{ + "2F8F21EFCAFD7ACFB07D5BB04F0D2E18587820C7611305BB674A64EAB0FA71E1"sv}) && + removeTrustLineToSelf( + sb, + uint256{ + "326035D5C0560A9DA8636545DD5A1B0DFCFF63E68D491B5522B767BB00564B1A"sv})) + { + JLOG(j_.warn()) << "fixTrustLinesToSelf amendment activation code " + "executed successfully"; + sb.apply(ctx_.rawView()); + } +} + TER Change::applyAmendment() { @@ -196,6 +280,9 @@ Change::applyAmendment() amendments.push_back(amendment); amendmentObject->setFieldV256(sfAmendments, amendments); + if (amendment == fixTrustLinesToSelf) + activateTrustLinesToSelfFix(); + ctx_.app.getAmendmentTable().enable(amendment); if (!ctx_.app.getAmendmentTable().isSupported(amendment)) diff --git a/src/ripple/app/tx/impl/Change.h b/src/ripple/app/tx/impl/Change.h index acd21837e..0ee7067b3 100644 --- a/src/ripple/app/tx/impl/Change.h +++ b/src/ripple/app/tx/impl/Change.h @@ -56,6 +56,9 @@ public: preclaim(PreclaimContext const& ctx); private: + void + activateTrustLinesToSelfFix(); + TER applyAmendment(); diff --git a/src/ripple/app/tx/impl/CreateOffer.cpp b/src/ripple/app/tx/impl/CreateOffer.cpp index 4ec41f2b3..4f1d9108b 100644 --- a/src/ripple/app/tx/impl/CreateOffer.cpp +++ b/src/ripple/app/tx/impl/CreateOffer.cpp @@ -728,7 +728,7 @@ CreateOffer::flowCross( // additional path with XRP as the intermediate between two books. // This second path we have to build ourselves. STPathSet paths; - if (!takerAmount.in.native() & !takerAmount.out.native()) + if (!takerAmount.in.native() && !takerAmount.out.native()) { STPath path; path.emplace_back(std::nullopt, xrpCurrency(), std::nullopt); @@ -1107,6 +1107,12 @@ CreateOffer::applyGuts(Sandbox& sb, Sandbox& sbCancel) if (bImmediateOrCancel) { JLOG(j_.trace()) << "Immediate or cancel: offer canceled"; + if (!crossed && sb.rules().enabled(featureImmediateOfferKilled)) + // If the ImmediateOfferKilled amendment is enabled, any + // ImmediateOrCancel offer that transfers absolutely no funds + // returns tecKILLED rather than tesSUCCESS. Motivation for the + // change is here: https://github.com/ripple/rippled/issues/4115 + return {tecKILLED, false}; return {tesSUCCESS, true}; } diff --git a/src/ripple/app/tx/impl/InvariantCheck.cpp b/src/ripple/app/tx/impl/InvariantCheck.cpp index 2715778ff..122dd68aa 100644 --- a/src/ripple/app/tx/impl/InvariantCheck.cpp +++ b/src/ripple/app/tx/impl/InvariantCheck.cpp @@ -18,6 +18,8 @@ //============================================================================== #include + +#include #include #include #include @@ -509,23 +511,27 @@ ValidNewAccountRoot::finalize( void ValidNFTokenPage::visitEntry( - bool, + bool isDelete, std::shared_ptr const& before, std::shared_ptr const& after) { static constexpr uint256 const& pageBits = nft::pageMask; static constexpr uint256 const accountBits = ~pageBits; - auto check = [this](std::shared_ptr const& sle) { - auto const account = sle->key() & accountBits; - auto const limit = sle->key() & pageBits; + auto check = [this, isDelete](std::shared_ptr const& sle) { + uint256 const account = sle->key() & accountBits; + uint256 const hiLimit = sle->key() & pageBits; + std::optional const prev = (*sle)[~sfPreviousPageMin]; - if (auto const prev = (*sle)[~sfPreviousPageMin]) + // Make sure that any page links... + // 1. Are properly associated with the owning account and + // 2. The page is correctly ordered between links. + if (prev) { if (account != (*prev & accountBits)) badLink_ = true; - if (limit <= (*prev & pageBits)) + if (hiLimit <= (*prev & pageBits)) badLink_ = true; } @@ -534,17 +540,42 @@ ValidNFTokenPage::visitEntry( if (account != (*next & accountBits)) badLink_ = true; - if (limit >= (*next & pageBits)) + if (hiLimit >= (*next & pageBits)) badLink_ = true; } - for (auto const& obj : sle->getFieldArray(sfNFTokens)) { - if ((obj[sfNFTokenID] & pageBits) >= limit) - badEntry_ = true; + auto const& nftokens = sle->getFieldArray(sfNFTokens); - if (auto uri = obj[~sfURI]; uri && uri->empty()) - badURI_ = true; + // An NFTokenPage should never contain too many tokens or be empty. + if (std::size_t const nftokenCount = nftokens.size(); + (!isDelete && nftokenCount == 0) || + nftokenCount > dirMaxTokensPerPage) + invalidSize_ = true; + + // If prev is valid, use it to establish a lower bound for + // page entries. If prev is not valid the lower bound is zero. + uint256 const loLimit = + prev ? *prev & pageBits : uint256(beast::zero); + + // Also verify that all NFTokenIDs in the page are sorted. + uint256 loCmp = loLimit; + for (auto const& obj : nftokens) + { + uint256 const tokenID = obj[sfNFTokenID]; + if (!nft::compareTokens(loCmp, tokenID)) + badSort_ = true; + loCmp = tokenID; + + // None of the NFTs on this page should belong on lower or + // higher pages. + if (uint256 const tokenPageBits = tokenID & pageBits; + tokenPageBits < loLimit || tokenPageBits >= hiLimit) + badEntry_ = true; + + if (auto uri = obj[~sfURI]; uri && uri->empty()) + badURI_ = true; + } } }; @@ -575,12 +606,24 @@ ValidNFTokenPage::finalize( return false; } + if (badSort_) + { + JLOG(j.fatal()) << "Invariant failed: NFTs on page are not sorted."; + return false; + } + if (badURI_) { JLOG(j.fatal()) << "Invariant failed: NFT contains empty URI."; return false; } + if (invalidSize_) + { + JLOG(j.fatal()) << "Invariant failed: NFT page has invalid size."; + return false; + } + return true; } diff --git a/src/ripple/app/tx/impl/InvariantCheck.h b/src/ripple/app/tx/impl/InvariantCheck.h index 5936b59b6..c3bb02164 100644 --- a/src/ripple/app/tx/impl/InvariantCheck.h +++ b/src/ripple/app/tx/impl/InvariantCheck.h @@ -320,9 +320,11 @@ public: class ValidNFTokenPage { - bool badLink_ = false; bool badEntry_ = false; + bool badLink_ = false; + bool badSort_ = false; bool badURI_ = false; + bool invalidSize_ = false; public: void diff --git a/src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp b/src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp index b7997996e..07fe9957a 100644 --- a/src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp +++ b/src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp @@ -63,36 +63,42 @@ NFTokenAcceptOffer::preflight(PreflightContext const& ctx) TER NFTokenAcceptOffer::preclaim(PreclaimContext const& ctx) { - auto const checkOffer = [&ctx](std::optional id) -> TER { + auto const checkOffer = [&ctx](std::optional id) + -> std::pair, TER> { if (id) { - auto const offer = ctx.view.read(keylet::nftoffer(*id)); + if (id->isZero()) + return {nullptr, tecOBJECT_NOT_FOUND}; - if (!offer) - return tecOBJECT_NOT_FOUND; + auto offerSLE = ctx.view.read(keylet::nftoffer(*id)); - if (hasExpired(ctx.view, (*offer)[~sfExpiration])) - return tecEXPIRED; + if (!offerSLE) + return {nullptr, tecOBJECT_NOT_FOUND}; + + if (hasExpired(ctx.view, (*offerSLE)[~sfExpiration])) + return {nullptr, tecEXPIRED}; + + // The initial implementation had a bug that allowed a negative + // amount. The fixNFTokenNegOffer amendment fixes that. + if ((*offerSLE)[sfAmount].negative() && + ctx.view.rules().enabled(fixNFTokenNegOffer)) + return {nullptr, temBAD_OFFER}; + + return {std::move(offerSLE), tesSUCCESS}; } - - return tesSUCCESS; + return {nullptr, tesSUCCESS}; }; - auto const buy = ctx.tx[~sfNFTokenBuyOffer]; - auto const sell = ctx.tx[~sfNFTokenSellOffer]; + auto const [bo, err1] = checkOffer(ctx.tx[~sfNFTokenBuyOffer]); + if (!isTesSuccess(err1)) + return err1; + auto const [so, err2] = checkOffer(ctx.tx[~sfNFTokenSellOffer]); + if (!isTesSuccess(err2)) + return err2; - if (auto const ret = checkOffer(buy); !isTesSuccess(ret)) - return ret; - - if (auto const ret = checkOffer(sell); !isTesSuccess(ret)) - return ret; - - if (buy && sell) + if (bo && so) { // Brokered mode: - auto const bo = ctx.view.read(keylet::nftoffer(*buy)); - auto const so = ctx.view.read(keylet::nftoffer(*sell)); - // The two offers being brokered must be for the same token: if ((*bo)[sfNFTokenID] != (*so)[sfNFTokenID]) return tecNFTOKEN_BUY_SELL_MISMATCH; @@ -106,6 +112,14 @@ NFTokenAcceptOffer::preclaim(PreclaimContext const& ctx) if ((*so)[sfAmount] > (*bo)[sfAmount]) return tecINSUFFICIENT_PAYMENT; + // If the buyer specified a destination, that destination must be + // the seller or the broker. + if (auto const dest = bo->at(~sfDestination)) + { + if (*dest != so->at(sfOwner) && *dest != ctx.tx[sfAccount]) + return tecNFTOKEN_BUY_SELL_MISMATCH; + } + // If the seller specified a destination, that destination must be // the buyer or the broker. if (auto const dest = so->at(~sfDestination)) @@ -131,10 +145,8 @@ NFTokenAcceptOffer::preclaim(PreclaimContext const& ctx) } } - if (buy) + if (bo) { - auto const bo = ctx.view.read(keylet::nftoffer(*buy)); - if (((*bo)[sfFlags] & lsfSellNFToken) == lsfSellNFToken) return tecNFTOKEN_OFFER_TYPE_MISMATCH; @@ -143,10 +155,19 @@ NFTokenAcceptOffer::preclaim(PreclaimContext const& ctx) return tecCANT_ACCEPT_OWN_NFTOKEN_OFFER; // If not in bridged mode, the account must own the token: - if (!sell && + if (!so && !nft::findToken(ctx.view, ctx.tx[sfAccount], (*bo)[sfNFTokenID])) return tecNO_PERMISSION; + // If not in bridged mode... + if (!so) + { + // If the offer has a Destination field, the acceptor must be the + // Destination. + if (auto const dest = bo->at(~sfDestination); + dest.has_value() && *dest != ctx.tx[sfAccount]) + return tecNO_PERMISSION; + } // The account offering to buy must have funds: auto const needed = bo->at(sfAmount); @@ -160,10 +181,8 @@ NFTokenAcceptOffer::preclaim(PreclaimContext const& ctx) return tecINSUFFICIENT_FUNDS; } - if (sell) + if (so) { - auto const so = ctx.view.read(keylet::nftoffer(*sell)); - if (((*so)[sfFlags] & lsfSellNFToken) != lsfSellNFToken) return tecNFTOKEN_OFFER_TYPE_MISMATCH; @@ -176,7 +195,7 @@ NFTokenAcceptOffer::preclaim(PreclaimContext const& ctx) return tecNO_PERMISSION; // If not in bridged mode... - if (!buy) + if (!bo) { // If the offer has a Destination field, the acceptor must be the // Destination. diff --git a/src/ripple/app/tx/impl/NFTokenBurn.cpp b/src/ripple/app/tx/impl/NFTokenBurn.cpp index f1f5ae8a7..da23d78bd 100644 --- a/src/ripple/app/tx/impl/NFTokenBurn.cpp +++ b/src/ripple/app/tx/impl/NFTokenBurn.cpp @@ -77,27 +77,9 @@ NFTokenBurn::preclaim(PreclaimContext const& ctx) } } - auto const id = ctx.tx[sfNFTokenID]; - - std::size_t totalOffers = 0; - - { - Dir buys(ctx.view, keylet::nft_buys(id)); - totalOffers += std::distance(buys.begin(), buys.end()); - } - - if (totalOffers > maxDeletableTokenOfferEntries) - return tefTOO_BIG; - - { - Dir sells(ctx.view, keylet::nft_sells(id)); - totalOffers += std::distance(sells.begin(), sells.end()); - } - - if (totalOffers > maxDeletableTokenOfferEntries) - return tefTOO_BIG; - - return tesSUCCESS; + // If there are too many offers, then burning the token would produce too + // much metadata. Disallow burning a token with too many offers. + return nft::notTooManyOffers(ctx.view, ctx.tx[sfNFTokenID]); } TER diff --git a/src/ripple/app/tx/impl/NFTokenCreateOffer.cpp b/src/ripple/app/tx/impl/NFTokenCreateOffer.cpp index bf92472e2..80e4c3964 100644 --- a/src/ripple/app/tx/impl/NFTokenCreateOffer.cpp +++ b/src/ripple/app/tx/impl/NFTokenCreateOffer.cpp @@ -46,7 +46,11 @@ NFTokenCreateOffer::preflight(PreflightContext const& ctx) auto const nftFlags = nft::getFlags(ctx.tx[sfNFTokenID]); { - auto const amount = ctx.tx[sfAmount]; + STAmount const amount = ctx.tx[sfAmount]; + + if (amount.negative() && ctx.rules.enabled(fixNFTokenNegOffer)) + // An offer for a negative amount makes no sense. + return temBAD_AMOUNT; if (!isXRP(amount)) { @@ -78,9 +82,14 @@ NFTokenCreateOffer::preflight(PreflightContext const& ctx) if (auto dest = ctx.tx[~sfDestination]) { - // The destination field is only valid on a sell offer; it makes no - // sense in a buy offer. - if (!isSellOffer) + // Some folks think it makes sense for a buy offer to specify a + // specific broker using the Destination field. This change doesn't + // deserve it's own amendment, so we're piggy-backing on + // fixNFTokenNegOffer. + // + // Prior to fixNFTokenNegOffer any use of the Destination field on + // a buy offer was malformed. + if (!isSellOffer && !ctx.rules.enabled(fixNFTokenNegOffer)) return temMALFORMED; // The destination can't be the account executing the transaction. diff --git a/src/ripple/app/tx/impl/NFTokenMint.cpp b/src/ripple/app/tx/impl/NFTokenMint.cpp index b4e391c3e..f4d3eb856 100644 --- a/src/ripple/app/tx/impl/NFTokenMint.cpp +++ b/src/ripple/app/tx/impl/NFTokenMint.cpp @@ -40,7 +40,23 @@ NFTokenMint::preflight(PreflightContext const& ctx) if (auto const ret = preflight1(ctx); !isTesSuccess(ret)) return ret; - if (ctx.tx.getFlags() & tfNFTokenMintMask) + // Prior to fixRemoveNFTokenAutoTrustLine, transfer of an NFToken between + // accounts allowed a TrustLine to be added to the issuer of that token + // without explicit permission from that issuer. This was enabled by + // minting the NFToken with the tfTrustLine flag set. + // + // That capability could be used to attack the NFToken issuer. It + // would be possible for two accounts to trade the NFToken back and forth + // building up any number of TrustLines on the issuer, increasing the + // issuer's reserve without bound. + // + // The fixRemoveNFTokenAutoTrustLine amendment disables minting with the + // tfTrustLine flag as a way to prevent the attack. But until the + // amendment passes we still need to keep the old behavior available. + std::uint32_t const NFTokenMintMask = + ctx.rules.enabled(fixRemoveNFTokenAutoTrustLine) ? tfNFTokenMintMask + : tfNFTokenMintOldMask; + if (ctx.tx.getFlags() & NFTokenMintMask) return temINVALID_FLAG; if (auto const f = ctx.tx[~sfTransferFee]) diff --git a/src/ripple/app/tx/impl/SetSignerList.cpp b/src/ripple/app/tx/impl/SetSignerList.cpp index 78409ba71..07cc705ba 100644 --- a/src/ripple/app/tx/impl/SetSignerList.cpp +++ b/src/ripple/app/tx/impl/SetSignerList.cpp @@ -82,6 +82,7 @@ SetSignerList::preflight(PreflightContext const& ctx) return ret; auto const result = determineOperation(ctx.tx, ctx.flags, ctx.j); + if (std::get<0>(result) != tesSUCCESS) return std::get<0>(result); @@ -98,7 +99,11 @@ SetSignerList::preflight(PreflightContext const& ctx) // Validate our settings. auto const account = ctx.tx.getAccountID(sfAccount); NotTEC const ter = validateQuorumAndSignerEntries( - std::get<1>(result), std::get<2>(result), account, ctx.j); + std::get<1>(result), + std::get<2>(result), + account, + ctx.j, + ctx.rules); if (ter != tesSUCCESS) { return ter; @@ -149,7 +154,7 @@ SetSignerList::preCompute() // is valid until the featureMultiSignReserve amendment passes. Once it // passes then just 1 OwnerCount is associated with a SignerList. static int -signerCountBasedOwnerCountDelta(std::size_t entryCount) +signerCountBasedOwnerCountDelta(std::size_t entryCount, Rules const& rules) { // We always compute the full change in OwnerCount, taking into account: // o The fact that we're adding/removing a SignerList and @@ -164,9 +169,10 @@ signerCountBasedOwnerCountDelta(std::size_t entryCount) // units. A SignerList with 8 entries would cost 10 OwnerCount units. // // The static_cast should always be safe since entryCount should always - // be in the range from 1 to 8. We've got a lot of room to grow. + // be in the range from 1 to 8 (or 32 if ExpandedSignerList is enabled). + // We've got a lot of room to grow. assert(entryCount >= STTx::minMultiSigners); - assert(entryCount <= STTx::maxMultiSigners); + assert(entryCount <= STTx::maxMultiSigners(&rules)); return 2 + static_cast(entryCount); } @@ -195,7 +201,8 @@ removeSignersFromLedger( { STArray const& actualList = signers->getFieldArray(sfSignerEntries); removeFromOwnerCount = - signerCountBasedOwnerCountDelta(actualList.size()) * -1; + signerCountBasedOwnerCountDelta(actualList.size(), view.rules()) * + -1; } // Remove the node from the account directory. @@ -238,13 +245,14 @@ SetSignerList::validateQuorumAndSignerEntries( std::uint32_t quorum, std::vector const& signers, AccountID const& account, - beast::Journal j) + beast::Journal j, + Rules const& rules) { // Reject if there are too many or too few entries in the list. { std::size_t const signerCount = signers.size(); if ((signerCount < STTx::minMultiSigners) || - (signerCount > STTx::maxMultiSigners)) + (signerCount > STTx::maxMultiSigners(&rules))) { JLOG(j.trace()) << "Too many or too few signers in signer list."; return temMALFORMED; @@ -259,6 +267,9 @@ SetSignerList::validateQuorumAndSignerEntries( return temBAD_SIGNER; } + // Is the ExpandedSignerList amendment active? + bool const expandedSignerList = rules.enabled(featureExpandedSignerList); + // Make sure no signers reference this account. Also make sure the // quorum can be reached. std::uint64_t allSignersWeight(0); @@ -279,6 +290,14 @@ SetSignerList::validateQuorumAndSignerEntries( return temBAD_SIGNER; } + if (signer.tag && !expandedSignerList) + { + JLOG(j.trace()) << "Malformed transaction: sfWalletLocator " + "specified in SignerEntry " + << "but featureExpandedSignerList is not enabled."; + return temMALFORMED; + } + // Don't verify that the signer accounts exist. Non-existent accounts // may be phantom accounts (which are permitted). } @@ -321,7 +340,8 @@ SetSignerList::replaceSignerList() std::uint32_t flags{lsfOneOwnerCount}; if (!ctx_.view().rules().enabled(featureMultiSignReserve)) { - addedOwnerCount = signerCountBasedOwnerCountDelta(signers_.size()); + addedOwnerCount = signerCountBasedOwnerCountDelta( + signers_.size(), ctx_.view().rules()); flags = 0; } @@ -389,6 +409,9 @@ SetSignerList::writeSignersToSLE( if (flags) // Only set flags if they are non-default (default is zero). ledgerEntry->setFieldU32(sfFlags, flags); + bool const expandedSignerList = + ctx_.view().rules().enabled(featureExpandedSignerList); + // Create the SignerListArray one SignerEntry at a time. STArray toLedger(signers_.size()); for (auto const& entry : signers_) @@ -398,6 +421,11 @@ SetSignerList::writeSignersToSLE( obj.reserve(2); obj.setAccountID(sfAccount, entry.account); obj.setFieldU16(sfSignerWeight, entry.weight); + + // This is a defensive check to make absolutely sure we will never write + // a tag into the ledger while featureExpandedSignerList is not enabled + if (expandedSignerList && entry.tag) + obj.setFieldH256(sfWalletLocator, *(entry.tag)); } // Assign the SignerEntries. diff --git a/src/ripple/app/tx/impl/SetSignerList.h b/src/ripple/app/tx/impl/SetSignerList.h index 345d59402..f8e49e4a7 100644 --- a/src/ripple/app/tx/impl/SetSignerList.h +++ b/src/ripple/app/tx/impl/SetSignerList.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -83,7 +84,8 @@ private: std::uint32_t quorum, std::vector const& signers, AccountID const& account, - beast::Journal j); + beast::Journal j, + Rules const&); TER replaceSignerList(); diff --git a/src/ripple/app/tx/impl/SetTrust.cpp b/src/ripple/app/tx/impl/SetTrust.cpp index 5f268f2c2..23af19c7b 100644 --- a/src/ripple/app/tx/impl/SetTrust.cpp +++ b/src/ripple/app/tx/impl/SetTrust.cpp @@ -104,19 +104,27 @@ SetTrust::preclaim(PreclaimContext const& ctx) auto const currency = saLimitAmount.getCurrency(); auto const uDstAccountID = saLimitAmount.getIssuer(); - if (id == uDstAccountID) + if (ctx.view.rules().enabled(fixTrustLinesToSelf)) { - // Prevent trustline to self from being created, - // unless one has somehow already been created - // (in which case doApply will clean it up). - auto const sleDelete = - ctx.view.read(keylet::line(id, uDstAccountID, currency)); - - if (!sleDelete) - { - JLOG(ctx.j.trace()) - << "Malformed transaction: Can not extend credit to self."; + if (id == uDstAccountID) return temDST_IS_SRC; + } + else + { + if (id == uDstAccountID) + { + // Prevent trustline to self from being created, + // unless one has somehow already been created + // (in which case doApply will clean it up). + auto const sleDelete = + ctx.view.read(keylet::line(id, uDstAccountID, currency)); + + if (!sleDelete) + { + JLOG(ctx.j.trace()) + << "Malformed transaction: Can not extend credit to self."; + return temDST_IS_SRC; + } } } @@ -183,18 +191,19 @@ SetTrust::doApply() auto viewJ = ctx_.app.journal("View"); - if (account_ == uDstAccountID) + // Trust lines to self are impossible but because of the old bug there are + // two on 19-02-2022. This code was here to allow those trust lines to be + // deleted. The fixTrustLinesToSelf fix amendment will remove them when it + // enables so this code will no longer be needed. + if (!view().rules().enabled(fixTrustLinesToSelf) && + account_ == uDstAccountID) { - // The only purpose here is to allow a mistakenly created - // trust line to oneself to be deleted. If no such trust - // lines exist now, why not remove this code and simply - // return an error? - SLE::pointer sleDelete = - view().peek(keylet::line(account_, uDstAccountID, currency)); - - JLOG(j_.warn()) << "Clearing redundant line."; - - return trustDelete(view(), sleDelete, account_, uDstAccountID, viewJ); + return trustDelete( + view(), + view().peek(keylet::line(account_, uDstAccountID, currency)), + account_, + uDstAccountID, + viewJ); } SLE::pointer sleDst = view().peek(keylet::account(uDstAccountID)); diff --git a/src/ripple/app/tx/impl/SignerEntries.cpp b/src/ripple/app/tx/impl/SignerEntries.cpp index 5081dc3e2..a948b2f90 100644 --- a/src/ripple/app/tx/impl/SignerEntries.cpp +++ b/src/ripple/app/tx/impl/SignerEntries.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace ripple { @@ -41,7 +42,7 @@ SignerEntries::deserialize( } std::vector accountVec; - accountVec.reserve(STTx::maxMultiSigners); + accountVec.reserve(STTx::maxMultiSigners()); STArray const& sEntries(obj.getFieldArray(sfSignerEntries)); for (STObject const& sEntry : sEntries) @@ -57,7 +58,9 @@ SignerEntries::deserialize( // Extract SignerEntry fields. AccountID const account = sEntry.getAccountID(sfAccount); std::uint16_t const weight = sEntry.getFieldU16(sfSignerWeight); - accountVec.emplace_back(account, weight); + std::optional const tag = sEntry.at(~sfWalletLocator); + + accountVec.emplace_back(account, weight, tag); } return accountVec; } diff --git a/src/ripple/app/tx/impl/SignerEntries.h b/src/ripple/app/tx/impl/SignerEntries.h index 96b5e29d9..cf4921ecf 100644 --- a/src/ripple/app/tx/impl/SignerEntries.h +++ b/src/ripple/app/tx/impl/SignerEntries.h @@ -23,9 +23,11 @@ #include // NotTEC #include // #include // beast::Journal +#include // Rules #include // STTx::maxMultiSigners #include // temMALFORMED #include // AccountID +#include namespace ripple { @@ -42,9 +44,13 @@ public: { AccountID account; std::uint16_t weight; + std::optional tag; - SignerEntry(AccountID const& inAccount, std::uint16_t inWeight) - : account(inAccount), weight(inWeight) + SignerEntry( + AccountID const& inAccount, + std::uint16_t inWeight, + std::optional inTag) + : account(inAccount), weight(inWeight), tag(inTag) { } diff --git a/src/ripple/app/tx/impl/apply.cpp b/src/ripple/app/tx/impl/apply.cpp index 332364863..cc1e792c0 100644 --- a/src/ripple/app/tx/impl/apply.cpp +++ b/src/ripple/app/tx/impl/apply.cpp @@ -54,7 +54,7 @@ checkValidity( ? STTx::RequireFullyCanonicalSig::yes : STTx::RequireFullyCanonicalSig::no; - auto const sigVerify = tx.checkSign(requireCanonicalSig); + auto const sigVerify = tx.checkSign(requireCanonicalSig, rules); if (!sigVerify) { router.setFlags(id, SF_SIGBAD); diff --git a/src/ripple/app/tx/impl/details/NFTokenUtils.cpp b/src/ripple/app/tx/impl/details/NFTokenUtils.cpp index f99c6cf6b..d1214a98e 100644 --- a/src/ripple/app/tx/impl/details/NFTokenUtils.cpp +++ b/src/ripple/app/tx/impl/details/NFTokenUtils.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -131,15 +132,40 @@ getPageForToken( cmp; }); - // If splitIter == begin(), then the entire page is filled with - // equivalent tokens. We cannot split the page, so we cannot - // insert the requested token. - // // There should be no circumstance when splitIter == end(), but if it // were to happen we should bail out because something is confused. - if (splitIter == narr.begin() || splitIter == narr.end()) + if (splitIter == narr.end()) return nullptr; + // If splitIter == begin(), then the entire page is filled with + // equivalent tokens. This requires special handling. + if (splitIter == narr.begin()) + { + // Prior to fixNFTokenDirV1 we simply stopped. + if (!view.rules().enabled(fixNFTokenDirV1)) + return nullptr; + else + { + // This would be an ideal place for the spaceship operator... + int const relation = compare(id & nft::pageMask, cmp); + if (relation == 0) + // If the passed in id belongs exactly on this (full) page + // this account simply cannot store the NFT. + return nullptr; + + else if (relation > 0) + // We need to leave the entire contents of this page in + // narr so carr stays empty. The new NFT will be + // inserted in carr. This keeps the NFTs that must be + // together all on their own page. + splitIter = narr.end(); + + // If neither of those conditions apply then put all of + // narr into carr and produce an empty narr where the new NFT + // will be inserted. Leave the split at narr.begin(). + } + } + // Split narr at splitIter. STArray newCarr( std::make_move_iterator(splitIter), @@ -148,8 +174,20 @@ getPageForToken( std::swap(carr, newCarr); } - auto np = std::make_shared( - keylet::nftpage(base, carr[0].getFieldH256(sfNFTokenID))); + // Determine the ID for the page index. This decision is conditional on + // fixNFTokenDirV1 being enabled. But the condition for the decision + // is not possible unless fixNFTokenDirV1 is enabled. + // + // Note that we use uint256::next() because there's a subtlety in the way + // NFT pages are structured. The low 96-bits of NFT ID must be strictly + // less than the low 96-bits of the enclosing page's index. In order to + // accommodate that requirement we use an index one higher than the + // largest NFT in the page. + uint256 const tokenIDForNewPage = narr.size() == dirMaxTokensPerPage + ? narr[dirMaxTokensPerPage - 1].getFieldH256(sfNFTokenID).next() + : carr[0].getFieldH256(sfNFTokenID); + + auto np = std::make_shared(keylet::nftpage(base, tokenIDForNewPage)); np->setFieldArray(sfNFTokens, narr); np->setFieldH256(sfNextPageMin, cp->key()); @@ -172,10 +210,17 @@ getPageForToken( createCallback(view, owner); - return (first.key <= np->key()) ? np : cp; + // fixNFTokenDirV1 corrects a bug in the initial implementation that + // would put an NFT in the wrong page. The problem was caused by an + // off-by-one subtlety that the NFT can only be stored in the first page + // with a key that's strictly greater than `first` + if (!view.rules().enabled(fixNFTokenDirV1)) + return (first.key <= np->key()) ? np : cp; + + return (first.key < np->key()) ? np : cp; } -static bool +bool compareTokens(uint256 const& a, uint256 const& b) { // The sort of NFTokens needs to be fully deterministic, but the sort @@ -505,6 +550,33 @@ removeAllTokenOffers(ApplyView& view, Keylet const& directory) }); } +TER +notTooManyOffers(ReadView const& view, uint256 const& nftokenID) +{ + std::size_t totalOffers = 0; + + { + Dir buys(view, keylet::nft_buys(nftokenID)); + for (auto iter = buys.begin(); iter != buys.end(); iter.next_page()) + { + totalOffers += iter.page_size(); + if (totalOffers > maxDeletableTokenOfferEntries) + return tefTOO_BIG; + } + } + + { + Dir sells(view, keylet::nft_sells(nftokenID)); + for (auto iter = sells.begin(); iter != sells.end(); iter.next_page()) + { + totalOffers += iter.page_size(); + if (totalOffers > maxDeletableTokenOfferEntries) + return tefTOO_BIG; + } + } + return tesSUCCESS; +} + bool deleteTokenOffer(ApplyView& view, std::shared_ptr const& offer) { diff --git a/src/ripple/app/tx/impl/details/NFTokenUtils.h b/src/ripple/app/tx/impl/details/NFTokenUtils.h index aac5dbf5f..fa8c43b58 100644 --- a/src/ripple/app/tx/impl/details/NFTokenUtils.h +++ b/src/ripple/app/tx/impl/details/NFTokenUtils.h @@ -53,15 +53,14 @@ constexpr std::uint16_t const flagOnlyXRP = 0x0002; constexpr std::uint16_t const flagCreateTrustLines = 0x0004; constexpr std::uint16_t const flagTransferable = 0x0008; -/** Erases the specified offer from the specified token offer directory. - - */ -void -removeTokenOffer(ApplyView& view, uint256 const& id); - +/** Deletes all offers from the specified token offer directory. */ void removeAllTokenOffers(ApplyView& view, Keylet const& directory); +/** Returns tesSUCCESS if NFToken has few enough offers that it can be burned */ +TER +notTooManyOffers(ReadView const& view, uint256 const& nftokenID); + /** Finds the specified token in the owner's token directory. */ std::optional findToken( @@ -179,6 +178,9 @@ getIssuer(uint256 const& id) return AccountID::fromVoid(id.data() + 4); } +bool +compareTokens(uint256 const& a, uint256 const& b); + } // namespace nft } // namespace ripple diff --git a/src/ripple/basics/Expected.h b/src/ripple/basics/Expected.h index 09d2bdc50..8dc368eef 100644 --- a/src/ripple/basics/Expected.h +++ b/src/ripple/basics/Expected.h @@ -21,7 +21,10 @@ #define RIPPLE_BASICS_EXPECTED_H_INCLUDED #include + #include + +#include #include #include @@ -132,17 +135,16 @@ class [[nodiscard]] Expected using Base = boost::outcome_v2::result; public: - template < - typename U, - typename = std::enable_if_t>> - constexpr Expected(U r) : Base(T{std::forward(r)}) + template + requires std::convertible_to constexpr Expected(U && r) + : Base(T{std::forward(r)}) { } - template < - typename U, - typename = std::enable_if_t>> - constexpr Expected(Unexpected e) : Base(E{std::forward(e.value())}) + template + requires std::convertible_to && + (!std::is_reference_v)constexpr Expected(Unexpected e) + : Base(E{std::move(e.value())}) { } @@ -215,10 +217,10 @@ public: { } - template < - typename U, - typename = std::enable_if_t>> - constexpr Expected(Unexpected e) : Base(E{std::forward(e.value())}) + template + requires std::convertible_to && + (!std::is_reference_v)constexpr Expected(Unexpected e) + : Base(E{std::move(e.value())}) { } diff --git a/src/ripple/basics/MathUtilities.h b/src/ripple/basics/MathUtilities.h index f9dbcbbbc..4e8e3751c 100644 --- a/src/ripple/basics/MathUtilities.h +++ b/src/ripple/basics/MathUtilities.h @@ -21,7 +21,7 @@ #define RIPPLE_BASICS_MATHUTILITIES_H_INCLUDED #include -#include +#include #include namespace ripple { diff --git a/src/ripple/basics/Slice.h b/src/ripple/basics/Slice.h index 67c954bb7..0ba6a94b6 100644 --- a/src/ripple/basics/Slice.h +++ b/src/ripple/basics/Slice.h @@ -48,7 +48,8 @@ private: std::size_t size_ = 0; public: - using const_iterator = std::uint8_t const*; + using value_type = std::uint8_t; + using const_iterator = value_type const*; /** Default constructed Slice has length 0. */ Slice() noexcept = default; @@ -75,13 +76,13 @@ public: This may be zero for an empty range. */ /** @{ */ - std::size_t + [[nodiscard]] std::size_t size() const noexcept { return size_; } - std::size_t + [[nodiscard]] std::size_t length() const noexcept { return size_; diff --git a/src/ripple/basics/StringUtilities.h b/src/ripple/basics/StringUtilities.h index 81e9ae826..48de772ca 100644 --- a/src/ripple/basics/StringUtilities.h +++ b/src/ripple/basics/StringUtilities.h @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,24 @@ template std::optional strUnHex(std::size_t strSize, Iterator begin, Iterator end) { + static constexpr std::array const unxtab = []() { + std::array t{}; + + for (auto& x : t) + x = -1; + + for (int i = 0; i < 10; ++i) + t['0' + i] = i; + + for (int i = 0; i < 6; ++i) + { + t['A' + i] = 10 + i; + t['a' + i] = 10 + i; + } + + return t; + }(); + Blob out; out.reserve((strSize + 1) / 2); @@ -56,25 +75,22 @@ strUnHex(std::size_t strSize, Iterator begin, Iterator end) if (strSize & 1) { - int c = charUnHex(*iter); + int c = unxtab[*iter++]; if (c < 0) return {}; out.push_back(c); - ++iter; } while (iter != end) { - int cHigh = charUnHex(*iter); - ++iter; + int cHigh = unxtab[*iter++]; if (cHigh < 0) return {}; - int cLow = charUnHex(*iter); - ++iter; + int cLow = unxtab[*iter++]; if (cLow < 0) return {}; diff --git a/src/ripple/basics/TaggedCache.h b/src/ripple/basics/TaggedCache.h index 548d21dc7..6765ff16b 100644 --- a/src/ripple/basics/TaggedCache.h +++ b/src/ripple/basics/TaggedCache.h @@ -196,14 +196,17 @@ public: return true; } + using SweptPointersVector = std::pair< + std::vector>, + std::vector>>; + void sweep() { // Keep references to all the stuff we sweep // For performance, each worker thread should exit before the swept data // is destroyed but still within the main cache lock. - std::vector>> allStuffToSweep( - m_cache.partitions()); + std::vector allStuffToSweep(m_cache.partitions()); clock_type::time_point const now(m_clock.now()); clock_type::time_point when_expire; @@ -652,9 +655,9 @@ private: clock_type::time_point const& when_expire, [[maybe_unused]] clock_type::time_point const& now, typename KeyValueCacheType::map_type& partition, - std::vector>& stuffToSweep, + SweptPointersVector& stuffToSweep, std::atomic& allRemovals, - std::lock_guard const& lock) + std::lock_guard const&) { return std::thread([&, this]() { int cacheRemovals = 0; @@ -662,7 +665,8 @@ private: // Keep references to all the stuff we sweep // so that we can destroy them outside the lock. - stuffToSweep.reserve(partition.size()); + stuffToSweep.first.reserve(partition.size()); + stuffToSweep.second.reserve(partition.size()); { auto cit = partition.begin(); while (cit != partition.end()) @@ -672,6 +676,8 @@ private: // weak if (cit->second.isExpired()) { + stuffToSweep.second.push_back( + std::move(cit->second.weak_ptr)); ++mapRemovals; cit = partition.erase(cit); } @@ -684,9 +690,10 @@ private: { // strong, expired ++cacheRemovals; - if (cit->second.ptr.unique()) + if (cit->second.ptr.use_count() == 1) { - stuffToSweep.push_back(cit->second.ptr); + stuffToSweep.first.push_back( + std::move(cit->second.ptr)); ++mapRemovals; cit = partition.erase(cit); } @@ -722,9 +729,9 @@ private: clock_type::time_point const& when_expire, clock_type::time_point const& now, typename KeyOnlyCacheType::map_type& partition, - std::vector>& stuffToSweep, + SweptPointersVector&, std::atomic& allRemovals, - std::lock_guard const& lock) + std::lock_guard const&) { return std::thread([&, this]() { int cacheRemovals = 0; @@ -732,7 +739,6 @@ private: // Keep references to all the stuff we sweep // so that we can destroy them outside the lock. - stuffToSweep.reserve(partition.size()); { auto cit = partition.begin(); while (cit != partition.end()) diff --git a/src/ripple/basics/ThreadSafetyAnalysis.h b/src/ripple/basics/ThreadSafetyAnalysis.h new file mode 100644 index 000000000..b1889d5b4 --- /dev/null +++ b/src/ripple/basics/ThreadSafetyAnalysis.h @@ -0,0 +1,63 @@ +#ifndef RIPPLE_BASICS_THREAD_SAFTY_ANALYSIS_H_INCLUDED +#define RIPPLE_BASICS_THREAD_SAFTY_ANALYSIS_H_INCLUDED + +#ifdef RIPPLE_ENABLE_THREAD_SAFETY_ANNOTATIONS +#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#else +#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op +#endif + +#define CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(capability(x)) + +#define SCOPED_CAPABILITY THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) + +#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) + +#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) + +#define ACQUIRED_BEFORE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) + +#define ACQUIRED_AFTER(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) + +#define REQUIRES(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__)) + +#define REQUIRES_SHARED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(requires_shared_capability(__VA_ARGS__)) + +#define ACQUIRE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__)) + +#define ACQUIRE_SHARED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquire_shared_capability(__VA_ARGS__)) + +#define RELEASE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__)) + +#define RELEASE_SHARED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(release_shared_capability(__VA_ARGS__)) + +#define RELEASE_GENERIC(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(release_generic_capability(__VA_ARGS__)) + +#define TRY_ACQUIRE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__)) + +#define TRY_ACQUIRE_SHARED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_shared_capability(__VA_ARGS__)) + +#define EXCLUDES(...) THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) + +#define ASSERT_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(assert_capability(x)) + +#define ASSERT_SHARED_CAPABILITY(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_capability(x)) + +#define RETURN_CAPABILITY(x) THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) + +#define NO_THREAD_SAFETY_ANALYSIS \ + THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) + +#endif diff --git a/src/ripple/basics/base_uint.h b/src/ripple/basics/base_uint.h index ccbb24a13..8f277c300 100644 --- a/src/ripple/basics/base_uint.h +++ b/src/ripple/basics/base_uint.h @@ -26,6 +26,7 @@ #define RIPPLE_BASICS_BASE_UINT_H_INCLUDED #include +#include #include #include #include @@ -56,6 +57,11 @@ struct is_contiguous_container< { }; +template <> +struct is_contiguous_container : std::true_type +{ +}; + } // namespace detail /** Integers of any length that is a multiple of 32-bits diff --git a/src/ripple/basics/impl/Archive.cpp b/src/ripple/basics/impl/Archive.cpp index 47e3d1ab3..73e14a936 100644 --- a/src/ripple/basics/impl/Archive.cpp +++ b/src/ripple/basics/impl/Archive.cpp @@ -23,6 +23,8 @@ #include #include +#include + namespace ripple { void diff --git a/src/ripple/basics/impl/StringUtilities.cpp b/src/ripple/basics/impl/StringUtilities.cpp index 8036cc3bf..bebbe1ef8 100644 --- a/src/ripple/basics/impl/StringUtilities.cpp +++ b/src/ripple/basics/impl/StringUtilities.cpp @@ -90,6 +90,13 @@ parseUrl(parsedURL& pUrl, std::string const& strUrl) if (!port.empty()) { pUrl.port = beast::lexicalCast(port); + + // For inputs larger than 2^32-1 (65535), lexicalCast returns 0. + // parseUrl returns false for such inputs. + if (pUrl.port == 0) + { + return false; + } } pUrl.path = smMatch[6]; diff --git a/src/ripple/basics/impl/make_SSLContext.cpp b/src/ripple/basics/impl/make_SSLContext.cpp index 67beb79d3..79a0e9009 100644 --- a/src/ripple/basics/impl/make_SSLContext.cpp +++ b/src/ripple/basics/impl/make_SSLContext.cpp @@ -17,18 +17,55 @@ */ //============================================================================== -#include #include #include -#include -#include -#include +#include #include namespace ripple { namespace openssl { namespace detail { +/** The default strength of self-signed RSA certifices. + + Per NIST Special Publication 800-57 Part 3, 2048-bit RSA is still + considered acceptably secure. Generally, we would want to go above + and beyond such recommendations (e.g. by using 3072 or 4096 bits) + but there is a computational cost associated with that may not + be worth paying, considering that: + + - We regenerate a new ephemeral certificate and a securely generated + random private key every time the server is started; and + - There should not be any truly secure information (e.g. seeds or private + keys) that gets relayed to the server anyways over these RPCs. + + @note If you increase the number of bits you need to generate new + default DH parameters and update defaultDH accordingly. + * */ +int defaultRSAKeyBits = 2048; + +/** The default DH parameters. + + These were generated using the OpenSSL command: `openssl dhparam 2048` + by Nik Bougalis on May, 29, 2022. + + It is safe to use this, but if you want you can generate different + parameters and put them here. There's no easy way to change this + via the config file at this time. + + @note If you increase the number of bits you need to update + defaultRSAKeyBits accordingly. + */ +static constexpr char const defaultDH[] = + "-----BEGIN DH PARAMETERS-----\n" + "MIIBCAKCAQEApKSWfR7LKy0VoZ/SDCObCvJ5HKX2J93RJ+QN8kJwHh+uuA8G+t8Q\n" + "MDRjL5HanlV/sKN9HXqBc7eqHmmbqYwIXKUt9MUZTLNheguddxVlc2IjdP5i9Ps8\n" + "l7su8tnP0l1JvC6Rfv3epRsEAw/ZW/lC2IwkQPpOmvnENQhQ6TgrUzcGkv4Bn0X6\n" + "pxrDSBpZ+45oehGCUAtcbY8b02vu8zPFoxqo6V/+MIszGzldlik5bVqrJpVF6E8C\n" + "tRqHjj6KuDbPbjc+pRGvwx/BSO3SULxmYu9J1NOk090MU1CMt6IJY7TpEc9Xrac9\n" + "9yqY3xXZID240RRcaJ25+U4lszFPqP+CEwIBAg==\n" + "-----END DH PARAMETERS-----"; + /** The default list of ciphers we accept over TLS. Generally we include cipher suites that are part of TLS v1.2, but @@ -43,188 +80,148 @@ namespace detail { global or per-port basis, using the `ssl_ciphers` directive in the config file. */ -std::string const defaultCipherList = "TLSv1.2:!DSS:!PSK:!eNULL:!aNULL"; - -template -struct custom_delete; - -template <> -struct custom_delete -{ - explicit custom_delete() = default; - - void - operator()(RSA* rsa) const - { - RSA_free(rsa); - } -}; - -template <> -struct custom_delete -{ - explicit custom_delete() = default; - - void - operator()(EVP_PKEY* evp_pkey) const - { - EVP_PKEY_free(evp_pkey); - } -}; - -template <> -struct custom_delete -{ - explicit custom_delete() = default; - - void - operator()(X509* x509) const - { - X509_free(x509); - } -}; - -template <> -struct custom_delete -{ - explicit custom_delete() = default; - - void - operator()(DH* dh) const - { - DH_free(dh); - } -}; - -template -using custom_delete_unique_ptr = std::unique_ptr>; - -// RSA - -using rsa_ptr = custom_delete_unique_ptr; - -static rsa_ptr -rsa_generate_key(int n_bits) -{ -#if OPENSSL_VERSION_NUMBER >= 0x00908000L - BIGNUM* bn = BN_new(); - BN_set_word(bn, RSA_F4); - - RSA* rsa = RSA_new(); - if (RSA_generate_key_ex(rsa, n_bits, bn, nullptr) != 1) - { - RSA_free(rsa); - rsa = nullptr; - } - - BN_free(bn); -#else - RSA* rsa = RSA_generate_key(n_bits, RSA_F4, nullptr, nullptr); -#endif - - if (rsa == nullptr) - LogicError("RSA_generate_key failed"); - - return rsa_ptr(rsa); -} - -// EVP_PKEY - -using evp_pkey_ptr = custom_delete_unique_ptr; - -static evp_pkey_ptr -evp_pkey_new() -{ - EVP_PKEY* evp_pkey = EVP_PKEY_new(); - - if (evp_pkey == nullptr) - LogicError("EVP_PKEY_new failed"); - - return evp_pkey_ptr(evp_pkey); -} - -static void -evp_pkey_assign_rsa(EVP_PKEY* evp_pkey, rsa_ptr rsa) -{ - if (!EVP_PKEY_assign_RSA(evp_pkey, rsa.get())) - LogicError("EVP_PKEY_assign_RSA failed"); - - rsa.release(); -} - -// X509 - -using x509_ptr = custom_delete_unique_ptr; - -static x509_ptr -x509_new() -{ - X509* x509 = X509_new(); - - if (x509 == nullptr) - LogicError("X509_new failed"); - - X509_set_version(x509, NID_X509); - - int const margin = 60 * 60; // 3600, one hour - int const length = 10 * 365.25 * 24 * 60 * 60; // 315576000, ten years - - X509_gmtime_adj(X509_get_notBefore(x509), -margin); - X509_gmtime_adj(X509_get_notAfter(x509), length); - - return x509_ptr(x509); -} - -static void -x509_set_pubkey(X509* x509, EVP_PKEY* evp_pkey) -{ - X509_set_pubkey(x509, evp_pkey); -} - -static void -x509_sign(X509* x509, EVP_PKEY* evp_pkey) -{ - if (!X509_sign(x509, evp_pkey, EVP_sha1())) - LogicError("X509_sign failed"); -} - -static void -ssl_ctx_use_certificate(SSL_CTX* const ctx, x509_ptr cert) -{ - if (SSL_CTX_use_certificate(ctx, cert.get()) <= 0) - LogicError("SSL_CTX_use_certificate failed"); -} - -static void -ssl_ctx_use_privatekey(SSL_CTX* const ctx, evp_pkey_ptr key) -{ - if (SSL_CTX_use_PrivateKey(ctx, key.get()) <= 0) - LogicError("SSL_CTX_use_PrivateKey failed"); -} - -static std::string -error_message(std::string const& what, boost::system::error_code const& ec) -{ - std::stringstream ss; - ss << what << ": " << ec.message() << " (" << ec.value() << ")"; - return ss.str(); -} +std::string const defaultCipherList = "TLSv1.2:!CBC:!DSS:!PSK:!eNULL:!aNULL"; static void initAnonymous(boost::asio::ssl::context& context) { using namespace openssl; - evp_pkey_ptr pkey = evp_pkey_new(); - evp_pkey_assign_rsa(pkey.get(), rsa_generate_key(2048)); + static auto defaultRSA = []() { + BIGNUM* bn = BN_new(); + BN_set_word(bn, RSA_F4); - x509_ptr cert = x509_new(); - x509_set_pubkey(cert.get(), pkey.get()); - x509_sign(cert.get(), pkey.get()); + auto rsa = RSA_new(); + + if (!rsa) + LogicError("RSA_new failed"); + + if (RSA_generate_key_ex(rsa, defaultRSAKeyBits, bn, nullptr) != 1) + LogicError("RSA_generate_key_ex failure"); + + BN_clear_free(bn); + + return rsa; + }(); + + static auto defaultEphemeralPrivateKey = []() { + auto pkey = EVP_PKEY_new(); + + if (!pkey) + LogicError("EVP_PKEY_new failed"); + + // We need to up the reference count of here, since we are retaining a + // copy of the key for (potential) reuse. + if (RSA_up_ref(defaultRSA) != 1) + LogicError( + "EVP_PKEY_assign_RSA: incrementing reference count failed"); + + if (!EVP_PKEY_assign_RSA(pkey, defaultRSA)) + LogicError("EVP_PKEY_assign_RSA failed"); + + return pkey; + }(); + + static auto defaultCert = []() { + auto x509 = X509_new(); + + if (x509 == nullptr) + LogicError("X509_new failed"); + + // According to the standards (X.509 et al), the value should be one + // less than the actualy certificate version we want. Since we want + // version 3, we must use a 2. + X509_set_version(x509, 2); + + // To avoid leaking information about the precise time that the + // server started up, we adjust the validity period: + char buf[16] = {0}; + + auto const ts = std::time(nullptr) - (25 * 60 * 60); + + int ret = std::strftime( + buf, sizeof(buf) - 1, "%y%m%d000000Z", std::gmtime(&ts)); + + buf[ret] = 0; + + if (ASN1_TIME_set_string_X509(X509_get_notBefore(x509), buf) != 1) + LogicError("Unable to set certificate validity date"); + + // And make it valid for two years + X509_gmtime_adj(X509_get_notAfter(x509), 2 * 365 * 24 * 60 * 60); + + // Set a serial number + if (auto b = BN_new(); b != nullptr) + { + if (BN_rand(b, 128, BN_RAND_TOP_ANY, BN_RAND_BOTTOM_ANY)) + { + if (auto a = ASN1_INTEGER_new(); a != nullptr) + { + if (BN_to_ASN1_INTEGER(b, a)) + X509_set_serialNumber(x509, a); + + ASN1_INTEGER_free(a); + } + } + + BN_clear_free(b); + } + + // Some certificate details + { + X509V3_CTX ctx; + + X509V3_set_ctx_nodb(&ctx); + X509V3_set_ctx(&ctx, x509, x509, nullptr, nullptr, 0); + + if (auto ext = X509V3_EXT_conf_nid( + nullptr, &ctx, NID_basic_constraints, "critical,CA:FALSE")) + { + X509_add_ext(x509, ext, -1); + X509_EXTENSION_free(ext); + } + + if (auto ext = X509V3_EXT_conf_nid( + nullptr, + &ctx, + NID_ext_key_usage, + "critical,serverAuth,clientAuth")) + { + X509_add_ext(x509, ext, -1); + X509_EXTENSION_free(ext); + } + + if (auto ext = X509V3_EXT_conf_nid( + nullptr, &ctx, NID_key_usage, "critical,digitalSignature")) + { + X509_add_ext(x509, ext, -1); + X509_EXTENSION_free(ext); + } + + if (auto ext = X509V3_EXT_conf_nid( + nullptr, &ctx, NID_subject_key_identifier, "hash")) + { + X509_add_ext(x509, ext, -1); + X509_EXTENSION_free(ext); + } + } + + // And a private key + X509_set_pubkey(x509, defaultEphemeralPrivateKey); + + if (!X509_sign(x509, defaultEphemeralPrivateKey, EVP_sha256())) + LogicError("X509_sign failed"); + + return x509; + }(); SSL_CTX* const ctx = context.native_handle(); - ssl_ctx_use_certificate(ctx, std::move(cert)); - ssl_ctx_use_privatekey(ctx, std::move(pkey)); + + if (SSL_CTX_use_certificate(ctx, defaultCert) <= 0) + LogicError("SSL_CTX_use_certificate failed"); + + if (SSL_CTX_use_PrivateKey(ctx, defaultEphemeralPrivateKey) <= 0) + LogicError("SSL_CTX_use_PrivateKey failed"); } static void @@ -234,6 +231,10 @@ initAuthenticated( std::string const& cert_file, std::string const& chain_file) { + auto fmt_error = [](boost::system::error_code ec) -> std::string { + return " [" + std::to_string(ec.value()) + ": " + ec.message() + "]"; + }; + SSL_CTX* const ssl = context.native_handle(); bool cert_set = false; @@ -246,10 +247,7 @@ initAuthenticated( cert_file, boost::asio::ssl::context::pem, ec); if (ec) - { - LogicError(error_message("Problem with SSL certificate file.", ec) - .c_str()); - } + LogicError("Problem with SSL certificate file" + fmt_error(ec)); cert_set = true; } @@ -261,11 +259,10 @@ initAuthenticated( if (!f) { - LogicError(error_message( - "Problem opening SSL chain file.", - boost::system::error_code( - errno, boost::system::generic_category())) - .c_str()); + LogicError( + "Problem opening SSL chain file" + + fmt_error(boost::system::error_code( + errno, boost::system::generic_category()))); } try @@ -312,8 +309,7 @@ initAuthenticated( if (ec) { LogicError( - error_message("Problem using the SSL private key file.", ec) - .c_str()); + "Problem using the SSL private key file" + fmt_error(ec)); } } @@ -324,7 +320,7 @@ initAuthenticated( } std::shared_ptr -get_context(std::string const& cipherList) +get_context(std::string cipherList) { auto c = std::make_shared( boost::asio::ssl::context::sslv23); @@ -338,55 +334,20 @@ get_context(std::string const& cipherList) boost::asio::ssl::context::single_dh_use | boost::asio::ssl::context::no_compression); - { - auto const& l = !cipherList.empty() ? cipherList : defaultCipherList; - auto result = SSL_CTX_set_cipher_list(c->native_handle(), l.c_str()); - if (result != 1) - LogicError("SSL_CTX_set_cipher_list failed"); - } + if (cipherList.empty()) + cipherList = defaultCipherList; - // These are the raw DH parameters that Ripple Labs has - // chosen for Ripple, in the binary format needed by - // d2i_DHparams. - // - unsigned char const params[] = { - 0x30, 0x82, 0x01, 0x08, 0x02, 0x82, 0x01, 0x01, 0x00, 0x8f, 0xca, 0x66, - 0x85, 0x33, 0xcb, 0xcf, 0x36, 0x27, 0xb2, 0x4c, 0xb8, 0x50, 0xb8, 0xf9, - 0x53, 0xf8, 0xb9, 0x2d, 0x1c, 0xa2, 0xad, 0x86, 0x58, 0x29, 0x3b, 0x88, - 0x3e, 0xf5, 0x65, 0xb8, 0xda, 0x22, 0xf4, 0x8b, 0x21, 0x12, 0x18, 0xf7, - 0x16, 0xcd, 0x7c, 0xc7, 0x3a, 0x2d, 0x61, 0xb7, 0x11, 0xf6, 0xb0, 0x65, - 0xa0, 0x5b, 0xa4, 0x06, 0x95, 0x28, 0xa4, 0x4f, 0x76, 0xc0, 0xeb, 0xfa, - 0x95, 0xdf, 0xbf, 0x19, 0x90, 0x64, 0x8f, 0x60, 0xd5, 0x36, 0xba, 0xab, - 0x0d, 0x5a, 0x5c, 0x94, 0xd5, 0xf7, 0x32, 0xd6, 0x2a, 0x76, 0x77, 0x83, - 0x10, 0xc4, 0x2f, 0x10, 0x96, 0x3e, 0x37, 0x84, 0x45, 0x9c, 0xef, 0x33, - 0xf6, 0xd0, 0x2a, 0xa7, 0xce, 0x0a, 0xce, 0x0d, 0xa1, 0xa7, 0x44, 0x5d, - 0x18, 0x3f, 0x4f, 0xa4, 0x23, 0x9c, 0x5d, 0x74, 0x4f, 0xee, 0xdf, 0xaa, - 0x0d, 0x0a, 0x52, 0x57, 0x73, 0xb1, 0xe4, 0xc5, 0x72, 0x93, 0x9d, 0x03, - 0xe9, 0xf5, 0x48, 0x8c, 0xd1, 0xe6, 0x7c, 0x21, 0x65, 0x4e, 0x16, 0x51, - 0xa3, 0x16, 0x51, 0x10, 0x75, 0x60, 0x37, 0x93, 0xb8, 0x15, 0xd6, 0x14, - 0x41, 0x4a, 0x61, 0xc9, 0x1a, 0x4e, 0x9f, 0x38, 0xd8, 0x2c, 0xa5, 0x31, - 0xe1, 0x87, 0xda, 0x1f, 0xa4, 0x31, 0xa2, 0xa4, 0x42, 0x1e, 0xe0, 0x30, - 0xea, 0x2f, 0x9b, 0x77, 0x91, 0x59, 0x3e, 0xd5, 0xd0, 0xc5, 0x84, 0x45, - 0x17, 0x19, 0x74, 0x8b, 0x18, 0xb0, 0xc1, 0xe0, 0xfc, 0x1c, 0xaf, 0xe6, - 0x2a, 0xef, 0x4e, 0x0e, 0x8a, 0x5c, 0xc2, 0x91, 0xb9, 0x2b, 0xf8, 0x17, - 0x8d, 0xed, 0x44, 0xaa, 0x47, 0xaa, 0x52, 0xa2, 0xdb, 0xb6, 0xf5, 0xa1, - 0x88, 0x85, 0xa1, 0xd5, 0x87, 0xb8, 0x07, 0xd3, 0x97, 0xbe, 0x37, 0x74, - 0x72, 0xf1, 0xa8, 0x29, 0xf1, 0xa7, 0x7d, 0x19, 0xc3, 0x27, 0x09, 0xcf, - 0x23, 0x02, 0x01, 0x02}; + if (auto result = + SSL_CTX_set_cipher_list(c->native_handle(), cipherList.c_str()); + result != 1) + LogicError("SSL_CTX_set_cipher_list failed"); - unsigned char const* data = ¶ms[0]; - - custom_delete_unique_ptr const dh{ - d2i_DHparams(nullptr, &data, sizeof(params))}; - if (!dh) - LogicError("d2i_DHparams returned nullptr."); - - SSL_CTX_set_tmp_dh(c->native_handle(), dh.get()); + c->use_tmp_dh({std::addressof(detail::defaultDH), sizeof(defaultDH)}); // Disable all renegotiation support in TLS v1.2. This can help prevent // exploitation of the bug described in CVE-2021-3499 (for details see - // https://www.openssl.org/news/secadv/20210325.txt) when linking against - // OpenSSL versions prior to 1.1.1k. + // https://www.openssl.org/news/secadv/20210325.txt) when linking + // against OpenSSL versions prior to 1.1.1k. SSL_CTX_set_options(c->native_handle(), SSL_OP_NO_RENEGOTIATION); return c; diff --git a/src/ripple/basics/spinlock.h b/src/ripple/basics/spinlock.h new file mode 100644 index 000000000..85a2ac41d --- /dev/null +++ b/src/ripple/basics/spinlock.h @@ -0,0 +1,223 @@ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright 2022, Nikolaos D. Bougalis + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ + +#ifndef RIPPLE_BASICS_SPINLOCK_H_INCLUDED +#define RIPPLE_BASICS_SPINLOCK_H_INCLUDED + +#include +#include +#include +#include + +#ifndef __aarch64__ +#include +#endif + +namespace ripple { + +namespace detail { +/** Inform the processor that we are in a tight spin-wait loop. + + Spinlocks caught in tight loops can result in the processor's pipeline + filling up with comparison operations, resulting in a misprediction at + the time the lock is finally acquired, necessitating pipeline flushing + which is ridiculously expensive and results in very high latency. + + This function instructs the processor to "pause" for some architecture + specific amount of time, to prevent this. + */ +inline void +spin_pause() noexcept +{ +#ifdef __aarch64__ + asm volatile("yield"); +#else + _mm_pause(); +#endif +} + +} // namespace detail + +/** @{ */ +/** Classes to handle arrays of spinlocks packed into a single atomic integer: + + Packed spinlocks allow for tremendously space-efficient lock-sharding + but they come at a cost. + + First, the implementation is necessarily low-level and uses advanced + features like memory ordering and highly platform-specific tricks to + maximize performance. This imposes a significant and ongoing cost to + developers. + + Second, and perhaps most important, is that the packing of multiple + locks into a single integer which, albeit space-efficient, also has + performance implications stemming from data dependencies, increased + cache-coherency traffic between processors and heavier loads on the + processor's load/store units. + + To be sure, these locks can have advantages but they are definitely + not general purpose locks and should not be thought of or used that + way. The use cases for them are likely few and far between; without + a compelling reason to use them, backed by profiling data, it might + be best to use one of the standard locking primitives instead. Note + that in most common platforms, `std::mutex` is so heavily optimized + that it can, usually, outperform spinlocks. + + @tparam T An unsigned integral type (e.g. std::uint16_t) + */ + +/** A class that grabs a single packed spinlock from an atomic integer. + + This class meets the requirements of Lockable: + https://en.cppreference.com/w/cpp/named_req/Lockable + */ +template +class packed_spinlock +{ + // clang-format off + static_assert(std::is_unsigned_v); + static_assert(std::atomic::is_always_lock_free); + static_assert( + std::is_same_v&>().fetch_or(0)), T> && + std::is_same_v&>().fetch_and(0)), T>, + "std::atomic::fetch_and(T) and std::atomic::fetch_and(T) are required by packed_spinlock"); + // clang-format on + +private: + std::atomic& bits_; + T const mask_; + +public: + packed_spinlock(packed_spinlock const&) = delete; + packed_spinlock& + operator=(packed_spinlock const&) = delete; + + /** A single spinlock packed inside the specified atomic + + @param lock The atomic integer inside which the spinlock is packed. + @param index The index of the spinlock this object acquires. + + @note For performance reasons, you should strive to have `lock` be + on a cacheline by itself. + */ + packed_spinlock(std::atomic& lock, int index) + : bits_(lock), mask_(static_cast(1) << index) + { + assert(index >= 0 && (mask_ != 0)); + } + + [[nodiscard]] bool + try_lock() + { + return (bits_.fetch_or(mask_, std::memory_order_acquire) & mask_) == 0; + } + + void + lock() + { + while (!try_lock()) + { + // The use of relaxed memory ordering here is intentional and + // serves to help reduce cache coherency traffic during times + // of contention by avoiding writes that would definitely not + // result in the lock being acquired. + while ((bits_.load(std::memory_order_relaxed) & mask_) != 0) + detail::spin_pause(); + } + } + + void + unlock() + { + bits_.fetch_and(~mask_, std::memory_order_release); + } +}; + +/** A spinlock implemented on top of an atomic integer. + + @note Using `packed_spinlock` and `spinlock` against the same underlying + atomic integer can result in `spinlock` not being able to actually + acquire the lock during periods of high contention, because of how + the two locks operate: `spinlock` will spin trying to grab all the + bits at once, whereas any given `packed_spinlock` will only try to + grab one bit at a time. Caveat emptor. + + This class meets the requirements of Lockable: + https://en.cppreference.com/w/cpp/named_req/Lockable + */ +template +class spinlock +{ + static_assert(std::is_unsigned_v); + static_assert(std::atomic::is_always_lock_free); + +private: + std::atomic& lock_; + +public: + spinlock(spinlock const&) = delete; + spinlock& + operator=(spinlock const&) = delete; + + /** Grabs the + + @param lock The atomic integer to spin against. + + @note For performance reasons, you should strive to have `lock` be + on a cacheline by itself. + */ + spinlock(std::atomic& lock) : lock_(lock) + { + } + + [[nodiscard]] bool + try_lock() + { + T expected = 0; + + return lock_.compare_exchange_weak( + expected, + std::numeric_limits::max(), + std::memory_order_acquire, + std::memory_order_relaxed); + } + + void + lock() + { + while (!try_lock()) + { + // The use of relaxed memory ordering here is intentional and + // serves to help reduce cache coherency traffic during times + // of contention by avoiding writes that would definitely not + // result in the lock being acquired. + while (lock_.load(std::memory_order_relaxed) != 0) + detail::spin_pause(); + } + } + + void + unlock() + { + lock_.store(0, std::memory_order_release); + } +}; +/** @} */ + +} // namespace ripple + +#endif diff --git a/src/ripple/basics/strHex.h b/src/ripple/basics/strHex.h index e48ea9215..257fb540b 100644 --- a/src/ripple/basics/strHex.h +++ b/src/ripple/basics/strHex.h @@ -25,21 +25,6 @@ namespace ripple { -/** @{ */ -/** Converts a hex digit to the corresponding integer - @param cDigit one of '0'-'9', 'A'-'F' or 'a'-'f' - @return an integer from 0 to 15 on success; -1 on failure. -*/ -int -charUnHex(unsigned char c); - -inline int -charUnHex(char c) -{ - return charUnHex(static_cast(c)); -} -/** @} */ - template std::string strHex(FwdIt begin, FwdIt end) diff --git a/src/ripple/beast/container/detail/aged_container_iterator.h b/src/ripple/beast/container/detail/aged_container_iterator.h index cd8677ce1..7467ad33c 100644 --- a/src/ripple/beast/container/detail/aged_container_iterator.h +++ b/src/ripple/beast/container/detail/aged_container_iterator.h @@ -30,23 +30,21 @@ class aged_ordered_container; namespace detail { -// Idea for Base template argument to prevent having to repeat -// the base class declaration comes from newbiz on ##c++/Freenode -// // If Iterator is SCARY then this iterator will be as well. -template < - bool is_const, - class Iterator, - class Base = std::iterator< - typename std::iterator_traits::iterator_category, - typename std::conditional< - is_const, - typename Iterator::value_type::stashed::value_type const, - typename Iterator::value_type::stashed::value_type>::type, - typename std::iterator_traits::difference_type>> -class aged_container_iterator : public Base +template +class aged_container_iterator { public: + using iterator_category = + typename std::iterator_traits::iterator_category; + using value_type = typename std::conditional< + is_const, + typename Iterator::value_type::stashed::value_type const, + typename Iterator::value_type::stashed::value_type>::type; + using difference_type = + typename std::iterator_traits::difference_type; + using pointer = value_type*; + using reference = value_type&; using time_point = typename Iterator::value_type::stashed::time_point; aged_container_iterator() = default; @@ -56,13 +54,11 @@ public: template < bool other_is_const, class OtherIterator, - class OtherBase, class = typename std::enable_if< (other_is_const == false || is_const == true) && std::is_same::value == false>::type> explicit aged_container_iterator( - aged_container_iterator const& - other) + aged_container_iterator const& other) : m_iter(other.m_iter) { } @@ -70,22 +66,19 @@ public: // Disable constructing a const_iterator from a non-const_iterator. template < bool other_is_const, - class OtherBase, class = typename std::enable_if< other_is_const == false || is_const == true>::type> aged_container_iterator( - aged_container_iterator const& - other) + aged_container_iterator const& other) : m_iter(other.m_iter) { } // Disable assigning a const_iterator to a non-const iterator - template + template auto operator=( - aged_container_iterator const& - other) -> + aged_container_iterator const& other) -> typename std::enable_if< other_is_const == false || is_const == true, aged_container_iterator&>::type @@ -94,20 +87,18 @@ public: return *this; } - template + template bool - operator==( - aged_container_iterator const& - other) const + operator==(aged_container_iterator const& + other) const { return m_iter == other.m_iter; } - template + template bool - operator!=( - aged_container_iterator const& - other) const + operator!=(aged_container_iterator const& + other) const { return m_iter != other.m_iter; } @@ -142,13 +133,13 @@ public: return prev; } - typename Base::reference + reference operator*() const { return m_iter->value; } - typename Base::pointer + pointer operator->() const { return &m_iter->value; @@ -167,7 +158,7 @@ private: template friend class aged_unordered_container; - template + template friend class aged_container_iterator; template diff --git a/src/ripple/beast/container/detail/aged_ordered_container.h b/src/ripple/beast/container/detail/aged_ordered_container.h index ed6585dd5..23534a26b 100644 --- a/src/ripple/beast/container/detail/aged_ordered_container.h +++ b/src/ripple/beast/container/detail/aged_ordered_container.h @@ -989,22 +989,20 @@ public: template < bool is_const, class Iterator, - class Base, class = std::enable_if_t::value>> - beast::detail::aged_container_iterator - erase(beast::detail::aged_container_iterator pos); + beast::detail::aged_container_iterator + erase(beast::detail::aged_container_iterator pos); // enable_if prevents erase (reverse_iterator first, reverse_iterator last) // from compiling template < bool is_const, class Iterator, - class Base, class = std::enable_if_t::value>> - beast::detail::aged_container_iterator + beast::detail::aged_container_iterator erase( - beast::detail::aged_container_iterator first, - beast::detail::aged_container_iterator last); + beast::detail::aged_container_iterator first, + beast::detail::aged_container_iterator last); template auto @@ -1019,10 +1017,9 @@ public: template < bool is_const, class Iterator, - class Base, class = std::enable_if_t::value>> void - touch(beast::detail::aged_container_iterator pos) + touch(beast::detail::aged_container_iterator pos) { touch(pos, clock().now()); } @@ -1264,11 +1261,10 @@ private: template < bool is_const, class Iterator, - class Base, class = std::enable_if_t::value>> void touch( - beast::detail::aged_container_iterator pos, + beast::detail::aged_container_iterator pos, typename clock_type::time_point const& now); template < @@ -1436,7 +1432,12 @@ template < class Allocator> aged_ordered_container:: aged_ordered_container(aged_ordered_container const& other) - : m_config(other.m_config), m_cont(other.m_cont.comp()) + : m_config(other.m_config) +#if BOOST_VERSION >= 108000 + , m_cont(other.m_cont.get_comp()) +#else + , m_cont(other.m_cont.comp()) +#endif { insert(other.cbegin(), other.cend()); } @@ -1453,7 +1454,12 @@ aged_ordered_container:: aged_ordered_container( aged_ordered_container const& other, Allocator const& alloc) - : m_config(other.m_config, alloc), m_cont(other.m_cont.comp()) + : m_config(other.m_config, alloc) +#if BOOST_VERSION >= 108000 + , m_cont(other.m_cont.get_comp()) +#else + , m_cont(other.m_cont.comp()) +#endif { insert(other.cbegin(), other.cend()); } @@ -1486,7 +1492,12 @@ aged_ordered_container:: aged_ordered_container&& other, Allocator const& alloc) : m_config(std::move(other.m_config), alloc) +#if BOOST_VERSION >= 108000 + , m_cont(std::move(other.m_cont.get_comp())) +#else , m_cont(std::move(other.m_cont.comp())) +#endif + { insert(other.cbegin(), other.cend()); other.clear(); @@ -2010,13 +2021,13 @@ template < class Clock, class Compare, class Allocator> -template -beast::detail::aged_container_iterator +template +beast::detail::aged_container_iterator aged_ordered_container:: - erase(beast::detail::aged_container_iterator pos) + erase(beast::detail::aged_container_iterator pos) { unlink_and_delete_element(&*((pos++).iterator())); - return beast::detail::aged_container_iterator( + return beast::detail::aged_container_iterator( pos.iterator()); } @@ -2028,17 +2039,17 @@ template < class Clock, class Compare, class Allocator> -template -beast::detail::aged_container_iterator +template +beast::detail::aged_container_iterator aged_ordered_container:: erase( - beast::detail::aged_container_iterator first, - beast::detail::aged_container_iterator last) + beast::detail::aged_container_iterator first, + beast::detail::aged_container_iterator last) { for (; first != last;) unlink_and_delete_element(&*((first++).iterator())); - return beast::detail::aged_container_iterator( + return beast::detail::aged_container_iterator( first.iterator()); } @@ -2173,11 +2184,11 @@ template < class Clock, class Compare, class Allocator> -template +template void aged_ordered_container:: touch( - beast::detail::aged_container_iterator pos, + beast::detail::aged_container_iterator pos, typename clock_type::time_point const& now) { auto& e(*pos.iterator()); diff --git a/src/ripple/beast/container/detail/aged_unordered_container.h b/src/ripple/beast/container/detail/aged_unordered_container.h index 8bc2330fa..920e6196b 100644 --- a/src/ripple/beast/container/detail/aged_unordered_container.h +++ b/src/ripple/beast/container/detail/aged_unordered_container.h @@ -1205,15 +1205,15 @@ public: return emplace(std::forward(args)...); } - template - beast::detail::aged_container_iterator - erase(beast::detail::aged_container_iterator pos); + template + beast::detail::aged_container_iterator + erase(beast::detail::aged_container_iterator pos); - template - beast::detail::aged_container_iterator + template + beast::detail::aged_container_iterator erase( - beast::detail::aged_container_iterator first, - beast::detail::aged_container_iterator last); + beast::detail::aged_container_iterator first, + beast::detail::aged_container_iterator last); template auto @@ -1222,9 +1222,9 @@ public: void swap(aged_unordered_container& other) noexcept; - template + template void - touch(beast::detail::aged_container_iterator pos) + touch(beast::detail::aged_container_iterator pos) { touch(pos, clock().now()); } @@ -1541,10 +1541,10 @@ private: insert_unchecked(first, last); } - template + template void touch( - beast::detail::aged_container_iterator pos, + beast::detail::aged_container_iterator pos, typename clock_type::time_point const& now) { auto& e(*pos.iterator()); @@ -3044,8 +3044,8 @@ template < class Hash, class KeyEqual, class Allocator> -template -beast::detail::aged_container_iterator +template +beast::detail::aged_container_iterator aged_unordered_container< IsMulti, IsMap, @@ -3054,11 +3054,11 @@ aged_unordered_container< Clock, Hash, KeyEqual, - Allocator>:: - erase(beast::detail::aged_container_iterator pos) + Allocator>::erase(beast::detail::aged_container_iterator + pos) { unlink_and_delete_element(&*((pos++).iterator())); - return beast::detail::aged_container_iterator( + return beast::detail::aged_container_iterator( pos.iterator()); } @@ -3071,8 +3071,8 @@ template < class Hash, class KeyEqual, class Allocator> -template -beast::detail::aged_container_iterator +template +beast::detail::aged_container_iterator aged_unordered_container< IsMulti, IsMap, @@ -3083,13 +3083,13 @@ aged_unordered_container< KeyEqual, Allocator>:: erase( - beast::detail::aged_container_iterator first, - beast::detail::aged_container_iterator last) + beast::detail::aged_container_iterator first, + beast::detail::aged_container_iterator last) { for (; first != last;) unlink_and_delete_element(&*((first++).iterator())); - return beast::detail::aged_container_iterator( + return beast::detail::aged_container_iterator( first.iterator()); } diff --git a/src/ripple/beast/core/List.h b/src/ripple/beast/core/List.h index 1daf5cda7..9b3c889d6 100644 --- a/src/ripple/beast/core/List.h +++ b/src/ripple/beast/core/List.h @@ -72,11 +72,12 @@ private: template class ListIterator - : public std::iterator { public: + using iterator_category = std::bidirectional_iterator_tag; using value_type = typename beast::detail::CopyConst::type; + using difference_type = std::ptrdiff_t; using pointer = value_type*; using reference = value_type&; using size_type = std::size_t; diff --git a/src/ripple/beast/core/LockFreeStack.h b/src/ripple/beast/core/LockFreeStack.h index ff022b96a..107564415 100644 --- a/src/ripple/beast/core/LockFreeStack.h +++ b/src/ripple/beast/core/LockFreeStack.h @@ -29,18 +29,7 @@ namespace beast { //------------------------------------------------------------------------------ template -class LockFreeStackIterator : public std::iterator< - std::forward_iterator_tag, - typename Container::value_type, - typename Container::difference_type, - typename std::conditional< - IsConst, - typename Container::const_pointer, - typename Container::pointer>::type, - typename std::conditional< - IsConst, - typename Container::const_reference, - typename Container::reference>::type> +class LockFreeStackIterator { protected: using Node = typename Container::Node; @@ -48,7 +37,9 @@ protected: typename std::conditional::type; public: + using iterator_category = std::forward_iterator_tag; using value_type = typename Container::value_type; + using difference_type = typename Container::difference_type; using pointer = typename std::conditional< IsConst, typename Container::const_pointer, diff --git a/src/ripple/beast/utility/PropertyStream.h b/src/ripple/beast/utility/PropertyStream.h index bfedb39ec..dbcc8a2d7 100644 --- a/src/ripple/beast/utility/PropertyStream.h +++ b/src/ripple/beast/utility/PropertyStream.h @@ -77,12 +77,6 @@ protected: add(std::string const& key, signed char value); virtual void add(std::string const& key, unsigned char value); - virtual void - add(std::string const& key, wchar_t value); -#if 0 - virtual void add (std::string const& key, char16_t value); - virtual void add (std::string const& key, char32_t value); -#endif virtual void add(std::string const& key, short value); virtual void @@ -139,12 +133,6 @@ protected: add(signed char value); virtual void add(unsigned char value); - virtual void - add(wchar_t value); -#if 0 - virtual void add (char16_t value); - virtual void add (char32_t value); -#endif virtual void add(short value); virtual void diff --git a/src/ripple/beast/utility/rngfill.h b/src/ripple/beast/utility/rngfill.h index d906a66a0..71f434bda 100644 --- a/src/ripple/beast/utility/rngfill.h +++ b/src/ripple/beast/utility/rngfill.h @@ -21,6 +21,7 @@ #define BEAST_RANDOM_RNGFILL_H_INCLUDED #include +#include #include #include #include @@ -32,6 +33,7 @@ void rngfill(void* buffer, std::size_t bytes, Generator& g) { using result_type = typename Generator::result_type; + while (bytes >= sizeof(result_type)) { auto const v = g(); @@ -39,15 +41,22 @@ rngfill(void* buffer, std::size_t bytes, Generator& g) buffer = reinterpret_cast(buffer) + sizeof(v); bytes -= sizeof(v); } + + assert(bytes < sizeof(result_type)); + #ifdef __GNUC__ // gcc 11.1 (falsely) warns about an array-bounds overflow in release mode. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Warray-bounds" +#endif + if (bytes > 0) { auto const v = g(); std::memcpy(buffer, &v, bytes); } + +#ifdef __GNUC__ #pragma GCC diagnostic pop #endif } diff --git a/src/ripple/beast/utility/src/beast_PropertyStream.cpp b/src/ripple/beast/utility/src/beast_PropertyStream.cpp index 70c5ab9a8..ecd707e95 100644 --- a/src/ripple/beast/utility/src/beast_PropertyStream.cpp +++ b/src/ripple/beast/utility/src/beast_PropertyStream.cpp @@ -414,24 +414,6 @@ PropertyStream::add(std::string const& key, unsigned char value) lexical_add(key, value); } -void -PropertyStream::add(std::string const& key, wchar_t value) -{ - lexical_add(key, value); -} - -#if 0 -void PropertyStream::add (std::string const& key, char16_t value) -{ - lexical_add (key, value); -} - -void PropertyStream::add (std::string const& key, char32_t value) -{ - lexical_add (key, value); -} -#endif - void PropertyStream::add(std::string const& key, short value) { @@ -525,24 +507,6 @@ PropertyStream::add(unsigned char value) lexical_add(value); } -void -PropertyStream::add(wchar_t value) -{ - lexical_add(value); -} - -#if 0 -void PropertyStream::add (char16_t value) -{ - lexical_add (value); -} - -void PropertyStream::add (char32_t value) -{ - lexical_add (value); -} -#endif - void PropertyStream::add(short value) { diff --git a/src/ripple/consensus/Consensus.h b/src/ripple/consensus/Consensus.h index a86144c38..1d02e0f13 100644 --- a/src/ripple/consensus/Consensus.h +++ b/src/ripple/consensus/Consensus.h @@ -295,7 +295,7 @@ class Consensus using Result = ConsensusResult; - // Helper class to ensure adaptor is notified whenver the ConsensusMode + // Helper class to ensure adaptor is notified whenever the ConsensusMode // changes class MonitoredMode { diff --git a/src/ripple/consensus/ConsensusProposal.h b/src/ripple/consensus/ConsensusProposal.h index a3eccbb01..c5103cfe0 100644 --- a/src/ripple/consensus/ConsensusProposal.h +++ b/src/ripple/consensus/ConsensusProposal.h @@ -16,13 +16,16 @@ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ //============================================================================== -#ifndef RIPPLE_CONSENSUS_ConsensusProposal_H_INCLUDED -#define RIPPLE_CONSENSUS_ConsensusProposal_H_INCLUDED +#ifndef RIPPLE_CONSENSUS_CONSENSUSPROPOSAL_H_INCLUDED +#define RIPPLE_CONSENSUS_CONSENSUSPROPOSAL_H_INCLUDED +#include #include #include +#include #include #include +#include namespace ripple { /** Represents a proposed position taken during a round of consensus. @@ -169,6 +172,7 @@ public: NetClock::time_point newCloseTime, NetClock::time_point now) { + signingHash_.reset(); position_ = newPosition; closeTime_ = newCloseTime; time_ = now; @@ -185,6 +189,7 @@ public: void bowOut(NetClock::time_point now) { + signingHash_.reset(); time_ = now; proposeSeq_ = seqLeave; } @@ -210,6 +215,23 @@ public: return ret; } + //! The digest for this proposal, used for signing purposes. + uint256 const& + signingHash() const + { + if (!signingHash_) + { + signingHash_ = sha512Half( + HashPrefix::proposal, + std::uint32_t(proposeSeq()), + closeTime().time_since_epoch().count(), + prevLedger(), + position()); + } + + return signingHash_.value(); + } + private: //! Unique identifier of prior ledger this proposal is based on LedgerID_t previousLedger_; @@ -228,6 +250,9 @@ private: //! The identifier of the node taking this position NodeID_t nodeID_; + + //! The signing hash for this proposal + mutable std::optional signingHash_; }; template diff --git a/src/ripple/consensus/Validations.h b/src/ripple/consensus/Validations.h index 9200ac883..46bf4322a 100644 --- a/src/ripple/consensus/Validations.h +++ b/src/ripple/consensus/Validations.h @@ -27,8 +27,10 @@ #include #include #include + #include #include +#include #include #include @@ -294,7 +296,7 @@ class Validations using NodeKey = typename Validation::NodeKey; using WrappedValidationType = std::decay_t< - std::result_of_t>; + std::invoke_result_t>; // Manages concurrent access to members mutable Mutex mutex_; diff --git a/src/ripple/core/Config.h b/src/ripple/core/Config.h index c4a5076e7..2d440a1af 100644 --- a/src/ripple/core/Config.h +++ b/src/ripple/core/Config.h @@ -57,7 +57,8 @@ enum class SizedItem : std::size_t { lgrDBCache, openFinalLimit, burstSize, - ramSizeGB + ramSizeGB, + accountIdCacheSize, }; // This entire derived class is deprecated. diff --git a/src/ripple/core/impl/Config.cpp b/src/ripple/core/impl/Config.cpp index e53d96883..c2cfb14d2 100644 --- a/src/ripple/core/impl/Config.cpp +++ b/src/ripple/core/impl/Config.cpp @@ -108,26 +108,27 @@ namespace ripple { // clang-format off // The configurable node sizes are "tiny", "small", "medium", "large", "huge" -inline constexpr std::array>, 12> +inline constexpr std::array>, 13> sizedItems {{ // FIXME: We should document each of these items, explaining exactly // what they control and whether there exists an explicit // config option that can be used to override the default. - // tiny small medium large huge - {SizedItem::sweepInterval, {{ 10, 30, 60, 90, 120 }}}, - {SizedItem::treeCacheSize, {{ 262144, 524288, 2097152, 4194304, 8388608 }}}, - {SizedItem::treeCacheAge, {{ 30, 60, 90, 120, 900 }}}, - {SizedItem::ledgerSize, {{ 32, 32, 64, 256, 384 }}}, - {SizedItem::ledgerAge, {{ 30, 60, 180, 300, 600 }}}, - {SizedItem::ledgerFetch, {{ 2, 3, 4, 5, 8 }}}, - {SizedItem::hashNodeDBCache, {{ 4, 12, 24, 64, 128 }}}, - {SizedItem::txnDBCache, {{ 4, 12, 24, 64, 128 }}}, - {SizedItem::lgrDBCache, {{ 4, 8, 16, 32, 128 }}}, - {SizedItem::openFinalLimit, {{ 8, 16, 32, 64, 128 }}}, - {SizedItem::burstSize, {{ 4, 8, 16, 32, 48 }}}, - {SizedItem::ramSizeGB, {{ 8, 12, 16, 24, 32 }}}, + // tiny small medium large huge + {SizedItem::sweepInterval, {{ 10, 30, 60, 90, 120 }}}, + {SizedItem::treeCacheSize, {{ 262144, 524288, 2097152, 4194304, 8388608 }}}, + {SizedItem::treeCacheAge, {{ 30, 60, 90, 120, 900 }}}, + {SizedItem::ledgerSize, {{ 32, 32, 64, 256, 384 }}}, + {SizedItem::ledgerAge, {{ 30, 60, 180, 300, 600 }}}, + {SizedItem::ledgerFetch, {{ 2, 3, 4, 5, 8 }}}, + {SizedItem::hashNodeDBCache, {{ 4, 12, 24, 64, 128 }}}, + {SizedItem::txnDBCache, {{ 4, 12, 24, 64, 128 }}}, + {SizedItem::lgrDBCache, {{ 4, 8, 16, 32, 128 }}}, + {SizedItem::openFinalLimit, {{ 8, 16, 32, 64, 128 }}}, + {SizedItem::burstSize, {{ 4, 8, 16, 32, 48 }}}, + {SizedItem::ramSizeGB, {{ 8, 12, 16, 24, 32 }}}, + {SizedItem::accountIdCacheSize, {{ 20047, 50053, 77081, 150061, 300007 }}} }}; // Ensure that the order of entries in the table corresponds to the @@ -262,8 +263,6 @@ getEnvVar(char const* name) return value; } -constexpr FeeUnit32 Config::TRANSACTION_FEE_BASE; - Config::Config() : j_(beast::Journal::getNullSink()), ramSize_(detail::getMemorySize()) { diff --git a/src/ripple/crypto/csprng.h b/src/ripple/crypto/csprng.h index 395063249..3ad5d7000 100644 --- a/src/ripple/crypto/csprng.h +++ b/src/ripple/crypto/csprng.h @@ -39,9 +39,6 @@ class csprng_engine private: std::mutex mutex_; - void - mix(void* buffer, std::size_t count, double bitsPerByte); - public: using result_type = std::uint64_t; diff --git a/src/ripple/crypto/impl/csprng.cpp b/src/ripple/crypto/impl/csprng.cpp index a166fe288..04b3b3fc3 100644 --- a/src/ripple/crypto/impl/csprng.cpp +++ b/src/ripple/crypto/impl/csprng.cpp @@ -17,7 +17,6 @@ */ //============================================================================== -#include #include #include #include @@ -28,25 +27,19 @@ namespace ripple { -void -csprng_engine::mix(void* data, std::size_t size, double bitsPerByte) -{ - assert(data != nullptr); - assert(size != 0); - assert(bitsPerByte != 0); - - std::lock_guard lock(mutex_); - RAND_add(data, size, (size * bitsPerByte) / 8.0); -} - csprng_engine::csprng_engine() { - mix_entropy(); + // This is not strictly necessary + if (RAND_poll() != 1) + Throw("CSPRNG: Initial polling failed"); } csprng_engine::~csprng_engine() { + // This cleanup function is not needed in newer versions of OpenSSL +#if (OPENSSL_VERSION_NUMBER < 0x10100000L) RAND_cleanup(); +#endif } void @@ -64,46 +57,44 @@ csprng_engine::mix_entropy(void* buffer, std::size_t count) e = rd(); } - // Assume 2 bits per byte for the system entropy: - mix(entropy.data(), - entropy.size() * sizeof(std::random_device::result_type), - 2.0); + std::lock_guard lock(mutex_); + + // We add data to the pool, but we conservatively assume that + // it contributes no actual entropy. + RAND_add( + entropy.data(), + entropy.size() * sizeof(std::random_device::result_type), + 0); - // We want to be extremely conservative about estimating - // how much entropy the buffer the user gives us contains - // and assume only 0.5 bits of entropy per byte: if (buffer != nullptr && count != 0) - mix(buffer, count, 0.5); + RAND_add(buffer, count, 0); +} + +void +csprng_engine::operator()(void* ptr, std::size_t count) +{ + // RAND_bytes is thread-safe on OpenSSL 1.1.0 and later when compiled + // with thread support, so we don't need to grab a mutex. + // https://mta.openssl.org/pipermail/openssl-users/2020-November/013146.html +#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || !defined(OPENSSL_THREADS) + std::lock_guard lock(mutex_); +#endif + + auto const result = + RAND_bytes(reinterpret_cast(ptr), count); + + if (result != 1) + Throw("CSPRNG: Insufficient entropy"); } csprng_engine::result_type csprng_engine::operator()() { result_type ret; - - std::lock_guard lock(mutex_); - - auto const result = - RAND_bytes(reinterpret_cast(&ret), sizeof(ret)); - - if (result == 0) - Throw("Insufficient entropy"); - + (*this)(&ret, sizeof(result_type)); return ret; } -void -csprng_engine::operator()(void* ptr, std::size_t count) -{ - std::lock_guard lock(mutex_); - - auto const result = - RAND_bytes(reinterpret_cast(ptr), count); - - if (result != 1) - Throw("Insufficient entropy"); -} - csprng_engine& crypto_prng() { diff --git a/src/ripple/ledger/Directory.h b/src/ripple/ledger/Directory.h index c24d348e7..0efcf43e7 100644 --- a/src/ripple/ledger/Directory.h +++ b/src/ripple/ledger/Directory.h @@ -79,6 +79,12 @@ public: const_iterator operator++(int); + const_iterator& + next_page(); + + std::size_t + page_size(); + Keylet const& page() const { diff --git a/src/ripple/ledger/OpenView.h b/src/ripple/ledger/OpenView.h index 8467e4abc..98b783e3a 100644 --- a/src/ripple/ledger/OpenView.h +++ b/src/ripple/ledger/OpenView.h @@ -77,7 +77,7 @@ private: }; // List of tx, key order - // Use the boost pmr functionality instead of the c++-17 standard pmr + // Use boost::pmr functionality instead of std::pmr // functions b/c clang does not support pmr yet (as-of 9/2020) using txs_map = std::map< key_type, diff --git a/src/ripple/ledger/ReadView.h b/src/ripple/ledger/ReadView.h index bc9c0906a..714f8dc94 100644 --- a/src/ripple/ledger/ReadView.h +++ b/src/ripple/ledger/ReadView.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -125,64 +126,6 @@ struct LedgerInfo //------------------------------------------------------------------------------ -class DigestAwareReadView; - -/** Rules controlling protocol behavior. */ -class Rules -{ -private: - class Impl; - - std::shared_ptr impl_; - -public: - Rules(Rules const&) = default; - Rules& - operator=(Rules const&) = default; - - Rules() = delete; - - /** Construct an empty rule set. - - These are the rules reflected by - the genesis ledger. - */ - explicit Rules(std::unordered_set> const& presets); - - /** Construct rules from a ledger. - - The ledger contents are analyzed for rules - and amendments and extracted to the object. - */ - explicit Rules( - DigestAwareReadView const& ledger, - std::unordered_set> const& presets); - - /** Returns `true` if a feature is enabled. */ - bool - enabled(uint256 const& id) const; - - /** Returns `true` if these rules don't match the ledger. */ - bool - changed(DigestAwareReadView const& ledger) const; - - /** Returns `true` if two rule sets are identical. - - @note This is for diagnostics. To determine if new - rules should be constructed, call changed() first instead. - */ - bool - operator==(Rules const&) const; - - bool - operator!=(Rules const& other) const - { - return !(*this == other); - } -}; - -//------------------------------------------------------------------------------ - /** A view into a ledger. This interface provides read access to state @@ -423,6 +366,11 @@ getCloseAgree(LedgerInfo const& info) void addRaw(LedgerInfo const&, Serializer&, bool includeHash = false); +Rules +makeRulesGivenLedger( + DigestAwareReadView const& ledger, + std::unordered_set> const& presets); + } // namespace ripple #include diff --git a/src/ripple/ledger/detail/RawStateTable.h b/src/ripple/ledger/detail/RawStateTable.h index 2bee9e2a9..2bb38dc49 100644 --- a/src/ripple/ledger/detail/RawStateTable.h +++ b/src/ripple/ledger/detail/RawStateTable.h @@ -119,7 +119,7 @@ private: } }; - // Use the boost pmr functionality instead of the c++-17 standard pmr + // Use boost::pmr functionality instead of the std::pmr // functions b/c clang does not support pmr yet (as-of 9/2020) using items_t = std::map< key_type, diff --git a/src/ripple/ledger/impl/Directory.cpp b/src/ripple/ledger/impl/Directory.cpp index de7ea0d2d..759b4d71b 100644 --- a/src/ripple/ledger/impl/Directory.cpp +++ b/src/ripple/ledger/impl/Directory.cpp @@ -1,5 +1,4 @@ -//------------ -//------------------------------------------------------------------ +//------------------------------------------------------------------------------ /* This file is part of rippled: https://github.com/ripple/rippled Copyright (c) 2012, 2015 Ripple Labs Inc. @@ -81,35 +80,11 @@ const_iterator::operator++() if (++it_ != std::end(*indexes_)) { index_ = *it_; - } - else - { - auto const next = sle_->getFieldU64(sfIndexNext); - if (next == 0) - { - page_.key = root_.key; - index_ = beast::zero; - } - else - { - page_ = keylet::page(root_, next); - sle_ = view_->read(page_); - assert(sle_); - indexes_ = &sle_->getFieldV256(sfIndexes); - if (indexes_->empty()) - { - index_ = beast::zero; - } - else - { - it_ = std::begin(*indexes_); - index_ = *it_; - } - } + cache_ = std::nullopt; + return *this; } - cache_ = std::nullopt; - return *this; + return next_page(); } const_iterator @@ -121,4 +96,39 @@ const_iterator::operator++(int) return tmp; } +const_iterator& +const_iterator::next_page() +{ + auto const next = sle_->getFieldU64(sfIndexNext); + if (next == 0) + { + page_.key = root_.key; + index_ = beast::zero; + } + else + { + page_ = keylet::page(root_, next); + sle_ = view_->read(page_); + assert(sle_); + indexes_ = &sle_->getFieldV256(sfIndexes); + if (indexes_->empty()) + { + index_ = beast::zero; + } + else + { + it_ = std::begin(*indexes_); + index_ = *it_; + } + } + cache_ = std::nullopt; + return *this; +} + +std::size_t +const_iterator::page_size() +{ + return indexes_->size(); +} + } // namespace ripple diff --git a/src/ripple/ledger/impl/ReadView.cpp b/src/ripple/ledger/impl/ReadView.cpp index 77db253aa..57af008b4 100644 --- a/src/ripple/ledger/impl/ReadView.cpp +++ b/src/ripple/ledger/impl/ReadView.cpp @@ -21,108 +21,6 @@ namespace ripple { -class Rules::Impl -{ -private: - std::unordered_set> set_; - std::optional digest_; - std::unordered_set> const& presets_; - -public: - explicit Impl(std::unordered_set> const& presets) - : presets_(presets) - { - } - - explicit Impl( - DigestAwareReadView const& ledger, - std::unordered_set> const& presets) - : presets_(presets) - { - auto const k = keylet::amendments(); - digest_ = ledger.digest(k.key); - if (!digest_) - return; - auto const sle = ledger.read(k); - if (!sle) - { - // LogicError() ? - return; - } - - for (auto const& item : sle->getFieldV256(sfAmendments)) - set_.insert(item); - } - - bool - enabled(uint256 const& feature) const - { - if (presets_.count(feature) > 0) - return true; - return set_.count(feature) > 0; - } - - bool - changed(DigestAwareReadView const& ledger) const - { - auto const digest = ledger.digest(keylet::amendments().key); - if (!digest && !digest_) - return false; - if (!digest || !digest_) - return true; - return *digest != *digest_; - } - - bool - operator==(Impl const& other) const - { - if (!digest_ && !other.digest_) - return true; - if (!digest_ || !other.digest_) - return false; - return *digest_ == *other.digest_; - } -}; - -//------------------------------------------------------------------------------ - -Rules::Rules( - DigestAwareReadView const& ledger, - std::unordered_set> const& presets) - : impl_(std::make_shared(ledger, presets)) -{ -} - -Rules::Rules(std::unordered_set> const& presets) - : impl_(std::make_shared(presets)) -{ -} - -bool -Rules::enabled(uint256 const& id) const -{ - assert(impl_); - return impl_->enabled(id); -} - -bool -Rules::changed(DigestAwareReadView const& ledger) const -{ - assert(impl_); - return impl_->changed(ledger); -} - -bool -Rules::operator==(Rules const& other) const -{ - assert(impl_ && other.impl_); - if (impl_.get() == other.impl_.get()) - return true; - return *impl_ == *other.impl_; -} - -//------------------------------------------------------------------------------ - ReadView::sles_type::sles_type(ReadView const& view) : ReadViewFwdRange(view) { } @@ -167,4 +65,20 @@ ReadView::txs_type::end() const -> iterator return iterator(view_, view_->txsEnd()); } +Rules +makeRulesGivenLedger( + DigestAwareReadView const& ledger, + std::unordered_set> const& presets) +{ + Keylet const k = keylet::amendments(); + std::optional digest = ledger.digest(k.key); + if (digest) + { + auto const sle = ledger.read(k); + if (sle) + return Rules(presets, digest, sle->getFieldV256(sfAmendments)); + } + return Rules(presets); +} + } // namespace ripple diff --git a/src/ripple/net/HTTPDownloader.h b/src/ripple/net/HTTPDownloader.h index 1f2243a4f..39b9a904a 100644 --- a/src/ripple/net/HTTPDownloader.h +++ b/src/ripple/net/HTTPDownloader.h @@ -61,6 +61,12 @@ public: virtual ~HTTPDownloader() = default; + bool + sessionIsActive() const; + + bool + isStopping() const; + protected: // must be accessed through a shared_ptr // use make_XXX functions to create @@ -88,7 +94,7 @@ private: std::atomic stop_; // Used to protect sessionActive_ - std::mutex m_; + mutable std::mutex m_; bool sessionActive_; std::condition_variable c_; diff --git a/src/ripple/net/InfoSub.h b/src/ripple/net/InfoSub.h index 3c170669b..fb44e23b7 100644 --- a/src/ripple/net/InfoSub.h +++ b/src/ripple/net/InfoSub.h @@ -128,6 +128,11 @@ public: virtual bool unsubLedger(std::uint64_t uListener) = 0; + virtual bool + subBookChanges(ref ispListener) = 0; + virtual bool + unsubBookChanges(std::uint64_t uListener) = 0; + virtual bool subManifests(ref ispListener) = 0; virtual bool diff --git a/src/ripple/net/impl/DatabaseBody.ipp b/src/ripple/net/impl/DatabaseBody.ipp index 061a63025..cdc7da2bc 100644 --- a/src/ripple/net/impl/DatabaseBody.ipp +++ b/src/ripple/net/impl/DatabaseBody.ipp @@ -17,7 +17,7 @@ */ //============================================================================== -#include +#include namespace ripple { diff --git a/src/ripple/net/impl/HTTPDownloader.cpp b/src/ripple/net/impl/HTTPDownloader.cpp index 5ed2ceae0..44d274662 100644 --- a/src/ripple/net/impl/HTTPDownloader.cpp +++ b/src/ripple/net/impl/HTTPDownloader.cpp @@ -293,6 +293,20 @@ HTTPDownloader::stop() } } +bool +HTTPDownloader::sessionIsActive() const +{ + std::lock_guard lock(m_); + return sessionActive_; +} + +bool +HTTPDownloader::isStopping() const +{ + std::lock_guard lock(m_); + return stop_; +} + void HTTPDownloader::fail( boost::filesystem::path dstPath, diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp index 820f25ddf..eb4906f3a 100644 --- a/src/ripple/net/impl/RPCCall.cpp +++ b/src/ripple/net/impl/RPCCall.cpp @@ -1227,10 +1227,7 @@ public: int maxParams; }; - // FIXME: replace this with a function-static std::map and the lookup - // code with std::map::find when the problem with magic statics on - // Visual Studio is fixed. - static Command const commands[] = { + static constexpr Command commands[] = { // Request-response methods // - Returns an error, or the request. // - To modify the method, provide a new method in the request. @@ -1242,6 +1239,7 @@ public: {"account_objects", &RPCParser::parseAccountItems, 1, 5}, {"account_offers", &RPCParser::parseAccountItems, 1, 4}, {"account_tx", &RPCParser::parseAccountTransactions, 1, 8}, + {"book_changes", &RPCParser::parseLedgerId, 1, 1}, {"book_offers", &RPCParser::parseBookOffers, 2, 7}, {"can_delete", &RPCParser::parseCanDelete, 0, 1}, {"channel_authorize", &RPCParser::parseChannelAuthorize, 3, 4}, diff --git a/src/ripple/nodestore/Database.h b/src/ripple/nodestore/Database.h index bb9304507..0f9e95b23 100644 --- a/src/ripple/nodestore/Database.h +++ b/src/ripple/nodestore/Database.h @@ -324,6 +324,11 @@ protected: // The earliest shard index std::uint32_t const earliestShardIndex_; + // The maximum number of requests a thread extracts from the queue in an + // attempt to minimize the overhead of mutex acquisition. This is an + // advanced tunable, via the config file. The default value is 4. + int const requestBundle_; + void storeStats(std::uint64_t count, std::uint64_t sz) { @@ -368,6 +373,7 @@ private: std::atomic readStopping_ = false; std::atomic readThreads_ = 0; + std::atomic runningThreads_ = 0; virtual std::shared_ptr fetchNodeObject( diff --git a/src/ripple/nodestore/backend/RocksDBFactory.cpp b/src/ripple/nodestore/backend/RocksDBFactory.cpp index e17dc55de..1a9e529e1 100644 --- a/src/ripple/nodestore/backend/RocksDBFactory.cpp +++ b/src/ripple/nodestore/backend/RocksDBFactory.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include // VFALCO Bad dependency #include @@ -30,6 +31,7 @@ #include #include #include + #include #include @@ -310,7 +312,8 @@ public: } else { - status = Status(customCode + getStatus.code()); + status = + Status(customCode + unsafe_cast(getStatus.code())); JLOG(m_journal.error()) << getStatus.ToString(); } diff --git a/src/ripple/nodestore/impl/Database.cpp b/src/ripple/nodestore/impl/Database.cpp index bf28f5bfb..70416c873 100644 --- a/src/ripple/nodestore/impl/Database.cpp +++ b/src/ripple/nodestore/impl/Database.cpp @@ -43,7 +43,8 @@ Database::Database( , earliestLedgerSeq_( get(config, "earliest_seq", XRP_LEDGER_EARLIEST_SEQ)) , earliestShardIndex_((earliestLedgerSeq_ - 1) / ledgersPerShard_) - , readThreads_(std::min(1, readThreads)) + , requestBundle_(get(config, "rq_bundle", 4)) + , readThreads_(std::max(1, readThreads)) { assert(readThreads != 0); @@ -53,29 +54,43 @@ Database::Database( if (earliestLedgerSeq_ < 1) Throw("Invalid earliest_seq"); - for (int i = 0; i != readThreads_.load(); ++i) + if (requestBundle_ < 1 || requestBundle_ > 64) + Throw("Invalid rq_bundle"); + + for (int i = readThreads_.load(); i != 0; --i) { std::thread t( [this](int i) { + runningThreads_++; + beast::setCurrentThreadName( "db prefetch #" + std::to_string(i)); decltype(read_) read; - while (!isStopping()) + while (true) { { std::unique_lock lock(readLock_); + if (isStopping()) + break; + if (read_.empty()) + { + runningThreads_--; readCondVar_.wait(lock); + runningThreads_++; + } if (isStopping()) - continue; + break; - // We extract up to 64 objects to minimize the overhead - // of acquiring the mutex. - for (int cnt = 0; !read_.empty() && cnt != 64; ++cnt) + // extract multiple object at a time to minimize the + // overhead of acquiring the mutex. + for (int cnt = 0; + !read_.empty() && cnt != requestBundle_; + ++cnt) read.insert(read_.extract(read_.begin())); } @@ -84,7 +99,7 @@ Database::Database( assert(!it->second.empty()); auto const& hash = it->first; - auto const& data = std::move(it->second); + auto const& data = it->second; auto const seqn = data[0].first; auto obj = @@ -108,6 +123,7 @@ Database::Database( read.clear(); } + --runningThreads_; --readThreads_; }, i); @@ -148,15 +164,34 @@ Database::maxLedgers(std::uint32_t shardIndex) const noexcept void Database::stop() { - if (!readStopping_.exchange(true, std::memory_order_relaxed)) { std::lock_guard lock(readLock_); - read_.clear(); - readCondVar_.notify_all(); + + if (!readStopping_.exchange(true, std::memory_order_relaxed)) + { + JLOG(j_.debug()) << "Clearing read queue because of stop request"; + read_.clear(); + readCondVar_.notify_all(); + } } + JLOG(j_.debug()) << "Waiting for stop request to complete..."; + + using namespace std::chrono; + + auto const start = steady_clock::now(); + while (readThreads_.load() != 0) + { + assert(steady_clock::now() - start < 30s); std::this_thread::yield(); + } + + JLOG(j_.debug()) << "Stop request completed in " + << duration_cast( + steady_clock::now() - start) + .count() + << " millseconds"; } void @@ -165,10 +200,13 @@ Database::asyncFetch( std::uint32_t ledgerSeq, std::function const&)>&& cb) { - // Post a read std::lock_guard lock(readLock_); - read_[hash].emplace_back(ledgerSeq, std::move(cb)); - readCondVar_.notify_one(); + + if (!isStopping()) + { + read_[hash].emplace_back(ledgerSeq, std::move(cb)); + readCondVar_.notify_one(); + } } void @@ -340,6 +378,16 @@ void Database::getCountsJson(Json::Value& obj) { assert(obj.isObject()); + + { + std::unique_lock lock(readLock_); + obj["read_queue"] = static_cast(read_.size()); + } + + obj["read_threads_total"] = readThreads_.load(); + obj["read_threads_running"] = runningThreads_.load(); + obj["read_request_bundle"] = requestBundle_; + obj[jss::node_writes] = std::to_string(storeCount_); obj[jss::node_reads_total] = std::to_string(fetchTotalCount_); obj[jss::node_reads_hit] = std::to_string(fetchHitCount_); diff --git a/src/ripple/nodestore/impl/DatabaseShardImp.cpp b/src/ripple/nodestore/impl/DatabaseShardImp.cpp index 32efaecdb..1efcbe2ac 100644 --- a/src/ripple/nodestore/impl/DatabaseShardImp.cpp +++ b/src/ripple/nodestore/impl/DatabaseShardImp.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include @@ -799,14 +799,12 @@ DatabaseShardImp::doImportDatabase() std::optional info; if (sortOrder == std::string("asc")) { - info = dynamic_cast( - &app_.getRelationalDBInterface()) + info = dynamic_cast(&app_.getRelationalDatabase()) ->getLimitedOldestLedgerInfo(earliestLedgerSeq()); } else { - info = dynamic_cast( - &app_.getRelationalDBInterface()) + info = dynamic_cast(&app_.getRelationalDatabase()) ->getLimitedNewestLedgerInfo(earliestLedgerSeq()); } if (info) @@ -925,7 +923,7 @@ DatabaseShardImp::doImportDatabase() // Verify SQLite ledgers are in the node store { auto const ledgerHashes{ - app_.getRelationalDBInterface().getHashesByIndex( + app_.getRelationalDatabase().getHashesByIndex( firstSeq, lastSeq)}; if (ledgerHashes.size() != maxLedgers(shardIndex)) continue; @@ -2026,6 +2024,13 @@ DatabaseShardImp::callForLedgerSQLByLedgerSeq( LedgerIndex ledgerSeq, std::function const& callback) { + if (ledgerSeq < earliestLedgerSeq_) + { + JLOG(j_.warn()) << "callForLedgerSQLByLedgerSeq ledger seq too early: " + << ledgerSeq; + return false; + } + return callForLedgerSQLByShardIndex(seqToShardIndex(ledgerSeq), callback); } diff --git a/src/ripple/nodestore/impl/Shard.cpp b/src/ripple/nodestore/impl/Shard.cpp index dc5033c96..030fbf4aa 100644 --- a/src/ripple/nodestore/impl/Shard.cpp +++ b/src/ripple/nodestore/impl/Shard.cpp @@ -19,8 +19,7 @@ #include #include -#include -#include +#include #include #include #include @@ -28,9 +27,6 @@ #include #include -#include -#include - namespace ripple { namespace NodeStore { @@ -227,6 +223,7 @@ Shard::storeNodeObject(std::shared_ptr const& nodeObject) try { + std::lock_guard lock(mutex_); backend_->store(nodeObject); } catch (std::exception const& e) @@ -253,6 +250,7 @@ Shard::fetchNodeObject(uint256 const& hash, FetchReport& fetchReport) Status status; try { + std::lock_guard lock(mutex_); status = backend_->fetch(hash.data(), &nodeObject); } catch (std::exception const& e) @@ -335,6 +333,7 @@ Shard::storeLedger( try { + std::lock_guard lock(mutex_); backend_->storeBatch(batch); } catch (std::exception const& e) @@ -542,6 +541,7 @@ Shard::getWriteLoad() auto const scopedCount{makeBackendCount()}; if (!scopedCount) return 0; + std::lock_guard lock(mutex_); return backend_->getWriteLoad(); } @@ -576,6 +576,8 @@ Shard::finalize(bool writeSQLite, std::optional const& referenceHash) try { + std::lock_guard lock(mutex_); + state_ = ShardState::finalizing; progress_ = 0; @@ -763,8 +765,11 @@ Shard::finalize(bool writeSQLite, std::optional const& referenceHash) try { - // Store final key's value, may already be stored - backend_->store(nodeObject); + { + // Store final key's value, may already be stored + std::lock_guard lock(mutex_); + backend_->store(nodeObject); + } // Do not allow all other threads work with the shard busy_ = true; @@ -823,7 +828,7 @@ Shard::open(std::lock_guard const& lock) using namespace boost::filesystem; Config const& config{app_.config()}; auto preexist{false}; - auto fail = [this, &preexist](std::string const& msg) { + auto fail = [this, &preexist](std::string const& msg) REQUIRES(mutex_) { backend_->close(); lgrSQLiteDB_.reset(); txSQLiteDB_.reset(); @@ -841,7 +846,7 @@ Shard::open(std::lock_guard const& lock) } return false; }; - auto createAcquireInfo = [this, &config]() { + auto createAcquireInfo = [this, &config]() REQUIRES(mutex_) { DatabaseCon::Setup setup; setup.startUp = config.standalone() ? config.LOAD : config.START_UP; setup.standAlone = config.standalone(); @@ -1028,6 +1033,8 @@ Shard::storeSQLite(std::shared_ptr const& ledger) try { + std::lock_guard lock(mutex_); + auto res = updateLedgerDBs( *txSQLiteDB_->checkoutDb(), *lgrSQLiteDB_->checkoutDb(), @@ -1190,6 +1197,7 @@ Shard::verifyFetch(uint256 const& hash) const try { + std::lock_guard lock(mutex_); switch (backend_->fetch(hash.data(), &nodeObject)) { case ok: diff --git a/src/ripple/nodestore/impl/Shard.h b/src/ripple/nodestore/impl/Shard.h index 17001a6b8..210bdd54a 100644 --- a/src/ripple/nodestore/impl/Shard.h +++ b/src/ripple/nodestore/impl/Shard.h @@ -21,11 +21,12 @@ #define RIPPLE_NODESTORE_SHARD_H_INCLUDED #include -#include +#include #include #include #include #include +#include #include #include #include @@ -210,6 +211,7 @@ public: std::string getStoredSeqs() { + std::lock_guard lock(mutex_); if (!acquireInfo_) return ""; @@ -316,29 +318,30 @@ private: boost::filesystem::path const dir_; // Storage space utilized by the shard - std::uint64_t fileSz_{0}; + GUARDED_BY(mutex_) std::uint64_t fileSz_{0}; // Number of file descriptors required by the shard - std::uint32_t fdRequired_{0}; + GUARDED_BY(mutex_) std::uint32_t fdRequired_{0}; // NuDB key/value store for node objects - std::unique_ptr backend_; + std::unique_ptr backend_ GUARDED_BY(mutex_); std::atomic backendCount_{0}; // Ledger SQLite database used for indexes - std::unique_ptr lgrSQLiteDB_; + std::unique_ptr lgrSQLiteDB_ GUARDED_BY(mutex_); // Transaction SQLite database used for indexes - std::unique_ptr txSQLiteDB_; + std::unique_ptr txSQLiteDB_ GUARDED_BY(mutex_); // Tracking information used only when acquiring a shard from the network. // If the shard is finalized, this member will be null. - std::unique_ptr acquireInfo_; + std::unique_ptr acquireInfo_ GUARDED_BY(mutex_); + ; // Older shard without an acquire database or final key // Eventually there will be no need for this and should be removed - bool legacy_{false}; + GUARDED_BY(mutex_) bool legacy_{false}; // Determines if the shard needs to stop processing for shutdown std::atomic stop_{false}; @@ -356,16 +359,17 @@ private: std::atomic removeOnDestroy_{false}; // The time of the last access of a shard with a finalized state - std::chrono::steady_clock::time_point lastAccess_; + std::chrono::steady_clock::time_point lastAccess_ GUARDED_BY(mutex_); + ; // Open shard databases [[nodiscard]] bool - open(std::lock_guard const& lock); + open(std::lock_guard const& lock) REQUIRES(mutex_); // Open/Create SQLite databases // Lock over mutex_ required [[nodiscard]] bool - initSQLite(std::lock_guard const&); + initSQLite(std::lock_guard const&) REQUIRES(mutex_); // Write SQLite entries for this ledger [[nodiscard]] bool @@ -374,7 +378,7 @@ private: // Set storage and file descriptor usage stats // Lock over mutex_ required void - setFileStats(std::lock_guard const&); + setFileStats(std::lock_guard const&) REQUIRES(mutex_); // Verify this ledger by walking its SHAMaps and verifying its Merkle trees // Every node object verified will be stored in the deterministic shard diff --git a/src/ripple/overlay/README.md b/src/ripple/overlay/README.md index 289c5f707..8be890ef7 100644 --- a/src/ripple/overlay/README.md +++ b/src/ripple/overlay/README.md @@ -296,8 +296,8 @@ For more on the Peer Crawler, please visit https://xrpl.org/peer-crawler.html. If present, identifies the hash of the last ledger that the sending server considers to be closed. -The value is presently encoded using **Base64** encoding, but implementations -should support both **Base64** and **HEX** encoding for this value. +The value is encoded as **HEX**, but implementations should support both +**Base64** and **HEX** encoding for this value for legacy purposes. | Field Name | Request | Response | |--------------------- |:-----------------: |:-----------------: | diff --git a/src/ripple/overlay/Slot.h b/src/ripple/overlay/Slot.h index b7a2129ed..1197eff56 100644 --- a/src/ripple/overlay/Slot.h +++ b/src/ripple/overlay/Slot.h @@ -494,16 +494,11 @@ template std::set Slot::getSelected() const { - std::set init; - return std::accumulate( - peers_.begin(), peers_.end(), init, [](auto& init, auto const& it) { - if (it.second.state == PeerState::Selected) - { - init.insert(it.first); - return init; - } - return init; - }); + std::set r; + for (auto const& [id, info] : peers_) + if (info.state == PeerState::Selected) + r.insert(id); + return r; } template @@ -513,20 +508,20 @@ std::unordered_map< Slot::getPeers() const { using namespace std::chrono; - auto init = std::unordered_map< + auto r = std::unordered_map< id_t, std::tuple>(); - return std::accumulate( - peers_.begin(), peers_.end(), init, [](auto& init, auto const& it) { - init.emplace(std::make_pair( - it.first, - std::move(std::make_tuple( - it.second.state, - it.second.count, - epoch(it.second.expire).count(), - epoch(it.second.lastMessage).count())))); - return init; - }); + + for (auto const& [id, info] : peers_) + r.emplace(std::make_pair( + id, + std::move(std::make_tuple( + info.state, + info.count, + epoch(info.expire).count(), + epoch(info.lastMessage).count())))); + + return r; } /** Slots is a container for validator's Slot and handles Slot update diff --git a/src/ripple/overlay/impl/Handshake.cpp b/src/ripple/overlay/impl/Handshake.cpp index 2ea208f55..793dec19e 100644 --- a/src/ripple/overlay/impl/Handshake.cpp +++ b/src/ripple/overlay/impl/Handshake.cpp @@ -204,6 +204,8 @@ buildHandshake( h.insert("Session-Signature", base64_encode(sig.data(), sig.size())); } + h.insert("Instance-Cookie", std::to_string(app.instanceID())); + if (!app.config().SERVER_DOMAIN.empty()) h.insert("Server-Domain", app.config().SERVER_DOMAIN); @@ -215,14 +217,8 @@ buildHandshake( if (auto const cl = app.getLedgerMaster().getClosedLedger()) { - // TODO: Use hex for these - h.insert( - "Closed-Ledger", - base64_encode(cl->info().hash.begin(), cl->info().hash.size())); - h.insert( - "Previous-Ledger", - base64_encode( - cl->info().parentHash.begin(), cl->info().parentHash.size())); + h.insert("Closed-Ledger", strHex(cl->info().hash)); + h.insert("Previous-Ledger", strHex(cl->info().parentHash)); } } @@ -306,7 +302,34 @@ verifyHandshake( }(); if (publicKey == app.nodeIdentity().first) + { + auto const peerInstanceID = [&headers]() { + std::uint64_t iid = 0; + + if (auto const iter = headers.find("Instance-Cookie"); + iter != headers.end()) + { + if (!beast::lexicalCastChecked(iid, iter->value().to_string())) + throw std::runtime_error("Invalid instance cookie"); + + if (iid == 0) + throw std::runtime_error("Invalid instance cookie"); + } + + return iid; + }(); + + // Attempt to differentiate self-connections as opposed to accidental + // node identity reuse caused by accidental misconfiguration. When we + // detect this, we stop the process and log an error message. + if (peerInstanceID != app.instanceID()) + { + app.signalStop("Remote server is using our node identity"); + throw std::runtime_error("Node identity reuse detected"); + } + throw std::runtime_error("Self connection"); + } // This check gets two birds with one stone: // diff --git a/src/ripple/overlay/impl/OverlayImpl.cpp b/src/ripple/overlay/impl/OverlayImpl.cpp index 47e03a76b..6ed046f04 100644 --- a/src/ripple/overlay/impl/OverlayImpl.cpp +++ b/src/ripple/overlay/impl/OverlayImpl.cpp @@ -22,8 +22,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include diff --git a/src/ripple/overlay/impl/OverlayImpl.h b/src/ripple/overlay/impl/OverlayImpl.h index 5f23b9150..2ba7999cb 100644 --- a/src/ripple/overlay/impl/OverlayImpl.h +++ b/src/ripple/overlay/impl/OverlayImpl.h @@ -125,8 +125,6 @@ private: // Peer IDs expecting to receive a last link notification std::set csIDs_; - std::optional networkID_; - reduce_relay::Slots slots_; // Transaction reduce-relay metrics @@ -391,7 +389,7 @@ public: std::optional networkID() const override { - return networkID_; + return setup_.networkID; } Json::Value diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index bc379c147..8e014a10f 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -642,11 +642,6 @@ PeerImp::gracefulClose() assert(socket_.is_open()); assert(!gracefulClose_); gracefulClose_ = true; -#if 0 - // Flush messages - while(send_queue_.size() > 1) - send_queue_.pop_back(); -#endif if (send_queue_.size() > 0) return; setTimer(); @@ -2119,7 +2114,7 @@ PeerImp::onMessage(std::shared_ptr const& m) m->ledgerseq(), app_.getLedgerMaster().getValidLedgerIndex()); } - app_.getOPs().pubPeerStatus([=]() -> Json::Value { + app_.getOPs().pubPeerStatus([=, this]() -> Json::Value { Json::Value j = Json::objectValue; if (m->has_newstatus()) @@ -3404,7 +3399,7 @@ PeerImp::getLedger(std::shared_ptr const& m) } else { - JLOG(p_journal_.warn()) << "getLedger: Unable to find ledger"; + JLOG(p_journal_.debug()) << "getLedger: Unable to find ledger"; } return ledger; diff --git a/src/ripple/overlay/impl/PeerReservationTable.cpp b/src/ripple/overlay/impl/PeerReservationTable.cpp index 6e88da123..6f39d12e9 100644 --- a/src/ripple/overlay/impl/PeerReservationTable.cpp +++ b/src/ripple/overlay/impl/PeerReservationTable.cpp @@ -19,8 +19,8 @@ #include -#include -#include +#include +#include #include #include #include diff --git a/src/ripple/overlay/impl/ProtocolVersion.cpp b/src/ripple/overlay/impl/ProtocolVersion.cpp index 4931dacb4..9a549b563 100644 --- a/src/ripple/overlay/impl/ProtocolVersion.cpp +++ b/src/ripple/overlay/impl/ProtocolVersion.cpp @@ -36,7 +36,6 @@ namespace ripple { // clang-format off constexpr ProtocolVersion const supportedProtocolList[] { - {2, 0}, {2, 1}, {2, 2} }; diff --git a/src/ripple/peerfinder/impl/Logic.h b/src/ripple/peerfinder/impl/Logic.h index 4e69c8bd3..ca14a5111 100644 --- a/src/ripple/peerfinder/impl/Logic.h +++ b/src/ripple/peerfinder/impl/Logic.h @@ -782,10 +782,14 @@ public: // Must be handshaked! assert(slot->state() == Slot::active); - preprocess(slot, list); - clock_type::time_point const now(m_clock.now()); + // Limit how often we accept new endpoints + if (slot->whenAcceptEndpoints > now) + return; + + preprocess(slot, list); + for (auto const& ep : list) { assert(ep.hops != 0); diff --git a/src/ripple/peerfinder/impl/StoreSqdb.h b/src/ripple/peerfinder/impl/StoreSqdb.h index 7d43136ae..879bee83b 100644 --- a/src/ripple/peerfinder/impl/StoreSqdb.h +++ b/src/ripple/peerfinder/impl/StoreSqdb.h @@ -20,7 +20,7 @@ #ifndef RIPPLE_PEERFINDER_STORESQDB_H_INCLUDED #define RIPPLE_PEERFINDER_STORESQDB_H_INCLUDED -#include +#include #include #include #include diff --git a/src/ripple/proto/org/xrpl/rpc/v1/account.proto b/src/ripple/proto/org/xrpl/rpc/v1/account.proto deleted file mode 100644 index 40e9a7a4f..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/account.proto +++ /dev/null @@ -1,14 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -// A representation of an account address -// Next field: 2 -message AccountAddress -{ - // base58 encoding of an account - string address = 1; -} - diff --git a/src/ripple/proto/org/xrpl/rpc/v1/amount.proto b/src/ripple/proto/org/xrpl/rpc/v1/amount.proto deleted file mode 100644 index 64ea2f641..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/amount.proto +++ /dev/null @@ -1,48 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -import "org/xrpl/rpc/v1/account.proto"; - -// Next field: 3 -message CurrencyAmount -{ - oneof amount - { - XRPDropsAmount xrp_amount = 1; - IssuedCurrencyAmount issued_currency_amount = 2; - } -} - -// A representation of an amount of XRP. -// Next field: 2 -message XRPDropsAmount -{ - uint64 drops = 1 [jstype=JS_STRING]; -} - -// A representation of an amount of issued currency. -// Next field: 4 -message IssuedCurrencyAmount -{ - // The currency used to value the amount. - Currency currency = 1; - - // The value of the amount. 8 bytes - string value = 2; - - // Unique account address of the entity issuing the currency. - AccountAddress issuer = 3; -} - -// Next field: 3 -message Currency -{ - // 3 character ASCII code - string name = 1; - - // 160 bit currency code. 20 bytes - bytes code = 2; -} diff --git a/src/ripple/proto/org/xrpl/rpc/v1/common.proto b/src/ripple/proto/org/xrpl/rpc/v1/common.proto deleted file mode 100644 index 81718b507..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/common.proto +++ /dev/null @@ -1,598 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -import "org/xrpl/rpc/v1/amount.proto"; -import "org/xrpl/rpc/v1/account.proto"; - -// These fields are used in many different message types. They can be present -// in one or more transactions, as well as metadata of one or more transactions. -// Each is defined as its own message type with a single field "value", to -// ensure the field is the correct type everywhere it's used - - -// *** Messages wrapping uint32 *** - -message BurnedNFTokens -{ - uint32 value = 1; -} - -message CancelAfter -{ - // time in seconds since Ripple epoch - uint32 value = 1; -} - -message ClearFlag -{ - uint32 value = 1; -} - -message CloseTime -{ - // time in seconds since Ripple epoch - uint32 value = 1; -} - -message Date -{ - // time in seconds since Ripple epoch - uint32 value = 1; -} - -message DestinationTag -{ - uint32 value = 1; -} - -message Expiration -{ - // time in seconds since Ripple epoch - uint32 value = 1; -} - -message FinishAfter -{ - // time in seconds since Ripple epoch - uint32 value = 1; -} - -message Flags -{ - uint32 value = 1; -} - -message HighQualityIn -{ - uint32 value = 1; -} - -message HighQualityOut -{ - uint32 value = 1; -} - -message FirstLedgerSequence -{ - uint32 value = 1; -} - -message LastLedgerSequence -{ - uint32 value = 1; -} - -message LowQualityIn -{ - uint32 value = 1; -} - -message LowQualityOut -{ - uint32 value = 1; -} - -message MintedNFTokens -{ - uint32 value = 1; -} - -message OfferSequence -{ - uint32 value = 1; -} - -message OwnerCount -{ - uint32 value = 1; -} - -message PreviousTransactionLedgerSequence -{ - uint32 value = 1; -} - -message QualityIn -{ - uint32 value = 1; -} - -message QualityOut -{ - uint32 value = 1; -} - -message ReferenceFeeUnits -{ - uint32 value = 1; -} - -message ReserveBase -{ - // in drops - uint32 value = 1; -} - -message ReserveIncrement -{ - // in drops - uint32 value = 1; -} - -message Sequence -{ - uint32 value = 1; -} - -message SetFlag -{ - uint32 value = 1; -} - -message SettleDelay -{ - uint32 value = 1; -} - -message SignerListID -{ - uint32 value = 1; -} - -message SignerQuorum -{ - uint32 value = 1; -} - -message SignerWeight -{ - // is actually uint16 - uint32 value = 1; -} - -message SourceTag -{ - uint32 value = 1; -} - -message TickSize -{ - // is actually uint8 - uint32 value = 1; -} - -message Ticket -{ - uint32 value = 1; -} - -message TicketCount -{ - uint32 value = 1; -} - -message TicketSequence -{ - uint32 value = 1; -} - -message NFTokenTaxon -{ - uint32 value = 1; -} - -message TransferFee -{ - // is actually uint16 - uint32 value = 1; -} - -message TransferRate -{ - uint32 value = 1; -} - - -// *** Messages wrapping uint64 *** - -message BaseFee -{ - // in drops - uint64 value = 1 [jstype=JS_STRING]; -} - -message BookNode -{ - uint64 value = 1 [jstype=JS_STRING]; -} - -message DestinationNode -{ - uint64 value = 1 [jstype=JS_STRING]; -} - -message HighNode -{ - uint64 value = 1 [jstype=JS_STRING]; -} - -message IndexNext -{ - uint64 value = 1 [jstype=JS_STRING]; -} - -message IndexPrevious -{ - uint64 value = 1 [jstype=JS_STRING]; -} - -message LowNode -{ - uint64 value = 1 [jstype=JS_STRING]; -} - -message NFTokenOfferNode -{ - uint64 value = 1 [jstype=JS_STRING]; -} - -message OwnerNode -{ - uint64 value = 1 [jstype=JS_STRING]; -} - - -// *** Messages wrapping 16 bytes *** - -message EmailHash -{ - bytes value = 1; -} - -message NFTokenID -{ - bytes value = 1; -} - - -// *** Messages wrapping 20 bytes *** - -message TakerGetsIssuer -{ - // 20 bytes - bytes value = 1; -} - -message TakerPaysIssuer -{ - // 20 bytes - bytes value = 1; -} - - -// *** Messages wrapping 32 bytes *** - -message AccountTransactionID -{ - // 32 bytes - bytes value = 1; -} - -message BookDirectory -{ - // 32 btes - bytes value = 1; -} - -message Channel -{ - // 32 bytes - bytes value = 1; -} - -message CheckID -{ - // 32 bytes - bytes value = 1; -} - -message Hash -{ - // 32 bytes - bytes value = 1; -} - -message Index -{ - // 32 bytes - bytes value = 1; -} - -message InvoiceID -{ - // 32 bytes - bytes value = 1; -} - -message NextPageMin -{ - // 32 bytes - bytes value = 1; -} - -message NFTokenBuyOffer -{ - // 32 bytes - bytes value = 1; -} - -message NFTokenSellOffer -{ - // 32 bytes - bytes value = 1; -} - -message PreviousPageMin -{ - // 32 bytes - bytes value = 1; -} - -message PreviousTransactionID -{ - // 32 bytes - bytes value = 1; -} - -message RootIndex -{ - // 32 bytes - bytes value = 1; -} - - -// *** Messages wrapping variable length byte arrays *** - -message Condition -{ - bytes value = 1; -} - -message Fulfillment -{ - bytes value = 1; -} - -message MemoData -{ - bytes value = 1; -} - -message MemoFormat -{ - bytes value = 1; -} - -message MemoType -{ - bytes value = 1; -} - -message MessageKey -{ - bytes value = 1; -} - -message PublicKey -{ - bytes value = 1; -} - -message PaymentChannelSignature -{ - bytes value = 1; -} - -message SigningPublicKey -{ - bytes value = 1; -} - -message TransactionSignature -{ - bytes value = 1; -} - -message ValidatorToDisable -{ - bytes value = 1; -} - -message ValidatorToReEnable -{ - bytes value = 1; -} - -// *** Messages wrapping a Currency value *** -// -// TODO: if there's a V2 of the API, fix this misspelling. -message TakerGetsCurreny -{ - Currency value = 1; -} - -message TakerPaysCurrency -{ - Currency value = 1; -} - - -// *** Messages wrapping a CurrencyAmount *** - -message Amount -{ - // Note, CurrencyAmount is a oneof, that can represent an XRP drops amount - // or an Issued Currency amount. However, in some transaction types/ledger - // objects, this value can only be in drops. For instance, the Amount field - // of a Payment transaction can be specified in XRP drops or an Issued - // Currency amount, but the Amount field of a PaymentChannelClaim - // transaction can only be an XRP drops amount. - CurrencyAmount value = 1; -} - -message Balance -{ - CurrencyAmount value = 1; -} - -message NFTokenBrokerFee -{ - CurrencyAmount value = 1; -} - -message DeliverMin -{ - CurrencyAmount value = 1; -} - -message DeliveredAmount -{ - CurrencyAmount value = 1; -} - -message HighLimit -{ - CurrencyAmount value = 1; -} - -message LimitAmount -{ - CurrencyAmount value = 1; -} - -message LowLimit -{ - CurrencyAmount value = 1; -} - -message SendMax -{ - CurrencyAmount value = 1; -} - -message TakerGets -{ - CurrencyAmount value = 1; -} - -message TakerPays -{ - CurrencyAmount value = 1; -} - - -// *** Messages wrapping an AccountAddress *** - -message Account -{ - AccountAddress value = 1; -} - -message Authorize -{ - AccountAddress value = 1; -} - -message Destination -{ - AccountAddress value = 1; -} - -message Issuer -{ - AccountAddress value = 1; -} - -message NFTokenMinter -{ - AccountAddress value = 1; -} - -message Owner -{ - AccountAddress value = 1; -} - -message RegularKey -{ - AccountAddress value = 1; -} - -message Unauthorize -{ - AccountAddress value = 1; -} - - -// *** Messages wrapping a string *** - -message Domain -{ - string value = 1; -} - -message URI -{ - string value = 1; -} - - -// *** Aggregate type messages - -// Next field: 3 -message NFToken -{ - NFTokenID nftoken_id = 1; - - URI uri = 2; -} - -// Next field: 3 -message SignerEntry -{ - Account account = 1; - - SignerWeight signer_weight = 2; -} - -// Next field: 3 -message DisabledValidator -{ - PublicKey public_key = 1; - - FirstLedgerSequence ledger_sequence = 2; -} - diff --git a/src/ripple/proto/org/xrpl/rpc/v1/get_account_info.proto b/src/ripple/proto/org/xrpl/rpc/v1/get_account_info.proto deleted file mode 100644 index bc23a8c66..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/get_account_info.proto +++ /dev/null @@ -1,93 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -import "org/xrpl/rpc/v1/ledger_objects.proto"; -import "org/xrpl/rpc/v1/amount.proto"; -import "org/xrpl/rpc/v1/account.proto"; -import "org/xrpl/rpc/v1/ledger.proto"; -import "org/xrpl/rpc/v1/common.proto"; - -// A request to get info about an account. -// Next field: 6 -message GetAccountInfoRequest -{ - // The address to get info about. - AccountAddress account = 1; - - bool strict = 2; - - // Which ledger to use to retrieve data. - // If this field is not set, the server will use the open ledger. - // The open ledger includes data that is not validated or final. - // To retrieve the most up to date and validated data, use - // SHORTCUT_VALIDATED - LedgerSpecifier ledger = 3; - - bool queue = 4; - - bool signer_lists = 5; - - string client_ip = 6; -} - -// Response to GetAccountInfo RPC -// Next field: 6 -message GetAccountInfoResponse -{ - AccountRoot account_data = 1; - - SignerList signer_list = 2; - - uint32 ledger_index = 3; - - QueueData queue_data = 4; - - bool validated = 5; -} - -// Aggregate data about queued transactions -// Next field: 11 -message QueueData -{ - uint32 txn_count = 1; - - bool auth_change_queued = 2; - - uint32 lowest_sequence = 3; - - uint32 highest_sequence = 4; - - XRPDropsAmount max_spend_drops_total = 5; - - repeated QueuedTransaction transactions = 6; - - uint32 lowest_ticket = 7; - - uint32 highest_ticket = 8; - - uint32 sequence_count = 9; - - uint32 ticket_count = 10; -} - -// Data about a single queued transaction -// Next field: 8 -message QueuedTransaction -{ - bool auth_change = 1; - - XRPDropsAmount fee = 2; - - uint64 fee_level = 3 [jstype=JS_STRING]; - - XRPDropsAmount max_spend_drops = 4; - - Sequence sequence = 5; - - LastLedgerSequence last_ledger_sequence = 6; - - Ticket ticket = 7; -} diff --git a/src/ripple/proto/org/xrpl/rpc/v1/get_account_transaction_history.proto b/src/ripple/proto/org/xrpl/rpc/v1/get_account_transaction_history.proto deleted file mode 100644 index c4889a6bd..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/get_account_transaction_history.proto +++ /dev/null @@ -1,75 +0,0 @@ -syntax = "proto3"; - -import "org/xrpl/rpc/v1/get_transaction.proto"; -import "org/xrpl/rpc/v1/account.proto"; -import "org/xrpl/rpc/v1/ledger.proto"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -// Next field: 8 -message GetAccountTransactionHistoryRequest -{ - AccountAddress account = 1; - - // What ledger to include results from. Specifying a not yet validated - // ledger results in an error. Not specifying a ledger uses the entire - // range of validated ledgers available to the server. - // Note, this parameter acts as a filter, and can only reduce the number of - // results. Specifying a single ledger will return only transactions from - // that ledger. This includes specifying a ledger with a Shortcut. For - // example, specifying SHORTCUT_VALIDATED will result in only transactions - // that were part of the most recently validated ledger being returned. - // Specifying a range of ledgers results in only transactions that were - // included in a ledger within the specified range being returned. - oneof ledger - { - LedgerSpecifier ledger_specifier = 2; - LedgerRange ledger_range = 3; - }; - - // Return results as binary blobs. Defaults to false. - bool binary = 4; - - // If set to true, returns values indexed by older ledger first. - // Default to false. - bool forward = 5; - - // Limit the number of results. Server may choose a lower limit. - // If this value is 0, the limit is ignored and the number of results - // returned is determined by the server - uint32 limit = 6; - - // Marker to resume where previous request left off - // Used for pagination - Marker marker = 7; -} - - -// Next field: 8 -message GetAccountTransactionHistoryResponse -{ - AccountAddress account = 1; - - uint32 ledger_index_min = 2; - - uint32 ledger_index_max = 3; - - uint32 limit = 4; - - Marker marker = 5; - - repeated GetTransactionResponse transactions = 6; - - bool validated = 7; -} - -// Next field: 3 -message Marker -{ - uint32 ledger_index = 1; - - uint32 account_sequence = 2; -} - diff --git a/src/ripple/proto/org/xrpl/rpc/v1/get_fee.proto b/src/ripple/proto/org/xrpl/rpc/v1/get_fee.proto deleted file mode 100644 index aec3f2413..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/get_fee.proto +++ /dev/null @@ -1,58 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -import "org/xrpl/rpc/v1/amount.proto"; - -// A request for the current transaction fee on the ledger. -// Next field: 1 -message GetFeeRequest -{ - string client_ip = 1; -} - -// Response to a GetFee RPC -// Next field: 8 -message GetFeeResponse -{ - uint64 current_ledger_size = 1 [jstype=JS_STRING]; - - uint64 current_queue_size = 2 [jstype=JS_STRING]; - - Fee fee = 3; - - uint64 expected_ledger_size = 4 [jstype=JS_STRING]; - - uint32 ledger_current_index = 5; - - FeeLevels levels = 6; - - uint64 max_queue_size = 7 [jstype=JS_STRING]; - -} - -// Next field: 5 -message Fee -{ - XRPDropsAmount base_fee = 1; - - XRPDropsAmount median_fee = 2; - - XRPDropsAmount minimum_fee = 3; - - XRPDropsAmount open_ledger_fee = 4; -} - -// Next field: 5 -message FeeLevels -{ - uint64 median_level = 1 [jstype=JS_STRING]; - - uint64 minimum_level = 2 [jstype=JS_STRING]; - - uint64 open_ledger_level = 3 [jstype=JS_STRING]; - - uint64 reference_level = 4 [jstype=JS_STRING]; -} diff --git a/src/ripple/proto/org/xrpl/rpc/v1/get_transaction.proto b/src/ripple/proto/org/xrpl/rpc/v1/get_transaction.proto deleted file mode 100644 index 93ba48a11..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/get_transaction.proto +++ /dev/null @@ -1,62 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -import "org/xrpl/rpc/v1/meta.proto"; -import "org/xrpl/rpc/v1/ledger.proto"; -import "org/xrpl/rpc/v1/transaction.proto"; -import "org/xrpl/rpc/v1/common.proto"; - -// Next field: 4 -message GetTransactionRequest { - - // hash of the transaction. 32 bytes - // ATTN: this is in binary, not hex. The JSON API accepts a hex string for - // a transaction hash, but here we need that hex string converted into its - // binary form. Each pair of hex characters should be converted into its - // corresponding byte. For example, the 4 character hex string "00FF" - // should be converted to a 2 byte array: [0, 255] - bytes hash = 1; - - // if true, return data in binary format. defaults to false - bool binary = 2; - - // If the transaction was not found, server will report whether the entire - // specified range was searched. The value is contained in the error message. - // The error message is of the form: - // "txn not found. searched_all = [true,false]" - // If the transaction was found, this parameter is ignored. - LedgerRange ledger_range = 3; - - string client_ip = 4; -} - -// Next field: 9 -message GetTransactionResponse { - - oneof serialized_transaction { - - Transaction transaction = 1; - // Variable length - bytes transaction_binary = 2; - }; - // Sequence number of ledger that contains this transaction - uint32 ledger_index = 3; - - // 32 bytes - bytes hash = 4; - - // whether the ledger has been validated - bool validated = 5; - - // metadata about the transaction - oneof serialized_meta { - Meta meta = 6; - // Variable length - bytes meta_binary = 7; - } - - Date date = 8; -} diff --git a/src/ripple/proto/org/xrpl/rpc/v1/ledger.proto b/src/ripple/proto/org/xrpl/rpc/v1/ledger.proto index 78e4211f5..3bb199de2 100644 --- a/src/ripple/proto/org/xrpl/rpc/v1/ledger.proto +++ b/src/ripple/proto/org/xrpl/rpc/v1/ledger.proto @@ -25,16 +25,6 @@ message LedgerSpecifier } } -// Next field: 3 -message LedgerRange -{ - uint32 ledger_index_min = 1; - - // Note, if ledger_index_min is non-zero and ledger_index_max is 0, the - // software will use the max validated ledger in place of ledger_index_max - uint32 ledger_index_max = 2; -}; - // Next field: 3 message RawLedgerObject diff --git a/src/ripple/proto/org/xrpl/rpc/v1/ledger_objects.proto b/src/ripple/proto/org/xrpl/rpc/v1/ledger_objects.proto deleted file mode 100644 index d6db469a2..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/ledger_objects.proto +++ /dev/null @@ -1,418 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -import "org/xrpl/rpc/v1/common.proto"; - -// Next field: 17 -message LedgerObject -{ - oneof object - { - AccountRoot account_root = 1; - Amendments amendments = 2; - Check check = 3; - DepositPreauthObject deposit_preauth = 4; - DirectoryNode directory_node = 5; - Escrow escrow = 6; - FeeSettings fee_settings = 7; - LedgerHashes ledger_hashes = 8; - NFTokenOffer nftoken_offer = 15; - NFTokenPage nftoken_page = 16; - Offer offer = 9; - PayChannel pay_channel = 10; - RippleState ripple_state = 11; - SignerList signer_list = 12; - NegativeUNL negative_unl = 13; - TicketObject ticket = 14; - } -} - -// Next field: 15 -enum LedgerEntryType -{ - LEDGER_ENTRY_TYPE_UNSPECIFIED = 0; - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT = 1; - LEDGER_ENTRY_TYPE_AMENDMENTS = 2; - LEDGER_ENTRY_TYPE_CHECK = 3; - LEDGER_ENTRY_TYPE_DEPOSIT_PREAUTH = 4; - LEDGER_ENTRY_TYPE_DIRECTORY_NODE = 5; - LEDGER_ENTRY_TYPE_ESCROW = 6; - LEDGER_ENTRY_TYPE_FEE_SETTINGS = 7; - LEDGER_ENTRY_TYPE_LEDGER_HASHES = 8; - LEDGER_ENTRY_TYPE_OFFER = 9; - LEDGER_ENTRY_TYPE_PAY_CHANNEL = 10; - LEDGER_ENTRY_TYPE_RIPPLE_STATE = 11; - LEDGER_ENTRY_TYPE_SIGNER_LIST = 12; - LEDGER_ENTRY_TYPE_NEGATIVE_UNL = 13; - LEDGER_ENTRY_TYPE_TICKET = 14; - LEDGER_ENTRY_TYPE_NFTOKEN_OFFER = 15; - LEDGER_ENTRY_TYPE_NFTOKEN_PAGE = 16; -} - -// Next field: 19 -message AccountRoot -{ - Account account = 1; - - Balance balance = 2; - - BurnedNFTokens burned_nftokens = 16; - - Sequence sequence = 3; - - Flags flags = 4; - - OwnerCount owner_count = 5; - - PreviousTransactionID previous_transaction_id = 6; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 7; - - AccountTransactionID account_transaction_id = 8; - - Domain domain = 9; - - EmailHash email_hash = 10; - - MessageKey message_key = 11; - - MintedNFTokens minted_nftokens = 17; - - NFTokenMinter nftoken_minter = 18; - - RegularKey regular_key = 12; - - TickSize tick_size = 13; - - TicketCount ticket_count = 15; - - TransferRate transfer_rate = 14; -} - -// Next field: 4 -message Amendments -{ - // Next field: 2 - message Amendment - { - // 32 bytes - bytes value = 1; - } - - // Next field: 3 - message Majority - { - Amendment amendment = 1; - - CloseTime close_time = 2; - } - - repeated Amendment amendments = 1; - - repeated Majority majorities = 2; - - Flags flags = 3; -} - -// Next field: 14 -message Check -{ - Account account = 1; - - Destination destination = 2; - - Flags flags = 3; - - OwnerNode owner_node = 4; - - PreviousTransactionID previous_transaction_id = 5; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 6; - - SendMax send_max = 7; - - Sequence sequence = 8; - - DestinationNode destination_node = 9; - - DestinationTag destination_tag = 10; - - Expiration expiration = 11; - - InvoiceID invoice_id = 12; - - SourceTag source_tag = 13; -} - -// Next field: 7 -message DepositPreauthObject -{ - Account account = 1; - - Authorize authorize = 2; - - Flags flags = 3; - - OwnerNode owner_node = 4; - - PreviousTransactionID previous_transaction_id = 5; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 6; -} - -// Next field: 12 -message DirectoryNode -{ - Flags flags = 1; - - RootIndex root_index = 2; - - repeated Index indexes = 3; - - IndexNext index_next = 4; - - IndexPrevious index_previous = 5; - - Owner owner = 6; - - TakerPaysCurrency taker_pays_currency = 7; - - TakerPaysIssuer taker_pays_issuer = 8; - - TakerGetsCurreny taker_gets_currency = 9; - - TakerGetsIssuer taker_gets_issuer = 10; - - NFTokenID nftoken_id = 11; -} - -// Next field: 14 -message Escrow -{ - Account account = 1; - - Destination destination = 2; - - Amount amount = 3; - - Condition condition = 4; - - CancelAfter cancel_after = 5; - - FinishAfter finish_after = 6; - - Flags flags = 7; - - SourceTag source_tag = 8; - - DestinationTag destination_tag = 9; - - OwnerNode owner_node = 10; - - DestinationNode destination_node = 11; - - PreviousTransactionID previous_transaction_id = 12; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 13; -} - -// Next field: 6 -message FeeSettings -{ - BaseFee base_fee = 1; - - ReferenceFeeUnits reference_fee_units = 2; - - ReserveBase reserve_base = 3; - - ReserveIncrement reserve_increment = 4; - - Flags flags = 5; -} - -// Next field: 4 -message LedgerHashes -{ - LastLedgerSequence last_ledger_sequence = 1; - - repeated Hash hashes = 2; - - Flags flags = 3; -} - -// Next field: 12 -message Offer -{ - Account account = 1; - - Sequence sequence = 2; - - Flags flags = 3; - - TakerPays taker_pays = 4; - - TakerGets taker_gets = 5; - - BookDirectory book_directory = 6; - - BookNode book_node = 7; - - OwnerNode owner_node = 8; - - Expiration expiration = 9; - - PreviousTransactionID previous_transaction_id = 10; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 11; -} - -// Next field: 11 -message NFTokenOffer -{ - Flags flags = 1; - - Owner owner = 2; - - NFTokenID nftoken_id = 3; - - Amount amount = 4; - - OwnerNode owner_node = 5; - - NFTokenOfferNode nftoken_offer_node = 6; - - Destination destination = 7; - - Expiration expiration = 8; - - PreviousTransactionID previous_transaction_id = 9; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 10; -} - -// Next field: 7 -message NFTokenPage -{ - Flags flags = 1; - - PreviousPageMin previous_page_min = 2; - - NextPageMin next_page_min = 3; - - repeated NFToken nftokens = 4; - - PreviousTransactionID previous_transaction_id = 5; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 6; -} - -// Next field: 13 -message PayChannel -{ - Account account = 1; - - Destination destination = 2; - - Amount amount = 3; - - Balance balance = 4; - - PublicKey public_key = 5; - - SettleDelay settle_delay = 6; - - OwnerNode owner_node = 7; - - PreviousTransactionID previous_transaction_id = 8; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 9; - - Flags flags = 10; - - Expiration expiration = 11; - - CancelAfter cancel_after = 12; - - SourceTag source_tag = 13; - - DestinationTag destination_tag = 14; - - DestinationNode destination_node = 15; -} - -// Next field: 13 -message RippleState -{ - Balance balance = 1; - - Flags flags = 2; - - LowLimit low_limit = 3; - - HighLimit high_limit = 4; - - LowNode low_node = 5; - - HighNode high_node = 6; - - LowQualityIn low_quality_in = 7; - - LowQualityOut low_quality_out = 8; - - HighQualityIn high_quality_in = 9; - - HighQualityOut high_quality_out = 10; - - PreviousTransactionID previous_transaction_id = 11; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 12; -} - -// Next field: 8 -message SignerList -{ - Flags flags = 1; - - PreviousTransactionID previous_transaction_id = 2; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 3; - - OwnerNode owner_node = 4; - - repeated SignerEntry signer_entries = 5; - - SignerListID signer_list_id = 6; - - SignerQuorum signer_quorum = 7; -} - -// Next field: 7 -message TicketObject -{ - Flags flags = 1; - - Account account = 2; - - OwnerNode owner_node = 3; - - PreviousTransactionID previous_transaction_id = 4; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 5; - - TicketSequence ticket_sequence = 6; -} - -// Next field: 5 -message NegativeUNL -{ - repeated DisabledValidator disabled_validators = 1; - - ValidatorToDisable validator_to_disable = 2; - - ValidatorToReEnable validator_to_re_enable = 3; - - Flags flags = 4; -} diff --git a/src/ripple/proto/org/xrpl/rpc/v1/meta.proto b/src/ripple/proto/org/xrpl/rpc/v1/meta.proto deleted file mode 100644 index 35660b3dd..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/meta.proto +++ /dev/null @@ -1,116 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -import "org/xrpl/rpc/v1/ledger_objects.proto"; -import "org/xrpl/rpc/v1/common.proto"; - -message SubmitMetadataRequest -{ - - repeated AffectedNode affected_nodes = 1; - uint32 ledger_sequence = 2; -} - -message SubmitMetadataResponse -{ - bool success = 1; - - string msg = 2; -} - -message PrepareLedgerRequest -{ - uint32 ledger_index = 1; -} - -message PrepareLedgerResponse -{ - bool success = 1; - - string msg = 2; -} - -// Next field: 5 -message Meta -{ - // index in ledger - uint64 transaction_index = 1 [jstype=JS_STRING]; - - // result code indicating whether the transaction succeeded or failed - TransactionResult transaction_result = 2; - - repeated AffectedNode affected_nodes = 3; - - DeliveredAmount delivered_amount = 4; -} - -// Next field: 3 -message TransactionResult -{ - // Next field: 7 - enum ResultType - { - RESULT_TYPE_UNSPECIFIED = 0; - // Claimed cost only - RESULT_TYPE_TEC = 1; - // Failure - RESULT_TYPE_TEF = 2; - // Local error - RESULT_TYPE_TEL = 3; - // Malformed transaction - RESULT_TYPE_TEM = 4; - // Retry - RESULT_TYPE_TER = 5; - // Success - RESULT_TYPE_TES = 6; - } - - // category of the transaction result - ResultType result_type = 1; - - // full result string, i.e. tesSUCCESS - string result = 2; -} - -// Next field: 6 -message AffectedNode -{ - LedgerEntryType ledger_entry_type = 1; - - // 32 bytes - bytes ledger_index = 2; - - oneof node - { - CreatedNode created_node = 3; - DeletedNode deleted_node = 4; - ModifiedNode modified_node = 5; - } -} - -// Next field: 2 -message CreatedNode -{ - LedgerObject new_fields = 1; -} - -// Next field: 2 -message DeletedNode -{ - LedgerObject final_fields = 1; -} - -// Next field: 5 -message ModifiedNode { - - LedgerObject final_fields = 1; - - LedgerObject previous_fields = 2; - - PreviousTransactionID previous_transaction_id = 3; - - PreviousTransactionLedgerSequence previous_transaction_ledger_sequence = 4; -} diff --git a/src/ripple/proto/org/xrpl/rpc/v1/submit.proto b/src/ripple/proto/org/xrpl/rpc/v1/submit.proto deleted file mode 100644 index f8b0a8270..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/submit.proto +++ /dev/null @@ -1,37 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -import "org/xrpl/rpc/v1/meta.proto"; - -// A request to submit the signed transaction to the ledger. -// Next field: 3 -message SubmitTransactionRequest -{ - // The signed transaction to submit. - bytes signed_transaction = 1; - - bool fail_hard = 2; - - string client_ip = 3; -} - -// A response when a signed transaction is submitted to the ledger. -// Next field: 5 -message SubmitTransactionResponse -{ - // Code indicating the preliminary result of the transaction. - TransactionResult engine_result = 1; - - // Numeric code indicating the preliminary result of the transaction, - // directly correlated to engine_result. - int64 engine_result_code = 2; - - // Human-readable explanation of the transaction's preliminary result. - string engine_result_message = 3; - - // 32 bytes - bytes hash = 4; -} diff --git a/src/ripple/proto/org/xrpl/rpc/v1/transaction.proto b/src/ripple/proto/org/xrpl/rpc/v1/transaction.proto deleted file mode 100644 index 05300422b..000000000 --- a/src/ripple/proto/org/xrpl/rpc/v1/transaction.proto +++ /dev/null @@ -1,390 +0,0 @@ -syntax = "proto3"; - -package org.xrpl.rpc.v1; -option java_package = "org.xrpl.rpc.v1"; -option java_multiple_files = true; - -import "org/xrpl/rpc/v1/common.proto"; -import "org/xrpl/rpc/v1/amount.proto"; -import "org/xrpl/rpc/v1/account.proto"; - -// A message encompassing all transaction types -// Next field: 37 -message Transaction -{ - Account account = 1; - - XRPDropsAmount fee = 2; - - Sequence sequence = 3; - - // Data specific to the type of transaction - oneof transaction_data - { - Payment payment = 4; - - AccountSet account_set = 13; - - AccountDelete account_delete = 14; - - CheckCancel check_cancel = 15; - - CheckCash check_cash = 16; - - CheckCreate check_create = 17; - - DepositPreauth deposit_preauth = 18; - - EscrowCancel escrow_cancel = 19; - - EscrowCreate escrow_create = 20; - - EscrowFinish escrow_finish = 21; - - NFTokenAcceptOffer nftoken_accept_offer = 32; - - NFTokenBurn nftoken_burn = 33; - - NFTokenCancelOffer nftoken_cancel_offer = 34; - - NFTokenCreateOffer nftoken_create_offer = 35; - - NFTokenMint nftoken_mint = 36; - - OfferCancel offer_cancel = 22; - - OfferCreate offer_create = 23; - - PaymentChannelClaim payment_channel_claim = 24; - - PaymentChannelCreate payment_channel_create= 25; - - PaymentChannelFund payment_channel_fund = 26; - - SetRegularKey set_regular_key = 27; - - SignerListSet signer_list_set = 28; - - TicketCreate ticket_create = 30; - - TrustSet trust_set = 29; - } - - SigningPublicKey signing_public_key = 5; - - TransactionSignature transaction_signature = 6; - - Flags flags = 7; - - LastLedgerSequence last_ledger_sequence = 8; - - SourceTag source_tag = 9; - - repeated Memo memos = 10; - - repeated Signer signers = 11; - - AccountTransactionID account_transaction_id = 12; - - TicketSequence ticket_sequence = 31; -} - -// Next field: 4 -message Memo -{ - MemoData memo_data = 1; - - MemoFormat memo_format = 2; - - MemoType memo_type = 3; -} - -// Next field: 4 -message Signer -{ - Account account = 1; - - TransactionSignature transaction_signature = 2; - - SigningPublicKey signing_public_key = 3; -} - -// Next field: 9 -message AccountSet -{ - ClearFlag clear_flag = 1; - - Domain domain = 2; - - EmailHash email_hash = 3; - - MessageKey message_key = 4; - - SetFlag set_flag = 5; - - TransferRate transfer_rate = 6; - - TickSize tick_size = 7; - - NFTokenMinter nftoken_minter = 8; -} - -// Next field: 3 -message AccountDelete -{ - Destination destination = 1; - - DestinationTag destination_tag = 2; -} - -// Next field: 2 -message CheckCancel -{ - CheckID check_id = 1; -} - -// Next field: 4 -message CheckCash -{ - CheckID check_id = 1; - - oneof amount_oneof - { - Amount amount = 2; - - DeliverMin deliver_min = 3; - } -} - -// Next field: 6 -message CheckCreate -{ - Destination destination = 1; - - SendMax send_max = 2; - - DestinationTag destination_tag = 3; - - Expiration expiration = 4; - - InvoiceID invoice_id = 5; -} - -// Next field: 3 -message DepositPreauth -{ - oneof authorization_oneof - { - Authorize authorize = 1; - - Unauthorize unauthorize = 2; - } -} - -// Next field: 3 -message EscrowCancel -{ - Owner owner = 1; - - OfferSequence offer_sequence = 2; -} - -// Next field: 7 -message EscrowCreate -{ - Amount amount = 1; - - Destination destination = 2; - - CancelAfter cancel_after = 3; - - FinishAfter finish_after = 4; - - Condition condition = 5; - - DestinationTag destination_tag = 6; -} - -// Next field: 5 -message EscrowFinish -{ - Owner owner = 1; - - OfferSequence offer_sequence = 2; - - Condition condition = 3; - - Fulfillment fulfillment = 4; -} - -// Next field: 4 -message NFTokenAcceptOffer -{ - NFTokenBrokerFee nftoken_broker_fee = 1; - - NFTokenBuyOffer nftoken_buy_offer = 2; - - NFTokenSellOffer nftoken_sell_offer = 3; -} - -// Next field: 3 -message NFTokenBurn -{ - Owner owner = 1; - - NFTokenID nftoken_id = 2; -} - -// Next field: 2 -message NFTokenCancelOffer -{ - repeated Index nftoken_offers = 1; -} - -// Next field: 6 -message NFTokenCreateOffer -{ - Amount amount = 1; - - Destination destination = 2; - - Expiration expiration = 3; - - Owner owner = 4; - - NFTokenID nftoken_id = 5; -} - -// Next field: 5 -message NFTokenMint -{ - Issuer issuer = 1; - - NFTokenTaxon nftoken_taxon = 2; - - TransferFee transfer_fee = 3; - - URI uri = 4; -} - -// Next field: 2 -message OfferCancel -{ - OfferSequence offer_sequence = 1; -} - -// Next field: 5 -message OfferCreate -{ - Expiration expiration = 1; - - OfferSequence offer_sequence = 2; - - TakerGets taker_gets = 3; - - TakerPays taker_pays = 4; -} - -// Next field: 8 -message Payment -{ - // Next field: 4 - message PathElement - { - AccountAddress account = 1; - - Currency currency = 2; - - AccountAddress issuer = 3; - } - - // Next field: 2 - message Path - { - repeated PathElement elements = 1; - } - - Amount amount = 1; - - Destination destination = 2; - - DestinationTag destination_tag = 3; - - InvoiceID invoice_id = 4; - - repeated Path paths = 5; - - SendMax send_max = 6; - - DeliverMin deliver_min = 7; -} - -// Next field: 6 -message PaymentChannelClaim -{ - Channel channel = 1; - - Balance balance = 2; - - Amount amount = 3; - - PaymentChannelSignature payment_channel_signature = 4; - - PublicKey public_key = 5; -} - -// Next field: 7 -message PaymentChannelCreate -{ - Amount amount = 1; - - Destination destination = 2; - - SettleDelay settle_delay = 3; - - PublicKey public_key = 4; - - CancelAfter cancel_after = 5; - - DestinationTag destination_tag = 6; -} - -// Next field: 4 -message PaymentChannelFund -{ - Channel channel = 1; - - Amount amount = 2; - - Expiration expiration = 3; -} - -// Next field: 2 -message SetRegularKey -{ - RegularKey regular_key = 1; -} - -// Next field: 3 -message SignerListSet -{ - SignerQuorum signer_quorum = 1; - - repeated SignerEntry signer_entries = 2; -} - -// Next field: 2 -message TicketCreate -{ - TicketCount count = 1; -} - -// Next field: 4 -message TrustSet -{ - LimitAmount limit_amount = 1; - - QualityIn quality_in = 2; - - QualityOut quality_out = 3; -} diff --git a/src/ripple/proto/org/xrpl/rpc/v1/xrp_ledger.proto b/src/ripple/proto/org/xrpl/rpc/v1/xrp_ledger.proto index 6f48d4860..995edba48 100644 --- a/src/ripple/proto/org/xrpl/rpc/v1/xrp_ledger.proto +++ b/src/ripple/proto/org/xrpl/rpc/v1/xrp_ledger.proto @@ -4,39 +4,18 @@ package org.xrpl.rpc.v1; option java_package = "org.xrpl.rpc.v1"; option java_multiple_files = true; -import "org/xrpl/rpc/v1/get_account_info.proto"; -import "org/xrpl/rpc/v1/get_fee.proto"; -import "org/xrpl/rpc/v1/submit.proto"; -import "org/xrpl/rpc/v1/get_transaction.proto"; -import "org/xrpl/rpc/v1/get_account_transaction_history.proto"; import "org/xrpl/rpc/v1/get_ledger.proto"; import "org/xrpl/rpc/v1/get_ledger_entry.proto"; import "org/xrpl/rpc/v1/get_ledger_data.proto"; import "org/xrpl/rpc/v1/get_ledger_diff.proto"; -// RPCs available to interact with the XRP Ledger. -// The gRPC API mimics the JSON API. Refer to xrpl.org for documentation +// These methods are binary only methods for retrieiving arbitrary ledger state +// via gRPC. These methods are used by clio and reporting mode, but can also be +// used by any client that wants to extract ledger state in an efficient manner. +// They do not directly mimic the JSON equivalent methods. service XRPLedgerAPIService { - // Get account info for an account on the XRP Ledger. - rpc GetAccountInfo (GetAccountInfoRequest) returns (GetAccountInfoResponse); - - // Get the fee for a transaction on the XRP Ledger. - rpc GetFee (GetFeeRequest) returns (GetFeeResponse); - - // Submit a signed transaction to the XRP Ledger. - rpc SubmitTransaction (SubmitTransactionRequest) returns (SubmitTransactionResponse); - - // Get the status of a transaction - rpc GetTransaction(GetTransactionRequest) returns (GetTransactionResponse); - - // Get all validated transactions associated with a given account - rpc GetAccountTransactionHistory(GetAccountTransactionHistoryRequest) returns (GetAccountTransactionHistoryResponse); - - ///////////////////////////////////////////////////////////////////////////// - // The below methods do not mimic the JSON API exactly, and are mostly binary - // Get a specific ledger, optionally including transactions and any modified, // added or deleted ledger objects rpc GetLedger(GetLedgerRequest) returns (GetLedgerResponse); diff --git a/src/ripple/protocol/AccountID.h b/src/ripple/protocol/AccountID.h index e514cfb74..79768eefd 100644 --- a/src/ripple/protocol/AccountID.h +++ b/src/ripple/protocol/AccountID.h @@ -106,41 +106,20 @@ operator<<(std::ostream& os, AccountID const& x) return os; } -//------------------------------------------------------------------------------ +/** Initialize the global cache used to map AccountID to base58 conversions. -/** Caches the base58 representations of AccountIDs + The cache is optional and need not be initialized. But because conversion + is expensive (it requires a SHA-256 operation) in most cases the overhead + of the cache is worth the benefit. - This operation occurs with sufficient frequency to - justify having a cache. In the future, rippled should - require clients to receive "binary" results, where - AccountIDs are hex-encoded. -*/ -class AccountIDCache -{ -private: - std::mutex mutable mutex_; - std::size_t capacity_; - hash_map mutable m0_; - hash_map mutable m1_; + @param count The number of entries the cache should accomodate. Zero will + disable the cache, releasing any memory associated with it. -public: - AccountIDCache(AccountIDCache const&) = delete; - AccountIDCache& - operator=(AccountIDCache const&) = delete; - - explicit AccountIDCache(std::size_t capacity); - - /** Return ripple::toBase58 for the AccountID - - Thread Safety: - Safe to call from any thread concurrently - - @note This function intentionally returns a - copy for correctness. - */ - std::string - toBase58(AccountID const&) const; -}; + @note The function will only initialize the cache the first time it is + invoked. Subsequent invocations do nothing. + */ +void +initAccountIdCache(std::size_t count); } // namespace ripple diff --git a/src/ripple/protocol/Feature.h b/src/ripple/protocol/Feature.h index bf56bced0..06c63900e 100644 --- a/src/ripple/protocol/Feature.h +++ b/src/ripple/protocol/Feature.h @@ -74,7 +74,7 @@ namespace detail { // Feature.cpp. Because it's only used to reserve storage, and determine how // large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than // the actual number of amendments. A LogicError on startup will verify this. -static constexpr std::size_t numFeatures = 47; +static constexpr std::size_t numFeatures = 54; /** Amendments that this server supports and the default voting behavior. Whether they are enabled depends on the Rules defined in the validated @@ -126,7 +126,6 @@ class FeatureBitset : private std::bitset public: using base::bitset; using base::operator==; - using base::operator!=; using base::all; using base::any; @@ -334,6 +333,13 @@ extern uint256 const fixSTAmountCanonicalize; extern uint256 const fixRmSmallIncreasedQOffers; extern uint256 const featureCheckCashMakesTrustLine; extern uint256 const featureNonFungibleTokensV1; +extern uint256 const featureExpandedSignerList; +extern uint256 const fixNFTokenDirV1; +extern uint256 const fixNFTokenNegOffer; +extern uint256 const featureNonFungibleTokensV1_1; +extern uint256 const fixTrustLinesToSelf; +extern uint256 const fixRemoveNFTokenAutoTrustLine; +extern uint256 const featureImmediateOfferKilled; extern uint256 const featurePaychanAndEscrowForTokens; } // namespace ripple diff --git a/src/ripple/protocol/Rules.h b/src/ripple/protocol/Rules.h new file mode 100644 index 000000000..d8190e86a --- /dev/null +++ b/src/ripple/protocol/Rules.h @@ -0,0 +1,86 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_LEDGER_RULES_H_INCLUDED +#define RIPPLE_LEDGER_RULES_H_INCLUDED + +#include +#include +#include +#include + +namespace ripple { + +class DigestAwareReadView; + +/** Rules controlling protocol behavior. */ +class Rules +{ +private: + class Impl; + + // Carrying impl by shared_ptr makes Rules comparatively cheap to pass + // by value. + std::shared_ptr impl_; + +public: + Rules(Rules const&) = default; + + Rules& + operator=(Rules const&) = default; + + Rules() = delete; + + /** Construct an empty rule set. + + These are the rules reflected by + the genesis ledger. + */ + explicit Rules(std::unordered_set> const& presets); + +private: + // Allow a friend function to construct Rules. + friend Rules + makeRulesGivenLedger( + DigestAwareReadView const& ledger, + std::unordered_set> const& presets); + + Rules( + std::unordered_set> const& presets, + std::optional const& digest, + STVector256 const& amendments); + +public: + /** Returns `true` if a feature is enabled. */ + bool + enabled(uint256 const& feature) const; + + /** Returns `true` if two rule sets are identical. + + @note This is for diagnostics. + */ + bool + operator==(Rules const&) const; + + bool + operator!=(Rules const& other) const; +}; + +} // namespace ripple +#endif diff --git a/src/ripple/protocol/SField.h b/src/ripple/protocol/SField.h index 7e4c4ccf1..5b514ad56 100644 --- a/src/ripple/protocol/SField.h +++ b/src/ripple/protocol/SField.h @@ -43,7 +43,7 @@ Some fields have a different meaning for their class STAccount; class STAmount; class STBlob; -template +template class STBitString; template class STInteger; diff --git a/src/ripple/protocol/STBitString.h b/src/ripple/protocol/STBitString.h index 1819d54d1..45d1a3d6f 100644 --- a/src/ripple/protocol/STBitString.h +++ b/src/ripple/protocol/STBitString.h @@ -25,9 +25,15 @@ namespace ripple { -template +// The template parameter could be an unsigned type, however there's a bug in +// gdb (last checked in gdb 12.1) that prevents gdb from finding the RTTI +// information of a template parameterized by an unsigned type. This RTTI +// information is needed to write gdb pretty printers. +template class STBitString final : public STBase { + static_assert(Bits > 0, "Number of bits must be positive"); + public: using value_type = base_uint; @@ -79,36 +85,36 @@ using STUInt128 = STBitString<128>; using STUInt160 = STBitString<160>; using STUInt256 = STBitString<256>; -template +template inline STBitString::STBitString(SField const& n) : STBase(n) { } -template +template inline STBitString::STBitString(const value_type& v) : value_(v) { } -template +template inline STBitString::STBitString(SField const& n, const value_type& v) : STBase(n), value_(v) { } -template +template inline STBitString::STBitString(SerialIter& sit, SField const& name) : STBitString(name, sit.getBitString()) { } -template +template STBase* STBitString::copy(std::size_t n, void* buf) const { return emplace(n, buf, *this); } -template +template STBase* STBitString::move(std::size_t n, void* buf) { @@ -136,14 +142,14 @@ STUInt256::getSType() const return STI_UINT256; } -template +template std::string STBitString::getText() const { return to_string(value_); } -template +template bool STBitString::isEquivalent(const STBase& t) const { @@ -151,7 +157,7 @@ STBitString::isEquivalent(const STBase& t) const return v && (value_ == v->value_); } -template +template void STBitString::add(Serializer& s) const { @@ -160,7 +166,7 @@ STBitString::add(Serializer& s) const s.addBitString(value_); } -template +template template void STBitString::setValue(base_uint const& v) @@ -168,20 +174,20 @@ STBitString::setValue(base_uint const& v) value_ = v; } -template +template typename STBitString::value_type const& STBitString::value() const { return value_; } -template +template STBitString::operator value_type() const { return value_; } -template +template bool STBitString::isDefault() const { diff --git a/src/ripple/protocol/STTx.h b/src/ripple/protocol/STTx.h index ca33abf8a..c6a9e053c 100644 --- a/src/ripple/protocol/STTx.h +++ b/src/ripple/protocol/STTx.h @@ -21,7 +21,9 @@ #define RIPPLE_PROTOCOL_STTX_H_INCLUDED #include +#include #include +#include #include #include #include @@ -47,7 +49,16 @@ class STTx final : public STObject, public CountedObject public: static std::size_t const minMultiSigners = 1; - static std::size_t const maxMultiSigners = 8; + + // if rules are not supplied then the largest possible value is returned + static std::size_t + maxMultiSigners(Rules const* rules = 0) + { + if (rules && !rules->enabled(featureExpandedSignerList)) + return 8; + + return 32; + } STTx() = delete; STTx(STTx const& other) = default; @@ -108,7 +119,8 @@ public: */ enum class RequireFullyCanonicalSig : bool { no, yes }; Expected - checkSign(RequireFullyCanonicalSig requireCanonicalSig) const; + checkSign(RequireFullyCanonicalSig requireCanonicalSig, Rules const& rules) + const; // SQL Functions with metadata. static std::string const& @@ -130,7 +142,9 @@ private: checkSingleSign(RequireFullyCanonicalSig requireCanonicalSig) const; Expected - checkMultiSign(RequireFullyCanonicalSig requireCanonicalSig) const; + checkMultiSign( + RequireFullyCanonicalSig requireCanonicalSig, + Rules const& rules) const; STBase* copy(std::size_t n, void* buf) const override; diff --git a/src/ripple/protocol/Seed.h b/src/ripple/protocol/Seed.h index c1768d205..2ebc64970 100644 --- a/src/ripple/protocol/Seed.h +++ b/src/ripple/protocol/Seed.h @@ -116,9 +116,13 @@ template <> std::optional parseBase58(std::string const& s); -/** Attempt to parse a string as a seed */ +/** Attempt to parse a string as a seed. + + @param str the string to parse + @param rfc1751 true if we should attempt RFC1751 style parsing (deprecated) + * */ std::optional -parseGenericSeed(std::string const& str); +parseGenericSeed(std::string const& str, bool rfc1751 = true); /** Encode a Seed in RFC1751 format */ std::string diff --git a/src/ripple/protocol/Serializer.h b/src/ripple/protocol/Serializer.h index 37058f562..7c3ccf958 100644 --- a/src/ripple/protocol/Serializer.h +++ b/src/ripple/protocol/Serializer.h @@ -251,22 +251,22 @@ public: } bool - operator==(Blob const& v) + operator==(Blob const& v) const { return v == mData; } bool - operator!=(Blob const& v) + operator!=(Blob const& v) const { return v != mData; } bool - operator==(const Serializer& v) + operator==(const Serializer& v) const { return v.mData == mData; } bool - operator!=(const Serializer& v) + operator!=(const Serializer& v) const { return v.mData != mData; } diff --git a/src/ripple/protocol/TxFlags.h b/src/ripple/protocol/TxFlags.h index 0b907c722..0ad088c41 100644 --- a/src/ripple/protocol/TxFlags.h +++ b/src/ripple/protocol/TxFlags.h @@ -120,9 +120,25 @@ constexpr std::uint32_t const tfOnlyXRP = 0x00000002; constexpr std::uint32_t const tfTrustLine = 0x00000004; constexpr std::uint32_t const tfTransferable = 0x00000008; -constexpr std::uint32_t const tfNFTokenMintMask = +// Prior to fixRemoveNFTokenAutoTrustLine, transfer of an NFToken between +// accounts allowed a TrustLine to be added to the issuer of that token +// without explicit permission from that issuer. This was enabled by +// minting the NFToken with the tfTrustLine flag set. +// +// That capability could be used to attack the NFToken issuer. It +// would be possible for two accounts to trade the NFToken back and forth +// building up any number of TrustLines on the issuer, increasing the +// issuer's reserve without bound. +// +// The fixRemoveNFTokenAutoTrustLine amendment disables minting with the +// tfTrustLine flag as a way to prevent the attack. But until the +// amendment passes we still need to keep the old behavior available. +constexpr std::uint32_t const tfNFTokenMintOldMask = ~(tfUniversal | tfBurnable | tfOnlyXRP | tfTrustLine | tfTransferable); +constexpr std::uint32_t const tfNFTokenMintMask = + ~(tfUniversal | tfBurnable | tfOnlyXRP | tfTransferable); + // NFTokenCreateOffer flags: constexpr std::uint32_t const tfSellNFToken = 0x00000001; constexpr std::uint32_t const tfNFTokenCreateOfferMask = diff --git a/src/ripple/protocol/impl/AccountID.cpp b/src/ripple/protocol/impl/AccountID.cpp index 8ca8d1d15..c615807cf 100644 --- a/src/ripple/protocol/impl/AccountID.cpp +++ b/src/ripple/protocol/impl/AccountID.cpp @@ -17,17 +17,95 @@ */ //============================================================================== +#include +#include #include #include #include #include +#include #include +#include namespace ripple { +namespace detail { + +/** Caches the base58 representations of AccountIDs */ +class AccountIdCache +{ +private: + struct CachedAccountID + { + AccountID id; + char encoding[40] = {0}; + }; + + // The actual cache + std::vector cache_; + + // We use a hash function designed to resist algorithmic complexity attacks + hardened_hash<> hasher_; + + // 64 spinlocks, packed into a single 64-bit value + std::atomic locks_ = 0; + +public: + AccountIdCache(std::size_t count) : cache_(count) + { + // This is non-binding, but we try to avoid wasting memory that + // is caused by overallocation. + cache_.shrink_to_fit(); + } + + std::string + toBase58(AccountID const& id) + { + auto const index = hasher_(id) % cache_.size(); + + packed_spinlock sl(locks_, index % 64); + + { + std::lock_guard lock(sl); + + // The check against the first character of the encoding ensures + // that we don't mishandle the case of the all-zero account: + if (cache_[index].encoding[0] != 0 && cache_[index].id == id) + return cache_[index].encoding; + } + + auto ret = + encodeBase58Token(TokenType::AccountID, id.data(), id.size()); + + assert(ret.size() <= 38); + + { + std::lock_guard lock(sl); + cache_[index].id = id; + std::strcpy(cache_[index].encoding, ret.c_str()); + } + + return ret; + } +}; + +} // namespace detail + +static std::unique_ptr accountIdCache; + +void +initAccountIdCache(std::size_t count) +{ + if (!accountIdCache && count != 0) + accountIdCache = std::make_unique(count); +} + std::string toBase58(AccountID const& v) { + if (accountIdCache) + return accountIdCache->toBase58(v); + return encodeBase58Token(TokenType::AccountID, v.data(), v.size()); } @@ -112,52 +190,4 @@ to_issuer(AccountID& issuer, std::string const& s) return true; } -//------------------------------------------------------------------------------ - -/* VFALCO NOTE - An alternate implementation could use a pair of insert-only - hash maps that each use a single large memory allocation - to store a fixed size hash table and all of the AccountID/string - pairs laid out in memory (wouldn't use std::string here just a - length prefixed or zero terminated array). Possibly using - boost::intrusive as the basis for the unordered container. - This would cut down to one allocate/free cycle per swap of - the map. -*/ - -AccountIDCache::AccountIDCache(std::size_t capacity) : capacity_(capacity) -{ - m1_.reserve(capacity_); -} - -std::string -AccountIDCache::toBase58(AccountID const& id) const -{ - std::lock_guard lock(mutex_); - auto iter = m1_.find(id); - if (iter != m1_.end()) - return iter->second; - iter = m0_.find(id); - std::string result; - if (iter != m0_.end()) - { - result = iter->second; - // Can use insert-only hash maps if - // we didn't erase from here. - m0_.erase(iter); - } - else - { - result = ripple::toBase58(id); - } - if (m1_.size() >= capacity_) - { - m0_ = std::move(m1_); - m1_.clear(); - m1_.reserve(capacity_); - } - m1_.emplace(id, result); - return result; -} - } // namespace ripple diff --git a/src/ripple/protocol/impl/BuildInfo.cpp b/src/ripple/protocol/impl/BuildInfo.cpp index 6cd3668dd..22f2f17be 100644 --- a/src/ripple/protocol/impl/BuildInfo.cpp +++ b/src/ripple/protocol/impl/BuildInfo.cpp @@ -33,7 +33,7 @@ namespace BuildInfo { // and follow the format described at http://semver.org/ //------------------------------------------------------------------------------ // clang-format off -char const* const versionString = "1.9.0" +char const* const versionString = "1.10.0-rc1" // clang-format on #if defined(DEBUG) || defined(SANITIZER) diff --git a/src/ripple/protocol/impl/Feature.cpp b/src/ripple/protocol/impl/Feature.cpp index 938a0d342..759608703 100644 --- a/src/ripple/protocol/impl/Feature.cpp +++ b/src/ripple/protocol/impl/Feature.cpp @@ -52,7 +52,7 @@ enum class Supported : bool { no = false, yes }; // enabled using run-time conditionals based on the state of the amendment. // There is value in retaining that conditional code for some time after // the amendment is enabled to make it simple to replay old transactions. -// However, once an Amendment has been enabled for, say, more than two years +// However, once an amendment has been enabled for, say, more than two years // then retaining that conditional code has less value since it is // uncommon to replay such old transactions. // @@ -61,10 +61,15 @@ enum class Supported : bool { no = false, yes }; // 2018 needs to happen on an older version of the server code. There's // a log message in Application.cpp that warns about replaying old ledgers. // -// At some point in the future someone may wish to remove Amendment -// conditional code for Amendments that were enabled after January 2018. +// At some point in the future someone may wish to remove amendment +// conditional code for amendments that were enabled after January 2018. // When that happens then the log message in Application.cpp should be // updated. +// +// Generally, amendments which introduce new features should be set as +// "DefaultVote::no" whereas in rare cases, amendments that fix critical +// bugs should be set as "DefaultVote::yes", if off-chain consensus is +// reached amongst reviewers, validator operators, and other participants. class FeatureCollections { @@ -431,13 +436,20 @@ REGISTER_FEATURE(RequireFullyCanonicalSig, Supported::yes, DefaultVote::yes REGISTER_FIX (fix1781, Supported::yes, DefaultVote::yes); REGISTER_FEATURE(HardenedValidations, Supported::yes, DefaultVote::yes); REGISTER_FIX (fixAmendmentMajorityCalc, Supported::yes, DefaultVote::yes); -REGISTER_FEATURE(NegativeUNL, Supported::yes, DefaultVote::no); +REGISTER_FEATURE(NegativeUNL, Supported::yes, DefaultVote::yes); REGISTER_FEATURE(TicketBatch, Supported::yes, DefaultVote::yes); REGISTER_FEATURE(FlowSortStrands, Supported::yes, DefaultVote::yes); REGISTER_FIX (fixSTAmountCanonicalize, Supported::yes, DefaultVote::yes); REGISTER_FIX (fixRmSmallIncreasedQOffers, Supported::yes, DefaultVote::yes); REGISTER_FEATURE(CheckCashMakesTrustLine, Supported::yes, DefaultVote::no); REGISTER_FEATURE(NonFungibleTokensV1, Supported::yes, DefaultVote::no); +REGISTER_FEATURE(ExpandedSignerList, Supported::yes, DefaultVote::no); +REGISTER_FIX (fixNFTokenDirV1, Supported::yes, DefaultVote::no); +REGISTER_FIX (fixNFTokenNegOffer, Supported::yes, DefaultVote::no); +REGISTER_FEATURE(NonFungibleTokensV1_1, Supported::yes, DefaultVote::no); +REGISTER_FIX (fixTrustLinesToSelf, Supported::yes, DefaultVote::no); +REGISTER_FIX (fixRemoveNFTokenAutoTrustLine, Supported::yes, DefaultVote::yes); +REGISTER_FEATURE(ImmediateOfferKilled, Supported::yes, DefaultVote::no); REGISTER_FEATURE(PaychanAndEscrowForTokens, Supported::yes, DefaultVote::no); // The following amendments have been active for at least two years. Their diff --git a/src/ripple/protocol/impl/InnerObjectFormats.cpp b/src/ripple/protocol/impl/InnerObjectFormats.cpp index c1b2acc87..1b4aa63c2 100644 --- a/src/ripple/protocol/impl/InnerObjectFormats.cpp +++ b/src/ripple/protocol/impl/InnerObjectFormats.cpp @@ -28,6 +28,7 @@ InnerObjectFormats::InnerObjectFormats() { {sfAccount, soeREQUIRED}, {sfSignerWeight, soeREQUIRED}, + {sfWalletLocator, soeOPTIONAL}, }); add(sfSigner.jsonName.c_str(), diff --git a/src/ripple/protocol/impl/PublicKey.cpp b/src/ripple/protocol/impl/PublicKey.cpp index 9fed78088..ac86634f1 100644 --- a/src/ripple/protocol/impl/PublicKey.cpp +++ b/src/ripple/protocol/impl/PublicKey.cpp @@ -24,7 +24,6 @@ #include #include #include -#include namespace ripple { @@ -186,14 +185,18 @@ PublicKey::PublicKey(PublicKey const& other) : size_(other.size_) { if (size_) std::memcpy(buf_, other.buf_, size_); -}; +} PublicKey& PublicKey::operator=(PublicKey const& other) { - size_ = other.size_; - if (size_) - std::memcpy(buf_, other.buf_, size_); + if (this != &other) + { + size_ = other.size_; + if (size_) + std::memcpy(buf_, other.buf_, size_); + } + return *this; } diff --git a/src/ripple/protocol/impl/Rules.cpp b/src/ripple/protocol/impl/Rules.cpp new file mode 100644 index 000000000..35a09b856 --- /dev/null +++ b/src/ripple/protocol/impl/Rules.cpp @@ -0,0 +1,113 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include + +namespace ripple { + +class Rules::Impl +{ +private: + std::unordered_set> set_; + std::optional digest_; + std::unordered_set> const& presets_; + +public: + explicit Impl(std::unordered_set> const& presets) + : presets_(presets) + { + } + + Impl( + std::unordered_set> const& presets, + std::optional const& digest, + STVector256 const& amendments) + : digest_(digest), presets_(presets) + { + set_.reserve(amendments.size()); + set_.insert(amendments.begin(), amendments.end()); + } + + bool + enabled(uint256 const& feature) const + { + if (presets_.count(feature) > 0) + return true; + return set_.count(feature) > 0; + } + + bool + operator==(Impl const& other) const + { + if (!digest_ && !other.digest_) + return true; + if (!digest_ || !other.digest_) + return false; + return *digest_ == *other.digest_; + } +}; + +Rules::Rules(std::unordered_set> const& presets) + : impl_(std::make_shared(presets)) +{ +} + +Rules::Rules( + std::unordered_set> const& presets, + std::optional const& digest, + STVector256 const& amendments) + : impl_(std::make_shared(presets, digest, amendments)) +{ +} + +bool +Rules::enabled(uint256 const& feature) const +{ + assert(impl_); + + // The functionality of the "NonFungibleTokensV1_1" amendment is + // precisely the functionality of the following three amendments + // so if their status is ever queried individually, we inject an + // extra check here to simplify the checking elsewhere. + if (feature == featureNonFungibleTokensV1 || + feature == fixNFTokenNegOffer || feature == fixNFTokenDirV1) + { + if (impl_->enabled(featureNonFungibleTokensV1_1)) + return true; + } + + return impl_->enabled(feature); +} + +bool +Rules::operator==(Rules const& other) const +{ + assert(impl_ && other.impl_); + if (impl_.get() == other.impl_.get()) + return true; + return *impl_ == *other.impl_; +} + +bool +Rules::operator!=(Rules const& other) const +{ + return !(*this == other); +} +} // namespace ripple diff --git a/src/ripple/protocol/impl/STTx.cpp b/src/ripple/protocol/impl/STTx.cpp index c8e05c742..1ce4ddb64 100644 --- a/src/ripple/protocol/impl/STTx.cpp +++ b/src/ripple/protocol/impl/STTx.cpp @@ -206,7 +206,9 @@ STTx::sign(PublicKey const& publicKey, SecretKey const& secretKey) } Expected -STTx::checkSign(RequireFullyCanonicalSig requireCanonicalSig) const +STTx::checkSign( + RequireFullyCanonicalSig requireCanonicalSig, + Rules const& rules) const { try { @@ -214,8 +216,9 @@ STTx::checkSign(RequireFullyCanonicalSig requireCanonicalSig) const // at the SigningPubKey. If it's empty we must be // multi-signing. Otherwise we're single-signing. Blob const& signingPubKey = getFieldVL(sfSigningPubKey); - return signingPubKey.empty() ? checkMultiSign(requireCanonicalSig) - : checkSingleSign(requireCanonicalSig); + return signingPubKey.empty() + ? checkMultiSign(requireCanonicalSig, rules) + : checkSingleSign(requireCanonicalSig); } catch (std::exception const&) { @@ -327,7 +330,9 @@ STTx::checkSingleSign(RequireFullyCanonicalSig requireCanonicalSig) const } Expected -STTx::checkMultiSign(RequireFullyCanonicalSig requireCanonicalSig) const +STTx::checkMultiSign( + RequireFullyCanonicalSig requireCanonicalSig, + Rules const& rules) const { // Make sure the MultiSigners are present. Otherwise they are not // attempting multi-signing and we just have a bad SigningPubKey. @@ -342,7 +347,8 @@ STTx::checkMultiSign(RequireFullyCanonicalSig requireCanonicalSig) const STArray const& signers{getFieldArray(sfSigners)}; // There are well known bounds that the number of signers must be within. - if (signers.size() < minMultiSigners || signers.size() > maxMultiSigners) + if (signers.size() < minMultiSigners || + signers.size() > maxMultiSigners(&rules)) return Unexpected("Invalid Signers array size."); // We can ease the computational load inside the loop a bit by @@ -475,11 +481,10 @@ isMemoOkay(STObject const& st, std::string& reason) // The only allowed characters for MemoType and MemoFormat are the // characters allowed in URLs per RFC 3986: alphanumerics and the // following symbols: -._~:/?#[]@!$&'()*+,;=% - static std::array const allowedSymbols = [] { - std::array a; - a.fill(0); + static constexpr std::array const allowedSymbols = []() { + std::array a{}; - std::string symbols( + std::string_view symbols( "0123456789" "-._~:/?#[]@!$&'()*+,;=%" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" diff --git a/src/ripple/protocol/impl/STVector256.cpp b/src/ripple/protocol/impl/STVector256.cpp index f74670ac0..0ef1295b1 100644 --- a/src/ripple/protocol/impl/STVector256.cpp +++ b/src/ripple/protocol/impl/STVector256.cpp @@ -26,19 +26,19 @@ namespace ripple { STVector256::STVector256(SerialIter& sit, SField const& name) : STBase(name) { - Blob data = sit.getVL(); - auto const count = data.size() / (256 / 8); - mValue.reserve(count); - Blob::iterator begin = data.begin(); - unsigned int uStart = 0; - for (unsigned int i = 0; i != count; i++) - { - unsigned int uEnd = uStart + (256 / 8); - // This next line could be optimized to construct a default - // uint256 in the vector and then copy into it - mValue.push_back(uint256(Blob(begin + uStart, begin + uEnd))); - uStart = uEnd; - } + auto const slice = sit.getSlice(sit.getVLDataLength()); + + if (slice.size() % uint256::size() != 0) + Throw( + "Bad serialization for STVector256: " + + std::to_string(slice.size())); + + auto const cnt = slice.size() / uint256::size(); + + mValue.reserve(cnt); + + for (std::size_t i = 0; i != cnt; ++i) + mValue.emplace_back(slice.substr(i * uint256::size(), uint256::size())); } STBase* diff --git a/src/ripple/protocol/impl/Seed.cpp b/src/ripple/protocol/impl/Seed.cpp index f4c6ee52b..49da20a42 100644 --- a/src/ripple/protocol/impl/Seed.cpp +++ b/src/ripple/protocol/impl/Seed.cpp @@ -87,7 +87,7 @@ parseBase58(std::string const& s) } std::optional -parseGenericSeed(std::string const& str) +parseGenericSeed(std::string const& str, bool rfc1751) { if (str.empty()) return std::nullopt; @@ -111,6 +111,7 @@ parseGenericSeed(std::string const& str) if (auto seed = parseBase58(str)) return seed; + if (rfc1751) { std::string key; if (RFC1751::getKeyFromEnglish(key, str) == 1) diff --git a/src/ripple/protocol/impl/TER.cpp b/src/ripple/protocol/impl/TER.cpp index 7989e6b2d..89c73723c 100644 --- a/src/ripple/protocol/impl/TER.cpp +++ b/src/ripple/protocol/impl/TER.cpp @@ -77,7 +77,7 @@ transResults() MAKE_ERROR(tecINVARIANT_FAILED, "One or more invariants for the transaction were not satisfied."), MAKE_ERROR(tecEXPIRED, "Expiration time is passed."), MAKE_ERROR(tecDUPLICATE, "Ledger object already exists."), - MAKE_ERROR(tecKILLED, "FillOrKill offer killed."), + MAKE_ERROR(tecKILLED, "No funds transferred and no offer created."), MAKE_ERROR(tecHAS_OBLIGATIONS, "The account cannot be deleted since it has obligations."), MAKE_ERROR(tecTOO_SOON, "It is too early to attempt the requested operation. Please wait."), MAKE_ERROR(tecMAX_SEQUENCE_REACHED, "The maximum sequence number was reached."), diff --git a/src/ripple/protocol/jss.h b/src/ripple/protocol/jss.h index c4fbbcb59..de71c6d1e 100644 --- a/src/ripple/protocol/jss.h +++ b/src/ripple/protocol/jss.h @@ -166,11 +166,13 @@ JSS(build_path); // in: TransactionSign JSS(build_version); // out: NetworkOPs JSS(cancel_after); // out: AccountChannels JSS(can_delete); // out: CanDelete +JSS(changes); // out: BookChanges JSS(channel_id); // out: AccountChannels JSS(channels); // out: AccountChannels JSS(check); // in: AccountObjects JSS(check_nodes); // in: LedgerCleaner JSS(clear); // in/out: FetchInfo +JSS(close); // out: BookChanges JSS(close_flags); // out: LedgerToJson JSS(close_time); // in: Application, out: NetworkOPs, // RCLCxPeerPos, LedgerToJson @@ -193,6 +195,8 @@ JSS(converge_time_s); // out: NetworkOPs JSS(cookie); // out: NetworkOPs JSS(count); // in: AccountTx*, ValidatorList JSS(counters); // in/out: retrieve counters +JSS(currency_a); // out: BookChanges +JSS(currency_b); // out: BookChanges JSS(currentShard); // out: NodeToShardStatus JSS(currentShardIndex); // out: NodeToShardStatus JSS(currency); // in: paths/PathRequest, STAmount @@ -282,6 +286,7 @@ JSS(hashes); // in: AccountObjects JSS(have_header); // out: InboundLedger JSS(have_state); // out: InboundLedger JSS(have_transactions); // out: InboundLedger +JSS(high); // out: BookChanges JSS(highest_sequence); // out: AccountInfo JSS(highest_ticket); // out: AccountInfo JSS(historical_perminute); // historical_perminute. @@ -363,6 +368,7 @@ JSS(load_fee); // out: LoadFeeTrackImp, NetworkOPs JSS(local); // out: resource/Logic.h JSS(local_txs); // out: GetCounts JSS(local_static_keys); // out: ValidatorList +JSS(low); // out: BookChanges JSS(locked_balance); // out: AccountLines JSS(lock_count); // out: AccountLines JSS(lowest_sequence); // out: AccountInfo @@ -641,6 +647,8 @@ JSS(validator_sites); // out: ValidatorSites JSS(value); // out: STAmount JSS(version); // out: RPCVersion JSS(vetoed); // out: AmendmentTableImpl +JSS(volume_a); // out: BookChanges +JSS(volume_b); // out: BookChanges JSS(vote); // in: Feature JSS(warning); // rpc: JSS(warnings); // out: server_info, server_state diff --git a/src/ripple/rpc/BookChanges.h b/src/ripple/rpc/BookChanges.h new file mode 100644 index 000000000..7d7978d3f --- /dev/null +++ b/src/ripple/rpc/BookChanges.h @@ -0,0 +1,210 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2019 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_RPC_BOOKCHANGES_H_INCLUDED +#define RIPPLE_RPC_BOOKCHANGES_H_INCLUDED + +namespace Json { +class Value; +} + +namespace ripple { + +class ReadView; +class Transaction; +class TxMeta; +class STTx; + +namespace RPC { + +template +Json::Value +computeBookChanges(std::shared_ptr const& lpAccepted) +{ + std::map< + std::string, + std::tuple< + STAmount, // side A volume + STAmount, // side B volume + STAmount, // high rate + STAmount, // low rate + STAmount, // open rate + STAmount // close rate + >> + tally; + + for (auto& tx : lpAccepted->txs) + { + if (!tx.first || !tx.second || + !tx.first->isFieldPresent(sfTransactionType)) + continue; + + std::optional offerCancel; + uint16_t tt = tx.first->getFieldU16(sfTransactionType); + switch (tt) + { + case ttOFFER_CANCEL: + case ttOFFER_CREATE: { + if (tx.first->isFieldPresent(sfOfferSequence)) + offerCancel = tx.first->getFieldU32(sfOfferSequence); + break; + } + // in future if any other ways emerge to cancel an offer + // this switch makes them easy to add + default: + break; + } + + for (auto const& node : tx.second->getFieldArray(sfAffectedNodes)) + { + SField const& metaType = node.getFName(); + uint16_t nodeType = node.getFieldU16(sfLedgerEntryType); + + // we only care about ltOFFER objects being modified or + // deleted + if (nodeType != ltOFFER || metaType == sfCreatedNode) + continue; + + // if either FF or PF are missing we can't compute + // but generally these are cancelled rather than crossed + // so skipping them is consistent + if (!node.isFieldPresent(sfFinalFields) || + !node.isFieldPresent(sfPreviousFields)) + continue; + + auto const& ffBase = node.peekAtField(sfFinalFields); + auto const& finalFields = ffBase.template downcast(); + auto const& pfBase = node.peekAtField(sfPreviousFields); + auto const& previousFields = pfBase.template downcast(); + + // defensive case that should never be hit + if (!finalFields.isFieldPresent(sfTakerGets) || + !finalFields.isFieldPresent(sfTakerPays) || + !previousFields.isFieldPresent(sfTakerGets) || + !previousFields.isFieldPresent(sfTakerPays)) + continue; + + // filter out any offers deleted by explicit offer cancels + if (metaType == sfDeletedNode && offerCancel && + finalFields.getFieldU32(sfSequence) == *offerCancel) + continue; + + // compute the difference in gets and pays actually + // affected onto the offer + STAmount deltaGets = finalFields.getFieldAmount(sfTakerGets) - + previousFields.getFieldAmount(sfTakerGets); + STAmount deltaPays = finalFields.getFieldAmount(sfTakerPays) - + previousFields.getFieldAmount(sfTakerPays); + + std::string g{to_string(deltaGets.issue())}; + std::string p{to_string(deltaPays.issue())}; + + bool const noswap = + isXRP(deltaGets) ? true : (isXRP(deltaPays) ? false : (g < p)); + + STAmount first = noswap ? deltaGets : deltaPays; + STAmount second = noswap ? deltaPays : deltaGets; + + // defensively programmed, should (probably) never happen + if (second == beast::zero) + continue; + + STAmount rate = divide(first, second, noIssue()); + + if (first < beast::zero) + first = -first; + + if (second < beast::zero) + second = -second; + + std::stringstream ss; + if (noswap) + ss << g << "|" << p; + else + ss << p << "|" << g; + + std::string key{ss.str()}; + + if (tally.find(key) == tally.end()) + tally[key] = { + first, // side A vol + second, // side B vol + rate, // high + rate, // low + rate, // open + rate // close + }; + else + { + // increment volume + auto& entry = tally[key]; + + std::get<0>(entry) += first; // side A vol + std::get<1>(entry) += second; // side B vol + + if (std::get<2>(entry) < rate) // high + std::get<2>(entry) = rate; + + if (std::get<3>(entry) > rate) // low + std::get<3>(entry) = rate; + + std::get<5>(entry) = rate; // close + } + } + } + + Json::Value jvObj(Json::objectValue); + jvObj[jss::type] = "bookChanges"; + jvObj[jss::ledger_index] = lpAccepted->info().seq; + jvObj[jss::ledger_hash] = to_string(lpAccepted->info().hash); + jvObj[jss::ledger_time] = Json::Value::UInt( + lpAccepted->info().closeTime.time_since_epoch().count()); + + jvObj[jss::changes] = Json::arrayValue; + + for (auto const& entry : tally) + { + Json::Value& inner = jvObj[jss::changes].append(Json::objectValue); + + STAmount volA = std::get<0>(entry.second); + STAmount volB = std::get<1>(entry.second); + + inner[jss::currency_a] = + (isXRP(volA) ? "XRP_drops" : to_string(volA.issue())); + inner[jss::currency_b] = + (isXRP(volB) ? "XRP_drops" : to_string(volB.issue())); + + inner[jss::volume_a] = + (isXRP(volA) ? to_string(volA.xrp()) : to_string(volA.iou())); + inner[jss::volume_b] = + (isXRP(volB) ? to_string(volB.xrp()) : to_string(volB.iou())); + + inner[jss::high] = to_string(std::get<2>(entry.second).iou()); + inner[jss::low] = to_string(std::get<3>(entry.second).iou()); + inner[jss::open] = to_string(std::get<4>(entry.second).iou()); + inner[jss::close] = to_string(std::get<5>(entry.second).iou()); + } + + return jvObj; +} + +} // namespace RPC +} // namespace ripple + +#endif diff --git a/src/ripple/rpc/DeliveredAmount.h b/src/ripple/rpc/DeliveredAmount.h index 8b417b954..94fff68f7 100644 --- a/src/ripple/rpc/DeliveredAmount.h +++ b/src/ripple/rpc/DeliveredAmount.h @@ -22,7 +22,6 @@ #include #include -#include #include #include diff --git a/src/ripple/rpc/GRPCHandlers.h b/src/ripple/rpc/GRPCHandlers.h index d7b1cc3a9..493de7a5c 100644 --- a/src/ripple/rpc/GRPCHandlers.h +++ b/src/ripple/rpc/GRPCHandlers.h @@ -34,28 +34,6 @@ namespace ripple { * the status will be sent to the client, and the response will be ommitted */ -std::pair -doAccountInfoGrpc( - RPC::GRPCContext& context); - -std::pair -doFeeGrpc(RPC::GRPCContext& context); - -std::pair -doSubmitGrpc( - RPC::GRPCContext& context); - -// NOTE, this only supports Payment transactions at this time -std::pair -doTxGrpc(RPC::GRPCContext& context); - -std::pair< - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse, - grpc::Status> -doAccountTxGrpc( - RPC::GRPCContext& - context); - std::pair doLedgerGrpc(RPC::GRPCContext& context); diff --git a/src/ripple/rpc/handlers/AccountChannels.cpp b/src/ripple/rpc/handlers/AccountChannels.cpp index cc79173e5..e5059d3ff 100644 --- a/src/ripple/rpc/handlers/AccountChannels.cpp +++ b/src/ripple/rpc/handlers/AccountChannels.cpp @@ -202,7 +202,7 @@ doAccountChannels(RPC::JsonContext& context) to_string(*marker) + "," + std::to_string(nextHint); } - result[jss::account] = context.app.accountIDCache().toBase58(accountID); + result[jss::account] = toBase58(accountID); for (auto const& item : visitData.items) addChannel(jsonChannels, *item); diff --git a/src/ripple/rpc/handlers/AccountInfo.cpp b/src/ripple/rpc/handlers/AccountInfo.cpp index 417a3ffcd..f08805761 100644 --- a/src/ripple/rpc/handlers/AccountInfo.cpp +++ b/src/ripple/rpc/handlers/AccountInfo.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include @@ -215,102 +214,11 @@ doAccountInfo(RPC::JsonContext& context) } else { - result[jss::account] = context.app.accountIDCache().toBase58(accountID); + result[jss::account] = toBase58(accountID); RPC::inject_error(rpcACT_NOT_FOUND, result); } return result; } -std::pair -doAccountInfoGrpc( - RPC::GRPCContext& context) -{ - // Return values - org::xrpl::rpc::v1::GetAccountInfoResponse result; - grpc::Status status = grpc::Status::OK; - - // input - org::xrpl::rpc::v1::GetAccountInfoRequest& params = context.params; - - // get ledger - std::shared_ptr ledger; - auto lgrStatus = RPC::ledgerFromRequest(ledger, context); - if (lgrStatus || !ledger) - { - grpc::Status errorStatus; - if (lgrStatus.toErrorCode() == rpcINVALID_PARAMS) - { - errorStatus = grpc::Status( - grpc::StatusCode::INVALID_ARGUMENT, lgrStatus.message()); - } - else - { - errorStatus = - grpc::Status(grpc::StatusCode::NOT_FOUND, lgrStatus.message()); - } - return {result, errorStatus}; - } - - result.set_ledger_index(ledger->info().seq); - result.set_validated( - RPC::isValidated(context.ledgerMaster, *ledger, context.app)); - - // decode account - AccountID accountID; - std::string strIdent = params.account().address(); - error_code_i code = - RPC::accountFromStringWithCode(accountID, strIdent, params.strict()); - if (code != rpcSUCCESS) - { - grpc::Status errorStatus{ - grpc::StatusCode::INVALID_ARGUMENT, "invalid account"}; - return {result, errorStatus}; - } - - // get account data - auto const sleAccepted = ledger->read(keylet::account(accountID)); - if (sleAccepted) - { - RPC::convert(*result.mutable_account_data(), *sleAccepted); - - // signer lists - if (params.signer_lists()) - { - auto const sleSigners = ledger->read(keylet::signers(accountID)); - if (sleSigners) - { - org::xrpl::rpc::v1::SignerList& signerListProto = - *result.mutable_signer_list(); - RPC::convert(signerListProto, *sleSigners); - } - } - - // queued transactions - if (params.queue()) - { - if (!ledger->open()) - { - grpc::Status errorStatus{ - grpc::StatusCode::INVALID_ARGUMENT, - "requested queue but ledger is not open"}; - return {result, errorStatus}; - } - std::vector const txs = - context.app.getTxQ().getAccountTxs(accountID); - org::xrpl::rpc::v1::QueueData& queueData = - *result.mutable_queue_data(); - RPC::convert(queueData, txs); - } - } - else - { - grpc::Status errorStatus{ - grpc::StatusCode::NOT_FOUND, "account not found"}; - return {result, errorStatus}; - } - - return {result, status}; -} - } // namespace ripple diff --git a/src/ripple/rpc/handlers/AccountLines.cpp b/src/ripple/rpc/handlers/AccountLines.cpp index bbcda69f1..a1c958f71 100644 --- a/src/ripple/rpc/handlers/AccountLines.cpp +++ b/src/ripple/rpc/handlers/AccountLines.cpp @@ -265,7 +265,7 @@ doAccountLines(RPC::JsonContext& context) to_string(*marker) + "," + std::to_string(nextHint); } - result[jss::account] = context.app.accountIDCache().toBase58(accountID); + result[jss::account] = toBase58(accountID); for (auto const& item : visitData.items) addLine(jsonLines, item); diff --git a/src/ripple/rpc/handlers/AccountObjects.cpp b/src/ripple/rpc/handlers/AccountObjects.cpp index 55fe4e413..6424c3afd 100644 --- a/src/ripple/rpc/handlers/AccountObjects.cpp +++ b/src/ripple/rpc/handlers/AccountObjects.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -101,21 +102,41 @@ doAccountNFTs(RPC::JsonContext& context) auto& nfts = (result[jss::account_nfts] = Json::arrayValue); // Continue iteration from the current page: - + bool pastMarker = marker.isZero(); + uint256 const maskedMarker = marker & nft::pageMask; while (cp) { auto arr = cp->getFieldArray(sfNFTokens); for (auto const& o : arr) { - if (o.getFieldH256(sfNFTokenID) <= marker) + // Scrolling past the marker gets weird. We need to look at + // a couple of conditions. + // + // 1. If the low 96-bits don't match, then we compare only + // against the low 96-bits, since that's what determines + // the sort order of the pages. + // + // 2. However, within one page there can be a number of + // NFTokenIDs that all have the same low 96 bits. If we're + // in that case then we need to compare against the full + // 256 bits. + uint256 const nftokenID = o[sfNFTokenID]; + uint256 const maskedNftokenID = nftokenID & nft::pageMask; + + if (!pastMarker && maskedNftokenID < maskedMarker) continue; + if (!pastMarker && maskedNftokenID == maskedMarker && + nftokenID <= marker) + continue; + + pastMarker = true; + { Json::Value& obj = nfts.append(o.getJson(JsonOptions::none)); // Pull out the components of the nft ID. - uint256 const nftokenID = o[sfNFTokenID]; obj[sfFlags.jsonName] = nft::getFlags(nftokenID); obj[sfIssuer.jsonName] = to_string(nft::getIssuer(nftokenID)); obj[sfNFTokenTaxon.jsonName] = @@ -139,7 +160,7 @@ doAccountNFTs(RPC::JsonContext& context) cp = nullptr; } - result[jss::account] = context.app.accountIDCache().toBase58(accountID); + result[jss::account] = toBase58(accountID); context.loadType = Resource::feeMediumBurdenRPC; return result; } @@ -254,7 +275,7 @@ doAccountObjects(RPC::JsonContext& context) result[jss::account_objects] = Json::arrayValue; } - result[jss::account] = context.app.accountIDCache().toBase58(accountID); + result[jss::account] = toBase58(accountID); context.loadType = Resource::feeMediumBurdenRPC; return result; } diff --git a/src/ripple/rpc/handlers/AccountOffers.cpp b/src/ripple/rpc/handlers/AccountOffers.cpp index d31787563..e957fe8a8 100644 --- a/src/ripple/rpc/handlers/AccountOffers.cpp +++ b/src/ripple/rpc/handlers/AccountOffers.cpp @@ -77,7 +77,7 @@ doAccountOffers(RPC::JsonContext& context) } // Get info on account. - result[jss::account] = context.app.accountIDCache().toBase58(accountID); + result[jss::account] = toBase58(accountID); if (!ledger->exists(keylet::account(accountID))) return rpcError(rpcACT_NOT_FOUND); diff --git a/src/ripple/rpc/handlers/AccountTx.cpp b/src/ripple/rpc/handlers/AccountTx.cpp index e383b66fe..67c80ad9b 100644 --- a/src/ripple/rpc/handlers/AccountTx.cpp +++ b/src/ripple/rpc/handlers/AccountTx.cpp @@ -21,8 +21,8 @@ #include #include #include -#include -#include +#include +#include #include #include #include @@ -35,84 +35,20 @@ #include #include #include -#include #include #include namespace ripple { -using TxnsData = RelationalDBInterface::AccountTxs; -using TxnsDataBinary = RelationalDBInterface::MetaTxsList; -using TxnDataBinary = RelationalDBInterface::txnMetaLedgerType; -using AccountTxArgs = RelationalDBInterface::AccountTxArgs; -using AccountTxResult = RelationalDBInterface::AccountTxResult; +using TxnsData = RelationalDatabase::AccountTxs; +using TxnsDataBinary = RelationalDatabase::MetaTxsList; +using TxnDataBinary = RelationalDatabase::txnMetaLedgerType; +using AccountTxArgs = RelationalDatabase::AccountTxArgs; +using AccountTxResult = RelationalDatabase::AccountTxResult; -using LedgerShortcut = RelationalDBInterface::LedgerShortcut; -using LedgerSpecifier = RelationalDBInterface::LedgerSpecifier; - -// parses args into a ledger specifier, or returns a grpc status object on error -std::variant, grpc::Status> -parseLedgerArgs( - org::xrpl::rpc::v1::GetAccountTransactionHistoryRequest const& params) -{ - grpc::Status status; - if (params.has_ledger_range()) - { - uint32_t min = params.ledger_range().ledger_index_min(); - uint32_t max = params.ledger_range().ledger_index_max(); - - // if min is set but not max, need to set max - if (min != 0 && max == 0) - { - max = UINT32_MAX; - } - - return LedgerRange{min, max}; - } - else if (params.has_ledger_specifier()) - { - LedgerSpecifier ledger; - - auto& specifier = params.ledger_specifier(); - using LedgerCase = org::xrpl::rpc::v1::LedgerSpecifier::LedgerCase; - LedgerCase ledgerCase = specifier.ledger_case(); - - if (ledgerCase == LedgerCase::kShortcut) - { - using LedgerSpecifier = org::xrpl::rpc::v1::LedgerSpecifier; - - if (specifier.shortcut() == LedgerSpecifier::SHORTCUT_VALIDATED) - ledger = LedgerShortcut::VALIDATED; - else if (specifier.shortcut() == LedgerSpecifier::SHORTCUT_CLOSED) - ledger = LedgerShortcut::CLOSED; - else if (specifier.shortcut() == LedgerSpecifier::SHORTCUT_CURRENT) - ledger = LedgerShortcut::CURRENT; - else - return {}; - } - else if (ledgerCase == LedgerCase::kSequence) - { - ledger = specifier.sequence(); - } - else if (ledgerCase == LedgerCase::kHash) - { - if (auto hash = uint256::fromVoidChecked(specifier.hash())) - { - ledger = *hash; - } - else - { - grpc::Status errorStatus{ - grpc::StatusCode::INVALID_ARGUMENT, - "ledger hash malformed"}; - return errorStatus; - } - } - return ledger; - } - return std::optional{}; -} +using LedgerShortcut = RelationalDatabase::LedgerShortcut; +using LedgerSpecifier = RelationalDatabase::LedgerSpecifier; // parses args into a ledger specifier, or returns a Json object on error std::variant, Json::Value> @@ -257,9 +193,15 @@ doAccountTxHelp(RPC::Context& context, AccountTxArgs const& args) { context.loadType = Resource::feeMediumBurdenRPC; if (context.app.config().reporting()) - return dynamic_cast( - &context.app.getRelationalDBInterface()) - ->getAccountTx(args); + { + auto const db = dynamic_cast( + &context.app.getRelationalDatabase()); + + if (!db) + Throw("Failed to get relational database"); + + return db->getAccountTx(args); + } AccountTxResult result; @@ -274,7 +216,7 @@ doAccountTxHelp(RPC::Context& context, AccountTxArgs const& args) result.marker = args.marker; - RelationalDBInterface::AccountTxPageOptions options = { + RelationalDatabase::AccountTxPageOptions options = { args.account, result.ledgerRange.min, result.ledgerRange.max, @@ -282,21 +224,23 @@ doAccountTxHelp(RPC::Context& context, AccountTxArgs const& args) args.limit, isUnlimited(context.role)}; + auto const db = + dynamic_cast(&context.app.getRelationalDatabase()); + + if (!db) + Throw("Failed to get relational database"); + if (args.binary) { if (args.forward) { - auto [tx, marker] = dynamic_cast( - &context.app.getRelationalDBInterface()) - ->oldestAccountTxPageB(options); + auto [tx, marker] = db->oldestAccountTxPageB(options); result.transactions = tx; result.marker = marker; } else { - auto [tx, marker] = dynamic_cast( - &context.app.getRelationalDBInterface()) - ->newestAccountTxPageB(options); + auto [tx, marker] = db->newestAccountTxPageB(options); result.transactions = tx; result.marker = marker; } @@ -305,17 +249,13 @@ doAccountTxHelp(RPC::Context& context, AccountTxArgs const& args) { if (args.forward) { - auto [tx, marker] = dynamic_cast( - &context.app.getRelationalDBInterface()) - ->oldestAccountTxPage(options); + auto [tx, marker] = db->oldestAccountTxPage(options); result.transactions = tx; result.marker = marker; } else { - auto [tx, marker] = dynamic_cast( - &context.app.getRelationalDBInterface()) - ->newestAccountTxPage(options); + auto [tx, marker] = db->newestAccountTxPage(options); result.transactions = tx; result.marker = marker; } @@ -327,131 +267,6 @@ doAccountTxHelp(RPC::Context& context, AccountTxArgs const& args) return {result, rpcSUCCESS}; } -std::pair< - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse, - grpc::Status> -populateProtoResponse( - std::pair const& res, - AccountTxArgs const& args, - RPC::GRPCContext< - org::xrpl::rpc::v1::GetAccountTransactionHistoryRequest> const& context) -{ - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse response; - grpc::Status status = grpc::Status::OK; - - RPC::Status const& error = res.second; - if (error.toErrorCode() != rpcSUCCESS) - { - if (error.toErrorCode() == rpcLGR_NOT_FOUND) - { - status = {grpc::StatusCode::NOT_FOUND, error.message()}; - } - else if (error.toErrorCode() == rpcNOT_SYNCED) - { - status = {grpc::StatusCode::FAILED_PRECONDITION, error.message()}; - } - else - { - status = {grpc::StatusCode::INVALID_ARGUMENT, error.message()}; - } - } - else - { - AccountTxResult const& result = res.first; - - // account_tx always returns validated data - response.set_validated(true); - response.set_limit(result.limit); - response.mutable_account()->set_address( - context.params.account().address()); - response.set_ledger_index_min(result.ledgerRange.min); - response.set_ledger_index_max(result.ledgerRange.max); - - if (auto txnsData = std::get_if(&result.transactions)) - { - assert(!args.binary); - for (auto const& [txn, txnMeta] : *txnsData) - { - if (txn) - { - auto txnProto = response.add_transactions(); - - RPC::convert( - *txnProto->mutable_transaction(), - txn->getSTransaction()); - - // account_tx always returns validated data - txnProto->set_validated(true); - txnProto->set_ledger_index(txn->getLedger()); - auto& hash = txn->getID(); - txnProto->set_hash(hash.data(), hash.size()); - auto closeTime = - context.app.getLedgerMaster().getCloseTimeBySeq( - txn->getLedger()); - if (closeTime) - txnProto->mutable_date()->set_value( - closeTime->time_since_epoch().count()); - if (txnMeta) - { - RPC::convert(*txnProto->mutable_meta(), txnMeta); - if (!txnProto->meta().has_delivered_amount()) - { - if (auto amt = getDeliveredAmount( - context, - txn->getSTransaction(), - *txnMeta, - txn->getLedger())) - { - RPC::convert( - *txnProto->mutable_meta() - ->mutable_delivered_amount(), - *amt); - } - } - } - } - } - } - else - { - assert(args.binary); - - for (auto const& binaryData : - std::get(result.transactions)) - { - auto txnProto = response.add_transactions(); - Blob const& txnBlob = std::get<0>(binaryData); - txnProto->set_transaction_binary( - txnBlob.data(), txnBlob.size()); - - Blob const& metaBlob = std::get<1>(binaryData); - txnProto->set_meta_binary(metaBlob.data(), metaBlob.size()); - - txnProto->set_ledger_index(std::get<2>(binaryData)); - - // account_tx always returns validated data - txnProto->set_validated(true); - - auto closeTime = - context.app.getLedgerMaster().getCloseTimeBySeq( - std::get<2>(binaryData)); - if (closeTime) - txnProto->mutable_date()->set_value( - closeTime->time_since_epoch().count()); - } - } - - if (result.marker) - { - response.mutable_marker()->set_ledger_index( - result.marker->ledgerSeq); - response.mutable_marker()->set_account_sequence( - result.marker->txnSeq); - } - } - return {response, status}; -} - Json::Value populateJsonResponse( std::pair const& res, @@ -593,59 +408,4 @@ doAccountTxJson(RPC::JsonContext& context) return populateJsonResponse(res, args, context); } -std::pair< - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse, - grpc::Status> -doAccountTxGrpc( - RPC::GRPCContext& - context) -{ - if (!context.app.config().useTxTables()) - { - return { - {}, - {grpc::StatusCode::UNIMPLEMENTED, "Not enabled in configuration."}}; - } - - // return values - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse response; - grpc::Status status = grpc::Status::OK; - AccountTxArgs args; - - auto& request = context.params; - - auto const account = parseBase58(request.account().address()); - if (!account) - { - return { - {}, - {grpc::StatusCode::INVALID_ARGUMENT, "Could not decode account"}}; - } - - args.account = *account; - args.limit = request.limit(); - args.binary = request.binary(); - args.forward = request.forward(); - - if (request.has_marker()) - { - args.marker = { - request.marker().ledger_index(), - request.marker().account_sequence()}; - } - - auto parseRes = parseLedgerArgs(request); - if (auto stat = std::get_if(&parseRes)) - { - return {response, *stat}; - } - else - { - args.ledger = std::get>(parseRes); - } - - auto res = doAccountTxHelp(context, args); - return populateProtoResponse(res, args, context); -} - } // namespace ripple diff --git a/src/ripple/rpc/handlers/AccountTxOld.cpp b/src/ripple/rpc/handlers/AccountTxOld.cpp deleted file mode 100644 index 9c5bb0bce..000000000 --- a/src/ripple/rpc/handlers/AccountTxOld.cpp +++ /dev/null @@ -1,255 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012-2014 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { - -// { -// account: account, -// ledger_index_min: ledger_index, -// ledger_index_max: ledger_index, -// binary: boolean, // optional, defaults to false -// count: boolean, // optional, defaults to false -// descending: boolean, // optional, defaults to false -// offset: integer, // optional, defaults to 0 -// limit: integer // optional -// } -Json::Value -doAccountTxOld(RPC::JsonContext& context) -{ - std::uint32_t offset = context.params.isMember(jss::offset) - ? context.params[jss::offset].asUInt() - : 0; - std::uint32_t limit = context.params.isMember(jss::limit) - ? context.params[jss::limit].asUInt() - : UINT32_MAX; - bool bBinary = context.params.isMember(jss::binary) && - context.params[jss::binary].asBool(); - bool bDescending = context.params.isMember(jss::descending) && - context.params[jss::descending].asBool(); - bool bCount = context.params.isMember(jss::count) && - context.params[jss::count].asBool(); - std::uint32_t uLedgerMin; - std::uint32_t uLedgerMax; - std::uint32_t uValidatedMin; - std::uint32_t uValidatedMax; - bool bValidated = - context.ledgerMaster.getValidatedRange(uValidatedMin, uValidatedMax); - - if (!context.params.isMember(jss::account)) - return rpcError(rpcINVALID_PARAMS); - - auto const raAccount = - parseBase58(context.params[jss::account].asString()); - if (!raAccount) - return rpcError(rpcACT_MALFORMED); - - if (offset > 3000) - return rpcError(rpcATX_DEPRECATED); - - context.loadType = Resource::feeHighBurdenRPC; - - // DEPRECATED - if (context.params.isMember(jss::ledger_min)) - { - context.params[jss::ledger_index_min] = context.params[jss::ledger_min]; - bDescending = true; - } - - // DEPRECATED - if (context.params.isMember(jss::ledger_max)) - { - context.params[jss::ledger_index_max] = context.params[jss::ledger_max]; - bDescending = true; - } - - if (context.params.isMember(jss::ledger_index_min) || - context.params.isMember(jss::ledger_index_max)) - { - std::int64_t iLedgerMin = context.params.isMember(jss::ledger_index_min) - ? context.params[jss::ledger_index_min].asInt() - : -1; - std::int64_t iLedgerMax = context.params.isMember(jss::ledger_index_max) - ? context.params[jss::ledger_index_max].asInt() - : -1; - - if (!bValidated && (iLedgerMin == -1 || iLedgerMax == -1)) - { - // Don't have a validated ledger range. - if (context.apiVersion == 1) - return rpcError(rpcLGR_IDXS_INVALID); - return rpcError(rpcNOT_SYNCED); - } - - uLedgerMin = iLedgerMin == -1 ? uValidatedMin : iLedgerMin; - uLedgerMax = iLedgerMax == -1 ? uValidatedMax : iLedgerMax; - - if (uLedgerMax < uLedgerMin) - { - if (context.apiVersion == 1) - return rpcError(rpcLGR_IDXS_INVALID); - return rpcError(rpcNOT_SYNCED); - } - } - else - { - std::shared_ptr ledger; - auto ret = RPC::lookupLedger(ledger, context); - - if (!ledger) - return ret; - - if (!ret[jss::validated].asBool() || - (ledger->info().seq > uValidatedMax) || - (ledger->info().seq < uValidatedMin)) - { - return rpcError(rpcLGR_NOT_VALIDATED); - } - - uLedgerMin = uLedgerMax = ledger->info().seq; - } - - int count = 0; - -#ifndef DEBUG - - try - { -#endif - - Json::Value ret(Json::objectValue); - - ret[jss::account] = context.app.accountIDCache().toBase58(*raAccount); - Json::Value& jvTxns = (ret[jss::transactions] = Json::arrayValue); - - RelationalDBInterface::AccountTxOptions options = { - *raAccount, - uLedgerMin, - uLedgerMax, - offset, - limit, - isUnlimited(context.role)}; - - if (bBinary) - { - std::vector txns; - - if (bDescending) - txns = dynamic_cast( - &context.app.getRelationalDBInterface()) - ->getNewestAccountTxsB(options); - else - txns = dynamic_cast( - &context.app.getRelationalDBInterface()) - ->getOldestAccountTxsB(options); - - for (auto it = txns.begin(), end = txns.end(); it != end; ++it) - { - ++count; - Json::Value& jvObj = jvTxns.append(Json::objectValue); - - std::uint32_t uLedgerIndex = std::get<2>(*it); - jvObj[jss::tx_blob] = strHex(std::get<0>(*it)); - jvObj[jss::meta] = strHex(std::get<1>(*it)); - jvObj[jss::ledger_index] = uLedgerIndex; - jvObj[jss::validated] = bValidated && - uValidatedMin <= uLedgerIndex && - uValidatedMax >= uLedgerIndex; - } - } - else - { - RelationalDBInterface::AccountTxs txns; - - if (bDescending) - txns = dynamic_cast( - &context.app.getRelationalDBInterface()) - ->getNewestAccountTxs(options); - else - txns = dynamic_cast( - &context.app.getRelationalDBInterface()) - ->getOldestAccountTxs(options); - - for (auto it = txns.begin(), end = txns.end(); it != end; ++it) - { - ++count; - Json::Value& jvObj = jvTxns.append(Json::objectValue); - - if (it->first) - jvObj[jss::tx] = - it->first->getJson(JsonOptions::include_date); - - if (it->second) - { - std::uint32_t uLedgerIndex = it->second->getLgrSeq(); - - auto meta = it->second->getJson(JsonOptions::none); - insertDeliveredAmount( - meta, context, it->first, *it->second); - jvObj[jss::meta] = std::move(meta); - - jvObj[jss::validated] = bValidated && - uValidatedMin <= uLedgerIndex && - uValidatedMax >= uLedgerIndex; - } - } - } - - // Add information about the original query - ret[jss::ledger_index_min] = uLedgerMin; - ret[jss::ledger_index_max] = uLedgerMax; - ret[jss::validated] = bValidated && uValidatedMin <= uLedgerMin && - uValidatedMax >= uLedgerMax; - ret[jss::offset] = offset; - - // We no longer return the full count but only the count of returned - // transactions. Computing this count was two expensive and this API is - // deprecated anyway. - if (bCount) - ret[jss::count] = count; - - if (context.params.isMember(jss::limit)) - ret[jss::limit] = limit; - - return ret; -#ifndef DEBUG - } - catch (std::exception const&) - { - return rpcError(rpcINTERNAL); - } - -#endif -} - -} // namespace ripple diff --git a/src/ripple/rpc/handlers/BookOffers.cpp b/src/ripple/rpc/handlers/BookOffers.cpp index a42a0de99..e85b6029b 100644 --- a/src/ripple/rpc/handlers/BookOffers.cpp +++ b/src/ripple/rpc/handlers/BookOffers.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -200,4 +201,16 @@ doBookOffers(RPC::JsonContext& context) return jvResult; } +Json::Value +doBookChanges(RPC::JsonContext& context) +{ + auto res = RPC::getLedgerByContext(context); + + if (std::holds_alternative(res)) + return std::get(res); + + return RPC::computeBookChanges( + std::get>(res)); +} + } // namespace ripple diff --git a/src/ripple/rpc/handlers/Fee1.cpp b/src/ripple/rpc/handlers/Fee1.cpp index f28f9b447..554480f10 100644 --- a/src/ripple/rpc/handlers/Fee1.cpp +++ b/src/ripple/rpc/handlers/Fee1.cpp @@ -38,49 +38,4 @@ doFee(RPC::JsonContext& context) return context.params; } -std::pair -doFeeGrpc(RPC::GRPCContext& context) -{ - org::xrpl::rpc::v1::GetFeeResponse reply; - grpc::Status status = grpc::Status::OK; - - Application& app = context.app; - auto const view = app.openLedger().current(); - if (!view) - { - BOOST_ASSERT(false); - return {reply, status}; - } - - auto const metrics = app.getTxQ().getMetrics(*view); - - // current ledger data - reply.set_current_ledger_size(metrics.txInLedger); - reply.set_current_queue_size(metrics.txCount); - reply.set_expected_ledger_size(metrics.txPerLedger); - reply.set_ledger_current_index(view->info().seq); - reply.set_max_queue_size(*metrics.txQMaxSize); - - // fee levels data - org::xrpl::rpc::v1::FeeLevels& levels = *reply.mutable_levels(); - levels.set_median_level(metrics.medFeeLevel.fee()); - levels.set_minimum_level(metrics.minProcessingFeeLevel.fee()); - levels.set_open_ledger_level(metrics.openLedgerFeeLevel.fee()); - levels.set_reference_level(metrics.referenceFeeLevel.fee()); - - // fee data - org::xrpl::rpc::v1::Fee& fee = *reply.mutable_fee(); - auto const baseFee = view->fees().base; - fee.mutable_base_fee()->set_drops( - toDrops(metrics.referenceFeeLevel, baseFee).drops()); - fee.mutable_minimum_fee()->set_drops( - toDrops(metrics.minProcessingFeeLevel, baseFee).drops()); - fee.mutable_median_fee()->set_drops( - toDrops(metrics.medFeeLevel, baseFee).drops()); - - fee.mutable_open_ledger_fee()->set_drops( - (toDrops(metrics.openLedgerFeeLevel - FeeLevel64{1}, baseFee) + 1) - .drops()); - return {reply, status}; -} } // namespace ripple diff --git a/src/ripple/rpc/handlers/GatewayBalances.cpp b/src/ripple/rpc/handlers/GatewayBalances.cpp index 825a74ab8..d0770f31e 100644 --- a/src/ripple/rpc/handlers/GatewayBalances.cpp +++ b/src/ripple/rpc/handlers/GatewayBalances.cpp @@ -80,7 +80,7 @@ doGatewayBalances(RPC::JsonContext& context) context.loadType = Resource::feeHighBurdenRPC; - result[jss::account] = context.app.accountIDCache().toBase58(accountID); + result[jss::account] = toBase58(accountID); // Parse the specified hotwallet(s), if any std::set hotWallets; diff --git a/src/ripple/rpc/handlers/GetCounts.cpp b/src/ripple/rpc/handlers/GetCounts.cpp index acb306449..cf3e72902 100644 --- a/src/ripple/rpc/handlers/GetCounts.cpp +++ b/src/ripple/rpc/handlers/GetCounts.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include @@ -75,23 +75,23 @@ getCountsJson(Application& app, int minObjectCount) if (!app.config().reporting() && app.config().useTxTables()) { - auto dbKB = dynamic_cast( - &app.getRelationalDBInterface()) - ->getKBUsedAll(); + auto const db = + dynamic_cast(&app.getRelationalDatabase()); + + if (!db) + Throw("Failed to get relational database"); + + auto dbKB = db->getKBUsedAll(); if (dbKB > 0) ret[jss::dbKBTotal] = dbKB; - dbKB = dynamic_cast( - &app.getRelationalDBInterface()) - ->getKBUsedLedger(); + dbKB = db->getKBUsedLedger(); if (dbKB > 0) ret[jss::dbKBLedger] = dbKB; - dbKB = dynamic_cast( - &app.getRelationalDBInterface()) - ->getKBUsedTransaction(); + dbKB = db->getKBUsedTransaction(); if (dbKB > 0) ret[jss::dbKBTransaction] = dbKB; diff --git a/src/ripple/rpc/handlers/Handlers.h b/src/ripple/rpc/handlers/Handlers.h index 1bb3be056..3c00899d7 100644 --- a/src/ripple/rpc/handlers/Handlers.h +++ b/src/ripple/rpc/handlers/Handlers.h @@ -43,6 +43,8 @@ doAccountTxJson(RPC::JsonContext&); Json::Value doBookOffers(RPC::JsonContext&); Json::Value +doBookChanges(RPC::JsonContext&); +Json::Value doBlackList(RPC::JsonContext&); Json::Value doCanDelete(RPC::JsonContext&); diff --git a/src/ripple/rpc/handlers/LedgerData.cpp b/src/ripple/rpc/handlers/LedgerData.cpp index fbc17b75b..7392b5051 100644 --- a/src/ripple/rpc/handlers/LedgerData.cpp +++ b/src/ripple/rpc/handlers/LedgerData.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include diff --git a/src/ripple/rpc/handlers/LedgerEntry.cpp b/src/ripple/rpc/handlers/LedgerEntry.cpp index 4b2526698..fff2ceac0 100644 --- a/src/ripple/rpc/handlers/LedgerEntry.cpp +++ b/src/ripple/rpc/handlers/LedgerEntry.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include namespace ripple { diff --git a/src/ripple/rpc/handlers/LedgerHandler.cpp b/src/ripple/rpc/handlers/LedgerHandler.cpp index e28b181fc..6b4fc7736 100644 --- a/src/ripple/rpc/handlers/LedgerHandler.cpp +++ b/src/ripple/rpc/handlers/LedgerHandler.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include namespace ripple { @@ -134,27 +133,38 @@ doLedgerGrpc(RPC::GRPCContext& context) if (request.transactions()) { - for (auto& i : ledger->txs) + try { - assert(i.first); - if (request.expand()) + for (auto& i : ledger->txs) { - auto txn = - response.mutable_transactions_list()->add_transactions(); - Serializer sTxn = i.first->getSerializer(); - txn->set_transaction_blob(sTxn.data(), sTxn.getLength()); - if (i.second) + assert(i.first); + if (request.expand()) { - Serializer sMeta = i.second->getSerializer(); - txn->set_metadata_blob(sMeta.data(), sMeta.getLength()); + auto txn = response.mutable_transactions_list() + ->add_transactions(); + Serializer sTxn = i.first->getSerializer(); + txn->set_transaction_blob(sTxn.data(), sTxn.getLength()); + if (i.second) + { + Serializer sMeta = i.second->getSerializer(); + txn->set_metadata_blob(sMeta.data(), sMeta.getLength()); + } + } + else + { + auto const& hash = i.first->getTransactionID(); + response.mutable_hashes_list()->add_hashes( + hash.data(), hash.size()); } } - else - { - auto const& hash = i.first->getTransactionID(); - response.mutable_hashes_list()->add_hashes( - hash.data(), hash.size()); - } + } + catch (std::exception const& e) + { + JLOG(context.j.error()) + << __func__ << " - Error deserializing transaction in ledger " + << ledger->info().seq + << " . skipping transaction and following transactions. You " + "should look into this further"; } } diff --git a/src/ripple/rpc/handlers/LedgerRequest.cpp b/src/ripple/rpc/handlers/LedgerRequest.cpp index 3fe238e92..88d26176d 100644 --- a/src/ripple/rpc/handlers/LedgerRequest.cpp +++ b/src/ripple/rpc/handlers/LedgerRequest.cpp @@ -26,7 +26,9 @@ #include #include #include +#include #include +#include namespace ripple { @@ -37,122 +39,17 @@ namespace ripple { Json::Value doLedgerRequest(RPC::JsonContext& context) { - if (context.app.config().reporting()) - return rpcError(rpcREPORTING_UNSUPPORTED); + auto res = getLedgerByContext(context); - auto const hasHash = context.params.isMember(jss::ledger_hash); - auto const hasIndex = context.params.isMember(jss::ledger_index); - std::uint32_t ledgerIndex = 0; + if (std::holds_alternative(res)) + return std::get(res); - auto& ledgerMaster = context.app.getLedgerMaster(); - LedgerHash ledgerHash; + auto const& ledger = std::get>(res); - if ((hasHash && hasIndex) || !(hasHash || hasIndex)) - { - return RPC::make_param_error( - "Exactly one of ledger_hash and ledger_index can be set."); - } - - context.loadType = Resource::feeHighBurdenRPC; - - if (hasHash) - { - auto const& jsonHash = context.params[jss::ledger_hash]; - if (!jsonHash.isString() || !ledgerHash.parseHex(jsonHash.asString())) - return RPC::invalid_field_error(jss::ledger_hash); - } - else - { - auto const& jsonIndex = context.params[jss::ledger_index]; - if (!jsonIndex.isInt()) - return RPC::invalid_field_error(jss::ledger_index); - - // We need a validated ledger to get the hash from the sequence - if (ledgerMaster.getValidatedLedgerAge() > - RPC::Tuning::maxValidatedLedgerAge) - { - if (context.apiVersion == 1) - return rpcError(rpcNO_CURRENT); - return rpcError(rpcNOT_SYNCED); - } - - ledgerIndex = jsonIndex.asInt(); - auto ledger = ledgerMaster.getValidatedLedger(); - - if (ledgerIndex >= ledger->info().seq) - return RPC::make_param_error("Ledger index too large"); - if (ledgerIndex <= 0) - return RPC::make_param_error("Ledger index too small"); - - auto const j = context.app.journal("RPCHandler"); - // Try to get the hash of the desired ledger from the validated ledger - auto neededHash = hashOfSeq(*ledger, ledgerIndex, j); - if (!neededHash) - { - // Find a ledger more likely to have the hash of the desired ledger - auto const refIndex = getCandidateLedger(ledgerIndex); - auto refHash = hashOfSeq(*ledger, refIndex, j); - assert(refHash); - - ledger = ledgerMaster.getLedgerByHash(*refHash); - if (!ledger) - { - // We don't have the ledger we need to figure out which ledger - // they want. Try to get it. - - if (auto il = context.app.getInboundLedgers().acquire( - *refHash, refIndex, InboundLedger::Reason::GENERIC)) - { - Json::Value jvResult = RPC::make_error( - rpcLGR_NOT_FOUND, - "acquiring ledger containing requested index"); - jvResult[jss::acquiring] = - getJson(LedgerFill(*il, &context)); - return jvResult; - } - - if (auto il = context.app.getInboundLedgers().find(*refHash)) - { - Json::Value jvResult = RPC::make_error( - rpcLGR_NOT_FOUND, - "acquiring ledger containing requested index"); - jvResult[jss::acquiring] = il->getJson(0); - return jvResult; - } - - // Likely the app is shutting down - return Json::Value(); - } - - neededHash = hashOfSeq(*ledger, ledgerIndex, j); - } - assert(neededHash); - ledgerHash = neededHash ? *neededHash : beast::zero; // kludge - } - - // Try to get the desired ledger - // Verify all nodes even if we think we have it - auto ledger = context.app.getInboundLedgers().acquire( - ledgerHash, ledgerIndex, InboundLedger::Reason::GENERIC); - - // In standalone mode, accept the ledger from the ledger cache - if (!ledger && context.app.config().standalone()) - ledger = ledgerMaster.getLedgerByHash(ledgerHash); - - if (ledger) - { - // We already had the entire ledger verified/acquired - Json::Value jvResult; - jvResult[jss::ledger_index] = ledger->info().seq; - addJson(jvResult, {*ledger, &context, 0}); - return jvResult; - } - - if (auto il = context.app.getInboundLedgers().find(ledgerHash)) - return il->getJson(0); - - return RPC::make_error( - rpcNOT_READY, "findCreate failed to return an inbound ledger"); + Json::Value jvResult; + jvResult[jss::ledger_index] = ledger->info().seq; + addJson(jvResult, {*ledger, &context, 0}); + return jvResult; } } // namespace ripple diff --git a/src/ripple/rpc/handlers/NFTOffers.cpp b/src/ripple/rpc/handlers/NFTOffers.cpp index 34bbc8446..69a090e27 100644 --- a/src/ripple/rpc/handlers/NFTOffers.cpp +++ b/src/ripple/rpc/handlers/NFTOffers.cpp @@ -41,12 +41,10 @@ appendNftOfferJson( obj[jss::nft_offer_index] = to_string(offer->key()); obj[jss::flags] = (*offer)[sfFlags]; - obj[jss::owner] = - app.accountIDCache().toBase58(offer->getAccountID(sfOwner)); + obj[jss::owner] = toBase58(offer->getAccountID(sfOwner)); if (offer->isFieldPresent(sfDestination)) - obj[jss::destination] = - app.accountIDCache().toBase58(offer->getAccountID(sfDestination)); + obj[jss::destination] = toBase58(offer->getAccountID(sfDestination)); if (offer->isFieldPresent(sfExpiration)) obj[jss::expiration] = offer->getFieldU32(sfExpiration); diff --git a/src/ripple/rpc/handlers/NoRippleCheck.cpp b/src/ripple/rpc/handlers/NoRippleCheck.cpp index 2a6ab7ca4..a2af9845f 100644 --- a/src/ripple/rpc/handlers/NoRippleCheck.cpp +++ b/src/ripple/rpc/handlers/NoRippleCheck.cpp @@ -40,7 +40,7 @@ fillTransaction( ReadView const& ledger) { txArray["Sequence"] = Json::UInt(sequence++); - txArray["Account"] = context.app.accountIDCache().toBase58(accountID); + txArray["Account"] = toBase58(accountID); auto& fees = ledger.fees(); // Convert the reference transaction cost in fee units to drops // scaled to represent the current fee load. diff --git a/src/ripple/rpc/handlers/Submit.cpp b/src/ripple/rpc/handlers/Submit.cpp index 220760172..2b5c8bba9 100644 --- a/src/ripple/rpc/handlers/Submit.cpp +++ b/src/ripple/rpc/handlers/Submit.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include @@ -194,102 +193,4 @@ doSubmit(RPC::JsonContext& context) } } -std::pair -doSubmitGrpc( - RPC::GRPCContext& context) -{ - // return values - org::xrpl::rpc::v1::SubmitTransactionResponse result; - grpc::Status status = grpc::Status::OK; - - // input - auto request = context.params; - - std::string const& tx = request.signed_transaction(); - - // convert to blob - Blob blob{tx.begin(), tx.end()}; - - // serialize - SerialIter sitTrans(makeSlice(blob)); - std::shared_ptr stpTrans; - try - { - stpTrans = std::make_shared(std::ref(sitTrans)); - } - catch (std::exception& e) - { - grpc::Status errorStatus{ - grpc::StatusCode::INVALID_ARGUMENT, - "invalid transaction: " + std::string(e.what())}; - return {result, errorStatus}; - } - - // check validity - { - if (!context.app.checkSigs()) - forceValidity( - context.app.getHashRouter(), - stpTrans->getTransactionID(), - Validity::SigGoodOnly); - auto [validity, reason] = checkValidity( - context.app.getHashRouter(), - *stpTrans, - context.ledgerMaster.getCurrentLedger()->rules(), - context.app.config()); - if (validity != Validity::Valid) - { - grpc::Status errorStatus{ - grpc::StatusCode::INVALID_ARGUMENT, - "invalid transaction: " + reason}; - return {result, errorStatus}; - } - } - - std::string reason; - auto tpTrans = std::make_shared(stpTrans, reason, context.app); - if (tpTrans->getStatus() != NEW) - { - grpc::Status errorStatus{ - grpc::StatusCode::INVALID_ARGUMENT, - "invalid transaction: " + reason}; - return {result, errorStatus}; - } - - try - { - auto const failType = NetworkOPs::doFailHard(request.fail_hard()); - - // submit to network - context.netOps.processTransaction( - tpTrans, isUnlimited(context.role), true, failType); - } - catch (std::exception& e) - { - grpc::Status errorStatus{ - grpc::StatusCode::INVALID_ARGUMENT, - "invalid transaction : " + std::string(e.what())}; - return {result, errorStatus}; - } - - // return preliminary result - if (temUNCERTAIN != tpTrans->getResult()) - { - RPC::convert(*result.mutable_engine_result(), tpTrans->getResult()); - - std::string sToken; - std::string sHuman; - - transResultInfo(tpTrans->getResult(), sToken, sHuman); - - result.mutable_engine_result()->set_result(sToken); - result.set_engine_result_code(TERtoInt(tpTrans->getResult())); - result.set_engine_result_message(sHuman); - - uint256 hash = tpTrans->getID(); - result.set_hash(hash.data(), hash.size()); - } - return {result, status}; -} - } // namespace ripple diff --git a/src/ripple/rpc/handlers/Subscribe.cpp b/src/ripple/rpc/handlers/Subscribe.cpp index a3c1d1e1c..f17aa62b6 100644 --- a/src/ripple/rpc/handlers/Subscribe.cpp +++ b/src/ripple/rpc/handlers/Subscribe.cpp @@ -136,6 +136,10 @@ doSubscribe(RPC::JsonContext& context) { context.netOps.subLedger(ispSub, jvResult); } + else if (streamName == "book_changes") + { + context.netOps.subBookChanges(ispSub); + } else if (streamName == "manifests") { context.netOps.subManifests(ispSub); diff --git a/src/ripple/rpc/handlers/Tx.cpp b/src/ripple/rpc/handlers/Tx.cpp index aa941d795..4a70f1fe0 100644 --- a/src/ripple/rpc/handlers/Tx.cpp +++ b/src/ripple/rpc/handlers/Tx.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include namespace ripple { @@ -251,101 +250,6 @@ doTxHelp(RPC::Context& context, TxArgs const& args) return {result, rpcSUCCESS}; } -std::pair -populateProtoResponse( - std::pair const& res, - TxArgs const& args, - RPC::GRPCContext const& context) -{ - org::xrpl::rpc::v1::GetTransactionResponse response; - grpc::Status status = grpc::Status::OK; - RPC::Status const& error = res.second; - TxResult const& result = res.first; - // handle errors - if (error.toErrorCode() != rpcSUCCESS) - { - if (error.toErrorCode() == rpcTXN_NOT_FOUND && - result.searchedAll != TxSearched::unknown) - { - status = { - grpc::StatusCode::NOT_FOUND, - "txn not found. searched_all = " + - to_string( - (result.searchedAll == TxSearched::all ? "true" - : "false"))}; - } - else - { - if (error.toErrorCode() == rpcTXN_NOT_FOUND) - status = {grpc::StatusCode::NOT_FOUND, "txn not found"}; - else - status = {grpc::StatusCode::INTERNAL, error.message()}; - } - } - // no errors - else if (result.txn) - { - auto& txn = result.txn; - - std::shared_ptr stTxn = txn->getSTransaction(); - if (args.binary) - { - Serializer s = stTxn->getSerializer(); - response.set_transaction_binary(s.data(), s.size()); - } - else - { - RPC::convert(*response.mutable_transaction(), stTxn); - } - - response.set_hash(context.params.hash()); - - auto ledgerIndex = txn->getLedger(); - response.set_ledger_index(ledgerIndex); - if (ledgerIndex) - { - auto ct = - context.app.getLedgerMaster().getCloseTimeBySeq(ledgerIndex); - if (ct) - response.mutable_date()->set_value( - ct->time_since_epoch().count()); - } - - RPC::convert( - *response.mutable_meta()->mutable_transaction_result(), - txn->getResult()); - response.mutable_meta()->mutable_transaction_result()->set_result( - transToken(txn->getResult())); - - // populate binary metadata - if (auto blob = std::get_if(&result.meta)) - { - assert(args.binary); - Slice slice = makeSlice(*blob); - response.set_meta_binary(slice.data(), slice.size()); - } - // populate meta data - else if (auto m = std::get_if>(&result.meta)) - { - auto& meta = *m; - if (meta) - { - RPC::convert(*response.mutable_meta(), meta); - auto amt = - getDeliveredAmount(context, stTxn, *meta, txn->getLedger()); - if (amt) - { - RPC::convert( - *response.mutable_meta()->mutable_delivered_amount(), - *amt); - } - } - } - response.set_validated(result.validated); - } - return {response, status}; -} - Json::Value populateJsonResponse( std::pair const& res, @@ -437,48 +341,4 @@ doTxJson(RPC::JsonContext& context) return populateJsonResponse(res, args, context); } -std::pair -doTxGrpc(RPC::GRPCContext& context) -{ - if (!context.app.config().useTxTables()) - { - return { - {}, - {grpc::StatusCode::UNIMPLEMENTED, "Not enabled in configuration."}}; - } - - // return values - org::xrpl::rpc::v1::GetTransactionResponse response; - grpc::Status status = grpc::Status::OK; - - // input - org::xrpl::rpc::v1::GetTransactionRequest& request = context.params; - - TxArgs args; - - if (auto hash = uint256::fromVoidChecked(request.hash())) - { - args.hash = *hash; - } - else - { - grpc::Status errorStatus{ - grpc::StatusCode::INVALID_ARGUMENT, "tx hash malformed"}; - return {response, errorStatus}; - } - - args.binary = request.binary(); - - if (request.ledger_range().ledger_index_min() != 0 && - request.ledger_range().ledger_index_max() != 0) - { - args.ledgerRange = std::make_pair( - request.ledger_range().ledger_index_min(), - request.ledger_range().ledger_index_max()); - } - - std::pair res = doTxHelp(context, args); - return populateProtoResponse(res, args, context); -} - } // namespace ripple diff --git a/src/ripple/rpc/handlers/TxHistory.cpp b/src/ripple/rpc/handlers/TxHistory.cpp index 7fa7fc76f..4c76bfac0 100644 --- a/src/ripple/rpc/handlers/TxHistory.cpp +++ b/src/ripple/rpc/handlers/TxHistory.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include @@ -54,8 +54,7 @@ doTxHistory(RPC::JsonContext& context) if ((startIndex > 10000) && (!isUnlimited(context.role))) return rpcError(rpcNO_PERMISSION); - auto trans = - context.app.getRelationalDBInterface().getTxHistory(startIndex); + auto trans = context.app.getRelationalDatabase().getTxHistory(startIndex); Json::Value obj; Json::Value& txs = obj[jss::txs]; diff --git a/src/ripple/rpc/impl/GRPCHelpers.cpp b/src/ripple/rpc/impl/GRPCHelpers.cpp deleted file mode 100644 index 558c9d535..000000000 --- a/src/ripple/rpc/impl/GRPCHelpers.cpp +++ /dev/null @@ -1,2183 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include - -namespace ripple { -namespace RPC { - -// In the below populateProto* functions, getProto is a function that returns -// a reference to the mutable protobuf message to be populated. The reason this -// is a function, as opposed to just a pointer or reference to the object, -// is that there is no way to get a non-const reference, and getting a pointer -// to the proto object causes default initialization of the object. However, -// if the corresponding field is not present in the STObject, we don't want to -// initialize the proto object. To get around this, getProto is a function that -// is called only if the field is present in the STObject -template -void -populateProtoPrimitive( - L const& getProto, - STObject const& from, - TypedField const& field) -{ - if (!from.isFieldPresent(field)) - return; - - if constexpr (std::is_integral_v) - { - getProto()->set_value(from[field]); - } - else - { - auto v = from[field]; - getProto()->set_value(v.data(), v.size()); - } -} - -template -void -populateProtoVLasString( - T const& getProto, - STObject const& from, - SF_VL const& field) -{ - if (from.isFieldPresent(field)) - { - auto data = from.getFieldVL(field); - getProto()->set_value( - reinterpret_cast(data.data()), data.size()); - } -} - -template -void -populateProtoVec256( - T const& getProto, - STObject const& from, - SF_VECTOR256 const& field) -{ - if (from.isFieldPresent(field)) - { - const STVector256& vec = from.getFieldV256(field); - for (size_t i = 0; i < vec.size(); ++i) - { - uint256 const& elt = vec[i]; - getProto()->set_value(elt.data(), elt.size()); - } - } -} - -template -void -populateProtoAccount( - T const& getProto, - STObject const& from, - SF_ACCOUNT const& field) -{ - if (from.isFieldPresent(field)) - { - getProto()->mutable_value()->set_address( - toBase58(from.getAccountID(field))); - } -} - -template -void -populateProtoAmount( - T const& getProto, - STObject const& from, - SF_AMOUNT const& field) -{ - if (from.isFieldPresent(field)) - { - auto amount = from.getFieldAmount(field); - convert(*getProto(), amount); - } -} - -template -void -populateProtoCurrency( - T const& getProto, - STObject const& from, - SF_UINT160 const& field) -{ - if (from.isFieldPresent(field)) - { - auto cur = from.getFieldH160(field); - auto proto = getProto()->mutable_value(); - proto->set_code(cur.data(), cur.size()); - proto->set_name(to_string(cur)); - } -} - -template -void -populateProtoArray( - T const& getProto, - R const& populateProto, - STObject const& from, - SField const& outerField, - SField const& innerField) -{ - if (from.isFieldPresent(outerField) && - from.peekAtField(outerField).getSType() == SerializedTypeID::STI_ARRAY) - { - auto arr = from.getFieldArray(outerField); - for (auto it = arr.begin(); it != arr.end(); ++it) - { - populateProto(*it, *getProto()); - } - } -} - -template -void -populateClearFlag(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_clear_flag(); }, from, sfClearFlag); -} - -template -void -populateDomain(T& to, STObject const& from) -{ - populateProtoVLasString( - [&to]() { return to.mutable_domain(); }, from, sfDomain); -} - -template -void -populateEmailHash(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_email_hash(); }, from, sfEmailHash); -} - -template -void -populateMessageKey(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_message_key(); }, from, sfMessageKey); -} - -template -void -populateSetFlag(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_set_flag(); }, from, sfSetFlag); -} - -template -void -populateTransferRate(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_transfer_rate(); }, from, sfTransferRate); -} - -template -void -populateTickSize(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_tick_size(); }, from, sfTickSize); -} - -template -void -populateExpiration(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_expiration(); }, from, sfExpiration); -} - -template -void -populateOfferSequence(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_offer_sequence(); }, from, sfOfferSequence); -} - -template -void -populateTakerGets(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_taker_gets(); }, from, sfTakerGets); -} - -template -void -populateTakerPays(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_taker_pays(); }, from, sfTakerPays); -} - -template -void -populateDestination(T& to, STObject const& from) -{ - populateProtoAccount( - [&to]() { return to.mutable_destination(); }, from, sfDestination); -} - -template -void -populateCheckID(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_check_id(); }, from, sfCheckID); -} - -template -void -populateAmount(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_amount(); }, from, sfAmount); -} - -template -void -populateDeliverMin(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_deliver_min(); }, from, sfDeliverMin); -} - -template -void -populateSendMax(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_send_max(); }, from, sfSendMax); -} - -template -void -populateDeliveredAmount(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_delivered_amount(); }, - from, - sfDeliveredAmount); -} - -template -void -populateDestinationTag(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_destination_tag(); }, - from, - sfDestinationTag); -} - -template -void -populateInvoiceID(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_invoice_id(); }, from, sfInvoiceID); -} - -template -void -populateAuthorize(T& to, STObject const& from) -{ - populateProtoAccount( - [&to]() { return to.mutable_authorize(); }, from, sfAuthorize); -} - -template -void -populateUnauthorize(T& to, STObject const& from) -{ - populateProtoAccount( - [&to]() { return to.mutable_unauthorize(); }, from, sfUnauthorize); -} - -template -void -populateOwner(T& to, STObject const& from) -{ - populateProtoAccount([&to]() { return to.mutable_owner(); }, from, sfOwner); -} - -template -void -populateCancelAfter(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_cancel_after(); }, from, sfCancelAfter); -} - -template -void -populateFinishAfter(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_finish_after(); }, from, sfFinishAfter); -} - -template -void -populateCondition(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_condition(); }, from, sfCondition); -} - -template -void -populateFulfillment(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_fulfillment(); }, from, sfFulfillment); -} - -template -void -populateChannel(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_channel(); }, from, sfChannel); -} - -template -void -populateBalance(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_balance(); }, from, sfBalance); -} - -template -void -populatePaymentChannelSignature(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_payment_channel_signature(); }, - from, - sfSignature); -} - -template -void -populatePublicKey(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_public_key(); }, from, sfPublicKey); -} - -template -void -populateSettleDelay(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_settle_delay(); }, from, sfSettleDelay); -} - -template -void -populateRegularKey(T& to, STObject const& from) -{ - populateProtoAccount( - [&to]() { return to.mutable_regular_key(); }, from, sfRegularKey); -} - -template -void -populateSignerQuorum(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_signer_quorum(); }, from, sfSignerQuorum); -} - -template -void -populateTicketCount(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_count(); }, from, sfTicketCount); -} - -template -void -populateLimitAmount(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_limit_amount(); }, from, sfLimitAmount); -} -template -void -populateQualityIn(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_quality_in(); }, from, sfQualityIn); -} - -template -void -populateQualityOut(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_quality_out(); }, from, sfQualityOut); -} - -template -void -populateAccount(T& to, STObject const& from) -{ - populateProtoAccount( - [&to]() { return to.mutable_account(); }, from, sfAccount); -} - -template -void -populateFee(T& to, STObject const& from) -{ - if (from.isFieldPresent(sfFee)) - { - to.mutable_fee()->set_drops(from.getFieldAmount(sfFee).xrp().drops()); - } -} - -template -void -populateSigningPublicKey(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_signing_public_key(); }, - from, - sfSigningPubKey); -} - -template -void -populateTransactionSignature(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_transaction_signature(); }, - from, - sfTxnSignature); -} - -template -void -populateFlags(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_flags(); }, from, sfFlags); -} - -template -void -populateFirstLedgerSequence(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_ledger_sequence(); }, - from, - sfFirstLedgerSequence); -} - -template -void -populateValidatorToDisable(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_validator_to_disable(); }, - from, - sfValidatorToDisable); -} - -template -void -populateValidatorToReEnable(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_validator_to_re_enable(); }, - from, - sfValidatorToReEnable); -} - -template -void -populateLastLedgerSequence(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_last_ledger_sequence(); }, - from, - sfLastLedgerSequence); -} - -template -void -populateSourceTag(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_source_tag(); }, from, sfSourceTag); -} - -template -void -populateAccountTransactionID(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_account_transaction_id(); }, - from, - sfAccountTxnID); -} - -template -void -populateMemoData(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_memo_data(); }, from, sfMemoData); -} - -template -void -populateMemoFormat(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_memo_format(); }, from, sfMemoFormat); -} - -template -void -populateMemoType(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_memo_type(); }, from, sfMemoType); -} - -template -void -populateSequence(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_sequence(); }, from, sfSequence); -} - -template -void -populateAmendment(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_amendment(); }, from, sfAmendment); -} - -template -void -populateCloseTime(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_close_time(); }, from, sfCloseTime); -} - -template -void -populateSignerWeight(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_signer_weight(); }, from, sfSignerWeight); -} - -template -void -populateAmendments(T& to, STObject const& from) -{ - populateProtoVec256( - [&to]() { return to.add_amendments(); }, from, sfAmendments); -} - -template -void -populateOwnerCount(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_owner_count(); }, from, sfOwnerCount); -} - -template -void -populatePreviousTransactionID(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_previous_transaction_id(); }, - from, - sfPreviousTxnID); -} - -template -void -populatePreviousTransactionLedgerSequence(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_previous_transaction_ledger_sequence(); }, - from, - sfPreviousTxnLgrSeq); -} - -template -void -populateLowLimit(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_low_limit(); }, from, sfLowLimit); -} - -template -void -populateHighLimit(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_high_limit(); }, from, sfHighLimit); -} - -template -void -populateLowNode(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_low_node(); }, from, sfLowNode); -} - -template -void -populateHighNode(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_high_node(); }, from, sfHighNode); -} - -template -void -populateLowQualityIn(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_low_quality_in(); }, from, sfLowQualityIn); -} - -template -void -populateLowQualityOut(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_low_quality_out(); }, - from, - sfLowQualityOut); -} - -template -void -populateHighQualityIn(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_high_quality_in(); }, - from, - sfHighQualityIn); -} - -template -void -populateHighQualityOut(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_high_quality_out(); }, - from, - sfHighQualityOut); -} - -template -void -populateBookDirectory(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_book_directory(); }, from, sfBookDirectory); -} - -template -void -populateBookNode(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_book_node(); }, from, sfBookNode); -} - -template -void -populateOwnerNode(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_owner_node(); }, from, sfOwnerNode); -} - -template -void -populateSignerListID(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_signer_list_id(); }, from, sfSignerListID); -} - -template -void -populateTicketSequence(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_ticket_sequence(); }, - from, - sfTicketSequence); -} - -template -void -populateHashes(T& to, STObject const& from) -{ - populateProtoVec256([&to]() { return to.add_hashes(); }, from, sfHashes); -} - -template -void -populateIndexes(T& to, STObject const& from) -{ - populateProtoVec256([&to]() { return to.add_indexes(); }, from, sfIndexes); -} - -template -void -populateNFTokenOffers(T& to, STObject const& from) -{ - populateProtoVec256( - [&to]() { return to.add_nftoken_offers(); }, from, sfNFTokenOffers); -} - -template -void -populateRootIndex(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_root_index(); }, from, sfRootIndex); -} - -template -void -populateIndexNext(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_index_next(); }, from, sfIndexNext); -} - -template -void -populateIndexPrevious(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_index_previous(); }, from, sfIndexPrevious); -} - -template -void -populateTakerPaysCurrency(T& to, STObject const& from) -{ - populateProtoCurrency( - [&to]() { return to.mutable_taker_pays_currency(); }, - from, - sfTakerPaysCurrency); -} - -template -void -populateTakerPaysIssuer(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_taker_pays_issuer(); }, - from, - sfTakerPaysIssuer); -} - -template -void -populateTakerGetsCurrency(T& to, STObject const& from) -{ - populateProtoCurrency( - [&to]() { return to.mutable_taker_gets_currency(); }, - from, - sfTakerGetsCurrency); -} - -template -void -populateTakerGetsIssuer(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_taker_gets_issuer(); }, - from, - sfTakerGetsIssuer); -} - -template -void -populateDestinationNode(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_destination_node(); }, - from, - sfDestinationNode); -} - -template -void -populateBaseFee(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_base_fee(); }, from, sfBaseFee); -} - -template -void -populateReferenceFeeUnits(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_reference_fee_units(); }, - from, - sfReferenceFeeUnits); -} - -template -void -populatePreviousPageMin(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_previous_page_min(); }, - from, - sfPreviousPageMin); -} - -template -void -populateNextPageMin(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_next_page_min(); }, from, sfNextPageMin); -} - -template -void -populateNFTokenID(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_nftoken_id(); }, from, sfNFTokenID); -} - -template -void -populateURI(T& to, STObject const& from) -{ - populateProtoVLasString([&to]() { return to.mutable_uri(); }, from, sfURI); -} - -template -void -populateBurnedNFTokens(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_burned_nftokens(); }, - from, - sfBurnedNFTokens); -} - -template -void -populateMintedNFTokens(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_minted_nftokens(); }, - from, - sfMintedNFTokens); -} - -template -void -populateNFTokenMinter(T& to, STObject const& from) -{ - populateProtoAccount( - [&to]() { return to.mutable_nftoken_minter(); }, from, sfNFTokenMinter); -} - -template -void -populateNFTokenBrokerFee(T& to, STObject const& from) -{ - populateProtoAmount( - [&to]() { return to.mutable_nftoken_broker_fee(); }, - from, - sfNFTokenBrokerFee); -} - -template -void -populateNFTokenBuyOffer(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_nftoken_buy_offer(); }, - from, - sfNFTokenBuyOffer); -} - -template -void -populateNFTokenSellOffer(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_nftoken_sell_offer(); }, - from, - sfNFTokenSellOffer); -} - -template -void -populateIssuer(T& to, STObject const& from) -{ - populateProtoAccount( - [&to]() { return to.mutable_issuer(); }, from, sfIssuer); -} - -template -void -populateNFTokenTaxon(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_nftoken_taxon(); }, from, sfNFTokenTaxon); -} - -template -void -populateTransferFee(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_transfer_fee(); }, from, sfTransferFee); -} - -template -void -populateReserveBase(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_reserve_base(); }, from, sfReserveBase); -} - -template -void -populateReserveIncrement(T& to, STObject const& from) -{ - populateProtoPrimitive( - [&to]() { return to.mutable_reserve_increment(); }, - from, - sfReserveIncrement); -} - -template -void -populateSignerEntries(T& to, STObject const& from) -{ - populateProtoArray( - [&to]() { return to.add_signer_entries(); }, - [](auto& innerObj, auto& innerProto) { - populateAccount(innerProto, innerObj); - populateSignerWeight(innerProto, innerObj); - }, - from, - sfSignerEntries, - sfSignerEntry); -} - -template -void -populateDisabledValidators(T& to, STObject const& from) -{ - populateProtoArray( - [&to]() { return to.add_disabled_validators(); }, - [](auto& innerObj, auto& innerProto) { - populatePublicKey(innerProto, innerObj); - populateFirstLedgerSequence(innerProto, innerObj); - }, - from, - sfDisabledValidators, - sfDisabledValidator); -} - -template -void -populateMemos(T& to, STObject const& from) -{ - populateProtoArray( - [&to]() { return to.add_memos(); }, - [](auto& innerObj, auto& innerProto) { - populateMemoData(innerProto, innerObj); - populateMemoType(innerProto, innerObj); - populateMemoFormat(innerProto, innerObj); - }, - from, - sfMemos, - sfMemo); -} - -template -void -populateSigners(T& to, STObject const& from) -{ - populateProtoArray( - [&to]() { return to.add_signers(); }, - [](auto& innerObj, auto& innerProto) { - populateAccount(innerProto, innerObj); - populateTransactionSignature(innerProto, innerObj); - populateSigningPublicKey(innerProto, innerObj); - }, - from, - sfSigners, - sfSigner); -} - -template -void -populateMajorities(T& to, STObject const& from) -{ - populateProtoArray( - [&to]() { return to.add_majorities(); }, - [](auto innerObj, auto innerProto) { - populateAmendment(innerProto, innerObj); - populateCloseTime(innerProto, innerObj); - }, - from, - sfMajorities, - sfMajority); -} - -template -void -populateNFTokens(T& to, STObject const& from) -{ - populateProtoArray( - [&to]() { return to.add_nftokens(); }, - [](auto innerObj, auto innerProto) { - populateNFTokenID(innerProto, innerObj); - populateURI(innerProto, innerObj); - }, - from, - sfNFTokens, - sfNFToken); -} - -void -convert(org::xrpl::rpc::v1::TransactionResult& to, TER from) -{ - if (isTecClaim(from)) - { - to.set_result_type( - org::xrpl::rpc::v1::TransactionResult::RESULT_TYPE_TEC); - } - if (isTefFailure(from)) - { - to.set_result_type( - org::xrpl::rpc::v1::TransactionResult::RESULT_TYPE_TEF); - } - if (isTelLocal(from)) - { - to.set_result_type( - org::xrpl::rpc::v1::TransactionResult::RESULT_TYPE_TEL); - } - if (isTemMalformed(from)) - { - to.set_result_type( - org::xrpl::rpc::v1::TransactionResult::RESULT_TYPE_TEM); - } - if (isTerRetry(from)) - { - to.set_result_type( - org::xrpl::rpc::v1::TransactionResult::RESULT_TYPE_TER); - } - if (isTesSuccess(from)) - { - to.set_result_type( - org::xrpl::rpc::v1::TransactionResult::RESULT_TYPE_TES); - } -} - -void -convert(org::xrpl::rpc::v1::AccountSet& to, STObject const& from) -{ - populateClearFlag(to, from); - - populateDomain(to, from); - - populateEmailHash(to, from); - - populateMessageKey(to, from); - - populateNFTokenMinter(to, from); - - populateSetFlag(to, from); - - populateTransferRate(to, from); - - populateTickSize(to, from); -} - -void -convert(org::xrpl::rpc::v1::OfferCreate& to, STObject const& from) -{ - populateExpiration(to, from); - - populateOfferSequence(to, from); - - populateTakerGets(to, from); - - populateTakerPays(to, from); -} - -void -convert(org::xrpl::rpc::v1::OfferCancel& to, STObject const& from) -{ - populateOfferSequence(to, from); -} - -void -convert(org::xrpl::rpc::v1::AccountDelete& to, STObject const& from) -{ - populateDestination(to, from); -} - -void -convert(org::xrpl::rpc::v1::CheckCancel& to, STObject const& from) -{ - populateCheckID(to, from); -} - -void -convert(org::xrpl::rpc::v1::CheckCash& to, STObject const& from) -{ - populateCheckID(to, from); - - populateAmount(to, from); - - populateDeliverMin(to, from); -} - -void -convert(org::xrpl::rpc::v1::CheckCreate& to, STObject const& from) -{ - populateDestination(to, from); - - populateSendMax(to, from); - - populateDestinationTag(to, from); - - populateExpiration(to, from); - - populateInvoiceID(to, from); -} - -void -convert(org::xrpl::rpc::v1::DepositPreauth& to, STObject const& from) -{ - populateAuthorize(to, from); - - populateUnauthorize(to, from); -} - -void -convert(org::xrpl::rpc::v1::EscrowCancel& to, STObject const& from) -{ - populateOwner(to, from); - - populateOfferSequence(to, from); -} - -void -convert(org::xrpl::rpc::v1::EscrowCreate& to, STObject const& from) -{ - populateAmount(to, from); - - populateDestination(to, from); - - populateCancelAfter(to, from); - - populateFinishAfter(to, from); - - populateCondition(to, from); - - populateDestinationTag(to, from); -} - -void -convert(org::xrpl::rpc::v1::EscrowFinish& to, STObject const& from) -{ - populateOwner(to, from); - - populateOfferSequence(to, from); - - populateCondition(to, from); - - populateFulfillment(to, from); -} - -void -convert(org::xrpl::rpc::v1::NFTokenAcceptOffer& to, STObject const& from) -{ - populateNFTokenBrokerFee(to, from); - - populateNFTokenBuyOffer(to, from); - - populateNFTokenSellOffer(to, from); -} - -void -convert(org::xrpl::rpc::v1::NFTokenBurn& to, STObject const& from) -{ - populateOwner(to, from); - - populateNFTokenID(to, from); -} - -void -convert(org::xrpl::rpc::v1::NFTokenCancelOffer& to, STObject const& from) -{ - populateNFTokenOffers(to, from); -} - -void -convert(org::xrpl::rpc::v1::NFTokenCreateOffer& to, STObject const& from) -{ - populateAmount(to, from); - - populateDestination(to, from); - - populateExpiration(to, from); - - populateOwner(to, from); - - populateNFTokenID(to, from); -} - -void -convert(org::xrpl::rpc::v1::NFTokenMint& to, STObject const& from) -{ - populateIssuer(to, from); - - populateNFTokenTaxon(to, from); - - populateTransferFee(to, from); - - populateURI(to, from); -} - -void -convert(org::xrpl::rpc::v1::PaymentChannelClaim& to, STObject const& from) -{ - populateChannel(to, from); - - populateBalance(to, from); - - populateAmount(to, from); - - populatePaymentChannelSignature(to, from); - - populatePublicKey(to, from); -} - -void -convert(org::xrpl::rpc::v1::PaymentChannelCreate& to, STObject const& from) -{ - populateAmount(to, from); - - populateDestination(to, from); - - populateSettleDelay(to, from); - - populatePublicKey(to, from); - - populateCancelAfter(to, from); - - populateDestinationTag(to, from); -} - -void -convert(org::xrpl::rpc::v1::PaymentChannelFund& to, STObject const& from) -{ - populateChannel(to, from); - - populateAmount(to, from); - - populateExpiration(to, from); -} - -void -convert(org::xrpl::rpc::v1::SetRegularKey& to, STObject const& from) -{ - populateRegularKey(to, from); -} - -void -convert(org::xrpl::rpc::v1::SignerListSet& to, STObject const& from) -{ - populateSignerQuorum(to, from); - - populateSignerEntries(to, from); -} - -void -convert(org::xrpl::rpc::v1::TicketCreate& to, STObject const& from) -{ - populateTicketCount(to, from); -} - -void -convert(org::xrpl::rpc::v1::TrustSet& to, STObject const& from) -{ - populateLimitAmount(to, from); - - populateQualityIn(to, from); - - populateQualityOut(to, from); -} - -void -convert(org::xrpl::rpc::v1::Payment& to, STObject const& from) -{ - populateAmount(to, from); - - populateDestination(to, from); - - populateDestinationTag(to, from); - - populateInvoiceID(to, from); - - populateSendMax(to, from); - - populateDeliverMin(to, from); - - if (from.isFieldPresent(sfPaths)) - { - // populate path data - STPathSet const& pathset = from.getFieldPathSet(sfPaths); - for (auto it = pathset.begin(); it < pathset.end(); ++it) - { - STPath const& path = *it; - - org::xrpl::rpc::v1::Payment_Path* protoPath = to.add_paths(); - - for (auto it2 = path.begin(); it2 != path.end(); ++it2) - { - org::xrpl::rpc::v1::Payment_PathElement* protoElement = - protoPath->add_elements(); - STPathElement const& elt = *it2; - - if (elt.isOffer()) - { - if (elt.hasCurrency()) - { - Currency const& currency = elt.getCurrency(); - protoElement->mutable_currency()->set_name( - to_string(currency)); - } - if (elt.hasIssuer()) - { - AccountID const& issuer = elt.getIssuerID(); - protoElement->mutable_issuer()->set_address( - toBase58(issuer)); - } - } - else if (elt.isAccount()) - { - AccountID const& pathAccount = elt.getAccountID(); - protoElement->mutable_account()->set_address( - toBase58(pathAccount)); - } - } - } - } -} - -void -convert(org::xrpl::rpc::v1::AccountRoot& to, STObject const& from) -{ - populateAccount(to, from); - - populateBalance(to, from); - - populateSequence(to, from); - - populateFlags(to, from); - - populateOwnerCount(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); - - populateAccountTransactionID(to, from); - - populateDomain(to, from); - - populateEmailHash(to, from); - - populateMessageKey(to, from); - - populateRegularKey(to, from); - - populateTickSize(to, from); - - populateTransferRate(to, from); - - populateBurnedNFTokens(to, from); - - populateMintedNFTokens(to, from); - - populateNFTokenMinter(to, from); -} - -void -convert(org::xrpl::rpc::v1::Amendments& to, STObject const& from) -{ - populateAmendments(to, from); - - populateMajorities(to, from); -} - -void -convert(org::xrpl::rpc::v1::Check& to, STObject const& from) -{ - populateAccount(to, from); - - populateDestination(to, from); - - populateFlags(to, from); - - populateOwnerNode(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); - - populateSendMax(to, from); - - populateSequence(to, from); - - populateDestinationNode(to, from); - - populateDestinationTag(to, from); - - populateExpiration(to, from); - - populateInvoiceID(to, from); - - populateSourceTag(to, from); -} - -void -convert(org::xrpl::rpc::v1::DepositPreauthObject& to, STObject const& from) -{ - populateAccount(to, from); - - populateAuthorize(to, from); - - populateFlags(to, from); - - populateOwnerNode(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); -} - -void -convert(org::xrpl::rpc::v1::FeeSettings& to, STObject const& from) -{ - populateBaseFee(to, from); - - populateReferenceFeeUnits(to, from); - - populateReserveBase(to, from); - - populateReserveIncrement(to, from); - - populateFlags(to, from); -} - -void -convert(org::xrpl::rpc::v1::Escrow& to, STObject const& from) -{ - populateAccount(to, from); - - populateDestination(to, from); - - populateAmount(to, from); - - populateCondition(to, from); - - populateCancelAfter(to, from); - - populateFinishAfter(to, from); - - populateFlags(to, from); - - populateSourceTag(to, from); - - populateDestinationTag(to, from); - - populateOwnerNode(to, from); - - populateDestinationNode(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); -} - -void -convert(org::xrpl::rpc::v1::LedgerHashes& to, STObject const& from) -{ - populateLastLedgerSequence(to, from); - - populateHashes(to, from); - - populateFlags(to, from); -} - -void -convert(org::xrpl::rpc::v1::PayChannel& to, STObject const& from) -{ - populateAccount(to, from); - - populateAmount(to, from); - - populateBalance(to, from); - - populatePublicKey(to, from); - - populateSettleDelay(to, from); - - populateOwnerNode(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); - - populateFlags(to, from); - - populateExpiration(to, from); - - populateCancelAfter(to, from); - - populateSourceTag(to, from); - - populateDestinationTag(to, from); - - populateDestinationNode(to, from); -} - -void -convert(org::xrpl::rpc::v1::DirectoryNode& to, STObject const& from) -{ - populateFlags(to, from); - - populateRootIndex(to, from); - - populateIndexes(to, from); - - populateIndexNext(to, from); - - populateIndexPrevious(to, from); - - populateTakerPaysIssuer(to, from); - - populateTakerPaysCurrency(to, from); - - populateTakerGetsCurrency(to, from); - - populateTakerGetsIssuer(to, from); - - populateNFTokenID(to, from); -} - -void -convert(org::xrpl::rpc::v1::Offer& to, STObject const& from) -{ - populateAccount(to, from); - - populateSequence(to, from); - - populateFlags(to, from); - - populateTakerPays(to, from); - - populateTakerGets(to, from); - - populateBookDirectory(to, from); - - populateBookNode(to, from); -} - -void -convert(org::xrpl::rpc::v1::RippleState& to, STObject const& from) -{ - populateBalance(to, from); - - populateFlags(to, from); - - populateLowNode(to, from); - - populateHighNode(to, from); - - populateLowQualityIn(to, from); - - populateLowQualityOut(to, from); - - populateHighQualityIn(to, from); - - populateHighQualityOut(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); -} - -void -convert(org::xrpl::rpc::v1::SignerList& to, STObject const& from) -{ - populateFlags(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); - - populateOwnerNode(to, from); - - populateSignerEntries(to, from); - - populateSignerQuorum(to, from); - - populateSignerListID(to, from); -} - -void -convert(org::xrpl::rpc::v1::NegativeUNL& to, STObject const& from) -{ - populateDisabledValidators(to, from); - - populateValidatorToDisable(to, from); - - populateValidatorToReEnable(to, from); - - populateFlags(to, from); -} - -void -convert(org::xrpl::rpc::v1::TicketObject& to, STObject const& from) -{ - populateAccount(to, from); - - populateFlags(to, from); - - populateOwnerNode(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); - - populateTicketSequence(to, from); -} - -void -convert(org::xrpl::rpc::v1::NFTokenOffer& to, STObject const& from) -{ - populateFlags(to, from); - - populateOwner(to, from); - - populateNFTokenID(to, from); - - populateAmount(to, from); - - populateOwnerNode(to, from); - - populateDestination(to, from); - - populateExpiration(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); -} - -void -convert(org::xrpl::rpc::v1::NFTokenPage& to, STObject const& from) -{ - populateFlags(to, from); - - populatePreviousPageMin(to, from); - - populateNextPageMin(to, from); - - populateNFTokens(to, from); - - populatePreviousTransactionID(to, from); - - populatePreviousTransactionLedgerSequence(to, from); -} - -void -setLedgerEntryType( - org::xrpl::rpc::v1::AffectedNode& proto, - std::uint16_t lgrType) -{ - switch (lgrType) - { - case ltACCOUNT_ROOT: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_ACCOUNT_ROOT); - break; - case ltDIR_NODE: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_DIRECTORY_NODE); - break; - case ltRIPPLE_STATE: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_RIPPLE_STATE); - break; - case ltSIGNER_LIST: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_SIGNER_LIST); - break; - case ltOFFER: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_OFFER); - break; - case ltLEDGER_HASHES: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_LEDGER_HASHES); - break; - case ltAMENDMENTS: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_AMENDMENTS); - break; - case ltFEE_SETTINGS: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_FEE_SETTINGS); - break; - case ltESCROW: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_ESCROW); - break; - case ltPAYCHAN: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_PAY_CHANNEL); - break; - case ltCHECK: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_CHECK); - break; - case ltDEPOSIT_PREAUTH: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_DEPOSIT_PREAUTH); - break; - case ltNEGATIVE_UNL: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_NEGATIVE_UNL); - break; - case ltTICKET: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_TICKET); - break; - case ltNFTOKEN_OFFER: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_NFTOKEN_OFFER); - break; - case ltNFTOKEN_PAGE: - proto.set_ledger_entry_type( - org::xrpl::rpc::v1::LEDGER_ENTRY_TYPE_NFTOKEN_PAGE); - break; - } -} - -template -void -convert(T& to, STObject& from, std::uint16_t type) -{ - switch (type) - { - case ltACCOUNT_ROOT: - RPC::convert(*to.mutable_account_root(), from); - break; - case ltAMENDMENTS: - RPC::convert(*to.mutable_amendments(), from); - break; - case ltDIR_NODE: - RPC::convert(*to.mutable_directory_node(), from); - break; - case ltRIPPLE_STATE: - RPC::convert(*to.mutable_ripple_state(), from); - break; - case ltSIGNER_LIST: - RPC::convert(*to.mutable_signer_list(), from); - break; - case ltOFFER: - RPC::convert(*to.mutable_offer(), from); - break; - case ltLEDGER_HASHES: - RPC::convert(*to.mutable_ledger_hashes(), from); - break; - case ltFEE_SETTINGS: - RPC::convert(*to.mutable_fee_settings(), from); - break; - case ltESCROW: - RPC::convert(*to.mutable_escrow(), from); - break; - case ltPAYCHAN: - RPC::convert(*to.mutable_pay_channel(), from); - break; - case ltCHECK: - RPC::convert(*to.mutable_check(), from); - break; - case ltDEPOSIT_PREAUTH: - RPC::convert(*to.mutable_deposit_preauth(), from); - break; - case ltNEGATIVE_UNL: - RPC::convert(*to.mutable_negative_unl(), from); - break; - case ltTICKET: - RPC::convert(*to.mutable_ticket(), from); - break; - case ltNFTOKEN_OFFER: - RPC::convert(*to.mutable_nftoken_offer(), from); - break; - case ltNFTOKEN_PAGE: - RPC::convert(*to.mutable_nftoken_page(), from); - break; - } -} - -template -void -populateFields( - T const& getProto, - STObject& obj, - SField const& field, - uint16_t lgrType) -{ - // final fields - if (obj.isFieldPresent(field)) - { - STObject& data = obj.getField(field).downcast(); - - convert(*getProto(), data, lgrType); - } -} - -template -void -populateFinalFields(T const& getProto, STObject& obj, uint16_t lgrType) -{ - populateFields(getProto, obj, sfFinalFields, lgrType); -} - -template -void -populatePreviousFields(T const& getProto, STObject& obj, uint16_t lgrType) -{ - populateFields(getProto, obj, sfPreviousFields, lgrType); -} - -template -void -populateNewFields(T const& getProto, STObject& obj, uint16_t lgrType) -{ - populateFields(getProto, obj, sfNewFields, lgrType); -} - -void -convert(org::xrpl::rpc::v1::Meta& to, std::shared_ptr const& from) -{ - to.set_transaction_index(from->getIndex()); - - convert(*to.mutable_transaction_result(), from->getResultTER()); - to.mutable_transaction_result()->set_result( - transToken(from->getResultTER())); - - if (from->hasDeliveredAmount()) - convert(*to.mutable_delivered_amount(), from->getDeliveredAmount()); - - STArray& nodes = from->getNodes(); - for (auto it = nodes.begin(); it != nodes.end(); ++it) - { - STObject& obj = *it; - org::xrpl::rpc::v1::AffectedNode* node = to.add_affected_nodes(); - - // ledger index - uint256 ledgerIndex = obj.getFieldH256(sfLedgerIndex); - node->set_ledger_index(ledgerIndex.data(), ledgerIndex.size()); - - // ledger entry type - std::uint16_t lgrType = obj.getFieldU16(sfLedgerEntryType); - setLedgerEntryType(*node, lgrType); - - // modified node - if (obj.getFName() == sfModifiedNode) - { - populateFinalFields( - [&node]() { - return node->mutable_modified_node() - ->mutable_final_fields(); - }, - obj, - lgrType); - - populatePreviousFields( - [&node]() { - return node->mutable_modified_node() - ->mutable_previous_fields(); - }, - obj, - lgrType); - - populatePreviousTransactionID(*node->mutable_modified_node(), obj); - - populatePreviousTransactionLedgerSequence( - *node->mutable_modified_node(), obj); - } - // created node - else if (obj.getFName() == sfCreatedNode) - { - populateNewFields( - [&node]() { - return node->mutable_created_node()->mutable_new_fields(); - }, - obj, - lgrType); - } - // deleted node - else if (obj.getFName() == sfDeletedNode) - { - populateFinalFields( - [&node]() { - return node->mutable_deleted_node()->mutable_final_fields(); - }, - obj, - lgrType); - } - } -} - -void -convert( - org::xrpl::rpc::v1::QueueData& to, - std::vector const& from) -{ - if (!from.empty()) - { - to.set_txn_count(from.size()); - - std::uint32_t seqCount = 0; - std::uint32_t ticketCount = 0; - std::optional lowestSeq; - std::optional highestSeq; - std::optional lowestTicket; - std::optional highestTicket; - bool anyAuthChanged = false; - XRPAmount totalSpend(0); - - for (auto const& tx : from) - { - org::xrpl::rpc::v1::QueuedTransaction& qt = *to.add_transactions(); - - if (tx.seqProxy.isSeq()) - { - qt.mutable_sequence()->set_value(tx.seqProxy.value()); - ++seqCount; - if (!lowestSeq) - lowestSeq = tx.seqProxy.value(); - highestSeq = tx.seqProxy.value(); - } - else - { - qt.mutable_ticket()->set_value(tx.seqProxy.value()); - ++ticketCount; - if (!lowestTicket) - lowestTicket = tx.seqProxy.value(); - highestTicket = tx.seqProxy.value(); - } - - qt.set_fee_level(tx.feeLevel.fee()); - if (tx.lastValid) - qt.mutable_last_ledger_sequence()->set_value(*tx.lastValid); - - qt.mutable_fee()->set_drops(tx.consequences.fee().drops()); - auto const spend = - tx.consequences.potentialSpend() + tx.consequences.fee(); - qt.mutable_max_spend_drops()->set_drops(spend.drops()); - totalSpend += spend; - bool const authChanged = tx.consequences.isBlocker(); - if (authChanged) - anyAuthChanged = true; - qt.set_auth_change(authChanged); - } - - if (seqCount) - to.set_sequence_count(seqCount); - if (ticketCount) - to.set_ticket_count(ticketCount); - if (lowestSeq) - to.set_lowest_sequence(*lowestSeq); - if (highestSeq) - to.set_highest_sequence(*highestSeq); - if (lowestTicket) - to.set_lowest_ticket(*lowestTicket); - if (highestTicket) - to.set_highest_ticket(*highestTicket); - - to.set_auth_change_queued(anyAuthChanged); - to.mutable_max_spend_drops_total()->set_drops(totalSpend.drops()); - } -} - -void -convert( - org::xrpl::rpc::v1::Transaction& to, - std::shared_ptr const& from) -{ - STObject const& fromObj = *from; - - populateAccount(to, fromObj); - - populateFee(to, fromObj); - - populateSequence(to, fromObj); - - populateSigningPublicKey(to, fromObj); - - populateTransactionSignature(to, fromObj); - - populateFlags(to, fromObj); - - populateLastLedgerSequence(to, fromObj); - - populateSourceTag(to, fromObj); - - populateAccountTransactionID(to, fromObj); - - populateMemos(to, fromObj); - - populateSigners(to, fromObj); - - populateTicketSequence(to, fromObj); - - auto type = safe_cast(fromObj.getFieldU16(sfTransactionType)); - - switch (type) - { - case TxType::ttPAYMENT: - convert(*to.mutable_payment(), fromObj); - break; - case TxType::ttESCROW_CREATE: - convert(*to.mutable_escrow_create(), fromObj); - break; - case TxType::ttESCROW_FINISH: - convert(*to.mutable_escrow_finish(), fromObj); - break; - case TxType::ttACCOUNT_SET: - convert(*to.mutable_account_set(), fromObj); - break; - case TxType::ttESCROW_CANCEL: - convert(*to.mutable_escrow_cancel(), fromObj); - break; - case TxType::ttREGULAR_KEY_SET: - convert(*to.mutable_set_regular_key(), fromObj); - break; - case TxType::ttOFFER_CREATE: - convert(*to.mutable_offer_create(), fromObj); - break; - case TxType::ttOFFER_CANCEL: - convert(*to.mutable_offer_cancel(), fromObj); - break; - case TxType::ttSIGNER_LIST_SET: - convert(*to.mutable_signer_list_set(), fromObj); - break; - case TxType::ttPAYCHAN_CREATE: - convert(*to.mutable_payment_channel_create(), fromObj); - break; - case TxType::ttPAYCHAN_FUND: - convert(*to.mutable_payment_channel_fund(), fromObj); - break; - case TxType::ttPAYCHAN_CLAIM: - convert(*to.mutable_payment_channel_claim(), fromObj); - break; - case TxType::ttCHECK_CREATE: - convert(*to.mutable_check_create(), fromObj); - break; - case TxType::ttCHECK_CASH: - convert(*to.mutable_check_cash(), fromObj); - break; - case TxType::ttCHECK_CANCEL: - convert(*to.mutable_check_cancel(), fromObj); - break; - case TxType::ttDEPOSIT_PREAUTH: - convert(*to.mutable_deposit_preauth(), fromObj); - break; - case TxType::ttTRUST_SET: - convert(*to.mutable_trust_set(), fromObj); - break; - case TxType::ttACCOUNT_DELETE: - convert(*to.mutable_account_delete(), fromObj); - break; - case TxType::ttTICKET_CREATE: - convert(*to.mutable_ticket_create(), fromObj); - break; - case TxType::ttNFTOKEN_MINT: - convert(*to.mutable_nftoken_mint(), fromObj); - break; - case TxType::ttNFTOKEN_BURN: - convert(*to.mutable_nftoken_burn(), fromObj); - break; - case TxType::ttNFTOKEN_CREATE_OFFER: - convert(*to.mutable_nftoken_create_offer(), fromObj); - break; - case TxType::ttNFTOKEN_CANCEL_OFFER: - convert(*to.mutable_nftoken_cancel_offer(), fromObj); - break; - case TxType::ttNFTOKEN_ACCEPT_OFFER: - convert(*to.mutable_nftoken_accept_offer(), fromObj); - break; - default: - break; - } -} - -} // namespace RPC -} // namespace ripple diff --git a/src/ripple/rpc/impl/GRPCHelpers.h b/src/ripple/rpc/impl/GRPCHelpers.h deleted file mode 100644 index 80c34f96c..000000000 --- a/src/ripple/rpc/impl/GRPCHelpers.h +++ /dev/null @@ -1,90 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#ifndef RIPPLE_RPC_GRPCHELPERS_H_INCLUDED -#define RIPPLE_RPC_GRPCHELPERS_H_INCLUDED - -#include "org/xrpl/rpc/v1/get_account_info.pb.h" -#include "org/xrpl/rpc/v1/ledger_objects.pb.h" -#include "org/xrpl/rpc/v1/meta.pb.h" -#include "org/xrpl/rpc/v1/transaction.pb.h" - -#include -#include -#include -#include -#include - -#include - -namespace ripple { -namespace RPC { - -void -convert(org::xrpl::rpc::v1::Meta& to, std::shared_ptr const& from); - -void -convert( - org::xrpl::rpc::v1::QueueData& to, - std::vector const& from); - -void -convert( - org::xrpl::rpc::v1::Transaction& to, - std::shared_ptr const& from); - -void -convert(org::xrpl::rpc::v1::TransactionResult& to, TER from); - -void -convert(org::xrpl::rpc::v1::AccountRoot& to, STObject const& from); - -void -convert(org::xrpl::rpc::v1::SignerList& to, STObject const& from); - -void -convert(org::xrpl::rpc::v1::NegativeUNL& to, STObject const& from); - -template -void -convert(T& to, STAmount const& from) -{ - if (from.native()) - { - to.mutable_value()->mutable_xrp_amount()->set_drops(from.xrp().drops()); - } - else - { - Issue const& issue = from.issue(); - - org::xrpl::rpc::v1::IssuedCurrencyAmount* issued = - to.mutable_value()->mutable_issued_currency_amount(); - - issued->mutable_currency()->set_name(to_string(issue.currency)); - issued->mutable_currency()->set_code( - issue.currency.data(), Currency::size()); - issued->mutable_issuer()->set_address(toBase58(issue.account)); - issued->set_value(to_string(from.iou())); - } -} - -} // namespace RPC -} // namespace ripple - -#endif diff --git a/src/ripple/rpc/impl/Handler.cpp b/src/ripple/rpc/impl/Handler.cpp index 15f2ea8f8..17a15eed3 100644 --- a/src/ripple/rpc/impl/Handler.cpp +++ b/src/ripple/rpc/impl/Handler.cpp @@ -72,6 +72,7 @@ Handler const handlerArray[]{ {"account_offers", byRef(&doAccountOffers), Role::USER, NO_CONDITION}, {"account_tx", byRef(&doAccountTxJson), Role::USER, NO_CONDITION}, {"blacklist", byRef(&doBlackList), Role::ADMIN, NO_CONDITION}, + {"book_changes", byRef(&doBookChanges), Role::USER, NO_CONDITION}, {"book_offers", byRef(&doBookOffers), Role::USER, NO_CONDITION}, {"can_delete", byRef(&doCanDelete), Role::ADMIN, NO_CONDITION}, {"channel_authorize", byRef(&doChannelAuthorize), Role::USER, NO_CONDITION}, diff --git a/src/ripple/rpc/impl/RPCHandler.cpp b/src/ripple/rpc/impl/RPCHandler.cpp index b04a6f0ed..c7984f830 100644 --- a/src/ripple/rpc/impl/RPCHandler.cpp +++ b/src/ripple/rpc/impl/RPCHandler.cpp @@ -17,7 +17,9 @@ */ //============================================================================== +#include #include +#include #include #include #include @@ -30,14 +32,17 @@ #include #include #include +#include #include #include +#include #include #include #include #include #include #include +#include namespace ripple { namespace RPC { diff --git a/src/ripple/rpc/impl/RPCHelpers.cpp b/src/ripple/rpc/impl/RPCHelpers.cpp index 499f12323..3d1bfe637 100644 --- a/src/ripple/rpc/impl/RPCHelpers.cpp +++ b/src/ripple/rpc/impl/RPCHelpers.cpp @@ -18,10 +18,11 @@ //============================================================================== #include +#include #include #include #include -#include +#include #include #include #include @@ -31,7 +32,7 @@ #include #include -#include +#include namespace ripple { namespace RPC { @@ -298,12 +299,6 @@ ledgerFromRequest(T& ledger, GRPCContext& context) return ledgerFromSpecifier(ledger, request.ledger(), context); } -// explicit instantiation of above function -template Status -ledgerFromRequest<>( - std::shared_ptr&, - GRPCContext&); - // explicit instantiation of above function template Status ledgerFromRequest<>( @@ -537,7 +532,7 @@ isValidated( { assert(hash->isNonZero()); uint256 valHash = - app.getRelationalDBInterface().getHashByIndex(seq); + app.getRelationalDatabase().getHashByIndex(seq); if (valHash == ledger.info().hash) { // SQL database doesn't match ledger chain @@ -696,19 +691,31 @@ parseRippleLibSeed(Json::Value const& value) std::optional getSeedFromRPC(Json::Value const& params, Json::Value& error) { - // The array should be constexpr, but that makes Visual Studio unhappy. - static char const* const seedTypes[]{ - jss::passphrase.c_str(), jss::seed.c_str(), jss::seed_hex.c_str()}; + using string_to_seed_t = + std::function(std::string const&)>; + using seed_match_t = std::pair; + + static seed_match_t const seedTypes[]{ + {jss::passphrase.c_str(), + [](std::string const& s) { return parseGenericSeed(s); }}, + {jss::seed.c_str(), + [](std::string const& s) { return parseBase58(s); }}, + {jss::seed_hex.c_str(), [](std::string const& s) { + uint128 i; + if (i.parseHex(s)) + return std::optional(Slice(i.data(), i.size())); + return std::optional{}; + }}}; // Identify which seed type is in use. - char const* seedType = nullptr; + seed_match_t const* seedType = nullptr; int count = 0; - for (auto t : seedTypes) + for (auto const& t : seedTypes) { - if (params.isMember(t)) + if (params.isMember(t.first)) { ++count; - seedType = t; + seedType = &t; } } @@ -722,28 +729,17 @@ getSeedFromRPC(Json::Value const& params, Json::Value& error) } // Make sure a string is present - if (!params[seedType].isString()) + auto const& param = params[seedType->first]; + if (!param.isString()) { - error = RPC::expected_field_error(seedType, "string"); + error = RPC::expected_field_error(seedType->first, "string"); return std::nullopt; } - auto const fieldContents = params[seedType].asString(); + auto const fieldContents = param.asString(); // Convert string to seed. - std::optional seed; - - if (seedType == jss::seed.c_str()) - seed = parseBase58(fieldContents); - else if (seedType == jss::passphrase.c_str()) - seed = parseGenericSeed(fieldContents); - else if (seedType == jss::seed_hex.c_str()) - { - uint128 s; - - if (s.parseHex(fieldContents)) - seed.emplace(Slice(s.data(), s.size())); - } + std::optional seed = seedType->second(fieldContents); if (!seed) error = rpcError(rpcBAD_SEED); @@ -757,7 +753,6 @@ keypairForSignature(Json::Value const& params, Json::Value& error) bool const has_key_type = params.isMember(jss::key_type); // All of the secret types we allow, but only one at a time. - // The array should be constexpr, but that makes Visual Studio unhappy. static char const* const secretTypes[]{ jss::passphrase.c_str(), jss::secret.c_str(), @@ -811,7 +806,9 @@ keypairForSignature(Json::Value const& params, Json::Value& error) return {}; } - if (secretType == jss::secret.c_str()) + // using strcmp as pointers may not match (see + // https://developercommunity.visualstudio.com/t/assigning-constexpr-char--to-static-cha/10021357?entry=problem) + if (strcmp(secretType, jss::secret.c_str()) == 0) { error = RPC::make_param_error( "The secret field is not allowed if " + @@ -823,7 +820,9 @@ keypairForSignature(Json::Value const& params, Json::Value& error) // ripple-lib encodes seed used to generate an Ed25519 wallet in a // non-standard way. While we never encode seeds that way, we try // to detect such keys to avoid user confusion. - if (secretType != jss::seed_hex.c_str()) + // using strcmp as pointers may not match (see + // https://developercommunity.visualstudio.com/t/assigning-constexpr-char--to-static-cha/10021357?entry=problem) + if (strcmp(secretType, jss::seed_hex.c_str()) != 0) { seed = RPC::parseRippleLibSeed(params[secretType]); @@ -952,5 +951,119 @@ getAPIVersionNumber(Json::Value const& jv, bool betaEnabled) return requestedVersion.asUInt(); } +std::variant, Json::Value> +getLedgerByContext(RPC::JsonContext& context) +{ + if (context.app.config().reporting()) + return rpcError(rpcREPORTING_UNSUPPORTED); + + auto const hasHash = context.params.isMember(jss::ledger_hash); + auto const hasIndex = context.params.isMember(jss::ledger_index); + std::uint32_t ledgerIndex = 0; + + auto& ledgerMaster = context.app.getLedgerMaster(); + LedgerHash ledgerHash; + + if ((hasHash && hasIndex) || !(hasHash || hasIndex)) + { + return RPC::make_param_error( + "Exactly one of ledger_hash and ledger_index can be set."); + } + + context.loadType = Resource::feeHighBurdenRPC; + + if (hasHash) + { + auto const& jsonHash = context.params[jss::ledger_hash]; + if (!jsonHash.isString() || !ledgerHash.parseHex(jsonHash.asString())) + return RPC::invalid_field_error(jss::ledger_hash); + } + else + { + auto const& jsonIndex = context.params[jss::ledger_index]; + if (!jsonIndex.isInt()) + return RPC::invalid_field_error(jss::ledger_index); + + // We need a validated ledger to get the hash from the sequence + if (ledgerMaster.getValidatedLedgerAge() > + RPC::Tuning::maxValidatedLedgerAge) + { + if (context.apiVersion == 1) + return rpcError(rpcNO_CURRENT); + return rpcError(rpcNOT_SYNCED); + } + + ledgerIndex = jsonIndex.asInt(); + auto ledger = ledgerMaster.getValidatedLedger(); + + if (ledgerIndex >= ledger->info().seq) + return RPC::make_param_error("Ledger index too large"); + if (ledgerIndex <= 0) + return RPC::make_param_error("Ledger index too small"); + + auto const j = context.app.journal("RPCHandler"); + // Try to get the hash of the desired ledger from the validated ledger + auto neededHash = hashOfSeq(*ledger, ledgerIndex, j); + if (!neededHash) + { + // Find a ledger more likely to have the hash of the desired ledger + auto const refIndex = getCandidateLedger(ledgerIndex); + auto refHash = hashOfSeq(*ledger, refIndex, j); + assert(refHash); + + ledger = ledgerMaster.getLedgerByHash(*refHash); + if (!ledger) + { + // We don't have the ledger we need to figure out which ledger + // they want. Try to get it. + + if (auto il = context.app.getInboundLedgers().acquire( + *refHash, refIndex, InboundLedger::Reason::GENERIC)) + { + Json::Value jvResult = RPC::make_error( + rpcLGR_NOT_FOUND, + "acquiring ledger containing requested index"); + jvResult[jss::acquiring] = + getJson(LedgerFill(*il, &context)); + return jvResult; + } + + if (auto il = context.app.getInboundLedgers().find(*refHash)) + { + Json::Value jvResult = RPC::make_error( + rpcLGR_NOT_FOUND, + "acquiring ledger containing requested index"); + jvResult[jss::acquiring] = il->getJson(0); + return jvResult; + } + + // Likely the app is shutting down + return Json::Value(); + } + + neededHash = hashOfSeq(*ledger, ledgerIndex, j); + } + assert(neededHash); + ledgerHash = neededHash ? *neededHash : beast::zero; // kludge + } + + // Try to get the desired ledger + // Verify all nodes even if we think we have it + auto ledger = context.app.getInboundLedgers().acquire( + ledgerHash, ledgerIndex, InboundLedger::Reason::GENERIC); + + // In standalone mode, accept the ledger from the ledger cache + if (!ledger && context.app.config().standalone()) + ledger = ledgerMaster.getLedgerByHash(ledgerHash); + + if (ledger) + return ledger; + + if (auto il = context.app.getInboundLedgers().find(ledgerHash)) + return il->getJson(0); + + return RPC::make_error( + rpcNOT_READY, "findCreate failed to return an inbound ledger"); +} } // namespace RPC } // namespace ripple diff --git a/src/ripple/rpc/impl/RPCHelpers.h b/src/ripple/rpc/impl/RPCHelpers.h index e3d44c2e7..2aa62f347 100644 --- a/src/ripple/rpc/impl/RPCHelpers.h +++ b/src/ripple/rpc/impl/RPCHelpers.h @@ -31,6 +31,7 @@ #include #include #include +#include namespace Json { class Value; @@ -287,6 +288,11 @@ chooseLedgerEntryType(Json::Value const& params); unsigned int getAPIVersionNumber(const Json::Value& value, bool betaEnabled); +/** Return a ledger based on ledger_hash or ledger_index, + or an RPC error */ +std::variant, Json::Value> +getLedgerByContext(RPC::JsonContext& context); + } // namespace RPC } // namespace ripple diff --git a/src/ripple/rpc/impl/ServerHandlerImp.cpp b/src/ripple/rpc/impl/ServerHandlerImp.cpp index 3ac5f04a9..cb70fdcab 100644 --- a/src/ripple/rpc/impl/ServerHandlerImp.cpp +++ b/src/ripple/rpc/impl/ServerHandlerImp.cpp @@ -873,9 +873,23 @@ ServerHandlerImp::processRequest( params, {user, forwardedFor}}; Json::Value result; + auto start = std::chrono::system_clock::now(); - RPC::doCommand(context, result); + + try + { + RPC::doCommand(context, result); + } + catch (std::exception const& ex) + { + result = RPC::make_error(rpcINTERNAL); + JLOG(m_journal.error()) << "Internal error : " << ex.what() + << " when processing request: " + << Json::Compact{Json::Value{params}}; + } + auto end = std::chrono::system_clock::now(); + logDuration(params, end - start, m_journal); usage.charge(loadType); diff --git a/src/ripple/rpc/impl/ShardArchiveHandler.cpp b/src/ripple/rpc/impl/ShardArchiveHandler.cpp index efa422c8b..2284780c2 100644 --- a/src/ripple/rpc/impl/ShardArchiveHandler.cpp +++ b/src/ripple/rpc/impl/ShardArchiveHandler.cpp @@ -18,7 +18,7 @@ //============================================================================== #include -#include +#include #include #include #include @@ -416,41 +416,42 @@ ShardArchiveHandler::complete(path dstPath) } // Make lambdas mutable captured vars can be moved from - auto wrapper = jobCounter_.wrap([=, - dstPath = std::move(dstPath)]() mutable { - if (stopping_) - return; + auto wrapper = + jobCounter_.wrap([=, this, dstPath = std::move(dstPath)]() mutable { + if (stopping_) + return; - // If not synced then defer and retry - auto const mode{app_.getOPs().getOperatingMode()}; - if (mode != OperatingMode::FULL) - { - std::lock_guard lock(m_); - timer_.expires_from_now(static_cast( - (static_cast(OperatingMode::FULL) - - static_cast(mode)) * - 10)); + // If not synced then defer and retry + auto const mode{app_.getOPs().getOperatingMode()}; + if (mode != OperatingMode::FULL) + { + std::lock_guard lock(m_); + timer_.expires_from_now(static_cast( + (static_cast(OperatingMode::FULL) - + static_cast(mode)) * + 10)); - auto wrapper = timerCounter_.wrap( - [=, dstPath = std::move(dstPath)]( - boost::system::error_code const& ec) mutable { - if (ec != boost::asio::error::operation_aborted) - complete(std::move(dstPath)); - }); + auto wrapper = timerCounter_.wrap( + [=, this, dstPath = std::move(dstPath)]( + boost::system::error_code const& ec) mutable { + if (ec != boost::asio::error::operation_aborted) + complete(std::move(dstPath)); + }); - if (!wrapper) - onClosureFailed( - "failed to wrap closure for operating mode timer", lock); + if (!wrapper) + onClosureFailed( + "failed to wrap closure for operating mode timer", + lock); + else + timer_.async_wait(*wrapper); + } else - timer_.async_wait(*wrapper); - } - else - { - process(dstPath); - std::lock_guard lock(m_); - removeAndProceed(lock); - } - }); + { + process(dstPath); + std::lock_guard lock(m_); + removeAndProceed(lock); + } + }); if (!wrapper) { diff --git a/src/ripple/rpc/impl/Status.cpp b/src/ripple/rpc/impl/Status.cpp index 0890daee6..e9e64da7a 100644 --- a/src/ripple/rpc/impl/Status.cpp +++ b/src/ripple/rpc/impl/Status.cpp @@ -23,8 +23,6 @@ namespace ripple { namespace RPC { -constexpr Status::Code Status::OK; - std::string Status::codeString() const { diff --git a/src/ripple/shamap/SHAMapInnerNode.h b/src/ripple/shamap/SHAMapInnerNode.h index 5f0765e9c..c85cdcbbc 100644 --- a/src/ripple/shamap/SHAMapInnerNode.h +++ b/src/ripple/shamap/SHAMapInnerNode.h @@ -147,7 +147,7 @@ public: getChildHash(int m) const; void - setChild(int m, std::shared_ptr const& child); + setChild(int m, std::shared_ptr child); void shareChild(int m, std::shared_ptr const& child); diff --git a/src/ripple/shamap/impl/SHAMap.cpp b/src/ripple/shamap/impl/SHAMap.cpp index 6f6acb9a7..1a5a283dd 100644 --- a/src/ripple/shamap/impl/SHAMap.cpp +++ b/src/ripple/shamap/impl/SHAMap.cpp @@ -118,7 +118,7 @@ SHAMap::dirtyUp( assert(branch >= 0); node = unshareNode(std::move(node), nodeID); - node->setChild(branch, child); + node->setChild(branch, std::move(child)); child = std::move(node); } @@ -718,7 +718,7 @@ SHAMap::delItem(uint256 const& id) stack.pop(); node = unshareNode(std::move(node), nodeID); - node->setChild(selectBranch(nodeID, id), prevNode); + node->setChild(selectBranch(nodeID, id), std::move(prevNode)); if (!nodeID.isRoot()) { @@ -795,8 +795,7 @@ SHAMap::addGiveItem(SHAMapNodeType type, std::shared_ptr item) auto inner = std::static_pointer_cast(node); int branch = selectBranch(nodeID, tag); assert(inner->isEmptyBranch(branch)); - auto newNode = makeTypedLeaf(type, std::move(item), cowid_); - inner->setChild(branch, newNode); + inner->setChild(branch, makeTypedLeaf(type, std::move(item), cowid_)); } else { diff --git a/src/ripple/shamap/impl/SHAMapInnerNode.cpp b/src/ripple/shamap/impl/SHAMapInnerNode.cpp index eb00f8587..6ea6f47eb 100644 --- a/src/ripple/shamap/impl/SHAMapInnerNode.cpp +++ b/src/ripple/shamap/impl/SHAMapInnerNode.cpp @@ -22,102 +22,19 @@ #include #include #include +#include #include #include #include #include #include -#include - #include #include #include -#ifndef __aarch64__ -// This is used for the _mm_pause instruction: -#include -#endif - namespace ripple { -/** A specialized 16-way spinlock used to protect inner node branches. - - This class packs 16 separate spinlocks into a single 16-bit value. It makes - it possible to lock any one lock at once or, alternatively, all together. - - The implementation tries to use portable constructs but has to be low-level - for performance. - */ -class SpinBitlock -{ -private: - std::atomic& bits_; - std::uint16_t mask_; - -public: - SpinBitlock(std::atomic& lock) : bits_(lock), mask_(0xFFFF) - { - } - - SpinBitlock(std::atomic& lock, int index) - : bits_(lock), mask_(1 << index) - { - assert(index >= 0 && index < 16); - } - - [[nodiscard]] bool - try_lock() - { - // If we want to grab all the individual bitlocks at once we cannot - // use `fetch_or`! To see why, imagine that `lock_ == 0x0020` which - // means that the `fetch_or` would return `0x0020` but all the bits - // would already be (incorrectly!) set. Oops! - std::uint16_t expected = 0; - - if (mask_ != 0xFFFF) - return (bits_.fetch_or(mask_, std::memory_order_acquire) & mask_) == - expected; - - return bits_.compare_exchange_weak( - expected, - mask_, - std::memory_order_acquire, - std::memory_order_relaxed); - } - - void - lock() - { - // Testing suggests that 99.9999% of the time this will succeed, so - // we try to optimize the fast path. - if (try_lock()) - return; - - do - { - // We try to spin for a few times: - for (int i = 0; i != 100; ++i) - { - if (try_lock()) - return; - -#ifndef __aarch64__ - _mm_pause(); -#endif - } - - std::this_thread::yield(); - } while ((bits_.load(std::memory_order_relaxed) & mask_) == 0); - } - - void - unlock() - { - bits_.fetch_and(~mask_, std::memory_order_release); - } -}; - SHAMapInnerNode::SHAMapInnerNode( std::uint32_t cowid, std::uint8_t numAllocatedChildren) @@ -185,7 +102,7 @@ SHAMapInnerNode::clone(std::uint32_t cowid) const }); } - SpinBitlock sl(lock_); + spinlock sl(lock_); std::lock_guard lock(sl); if (thisIsSparse) @@ -367,7 +284,7 @@ SHAMapInnerNode::getString(const SHAMapNodeID& id) const // We are modifying an inner node void -SHAMapInnerNode::setChild(int m, std::shared_ptr const& child) +SHAMapInnerNode::setChild(int m, std::shared_ptr child) { assert((m >= 0) && (m < branchFactor)); assert(cowid_ != 0); @@ -393,7 +310,7 @@ SHAMapInnerNode::setChild(int m, std::shared_ptr const& child) auto const childIndex = *getChildIndex(m); auto [_, hashes, children] = hashesAndChildren_.getHashesAndChildren(); hashes[childIndex].zero(); - children[childIndex] = child; + children[childIndex] = std::move(child); } hash_.zero(); @@ -422,7 +339,7 @@ SHAMapInnerNode::getChildPointer(int branch) auto const index = *getChildIndex(branch); - SpinBitlock sl(lock_, index); + packed_spinlock sl(lock_, index); std::lock_guard lock(sl); return hashesAndChildren_.getChildren()[index].get(); } @@ -435,7 +352,7 @@ SHAMapInnerNode::getChild(int branch) auto const index = *getChildIndex(branch); - SpinBitlock sl(lock_, index); + packed_spinlock sl(lock_, index); std::lock_guard lock(sl); return hashesAndChildren_.getChildren()[index]; } @@ -462,7 +379,7 @@ SHAMapInnerNode::canonicalizeChild( auto [_, hashes, children] = hashesAndChildren_.getHashesAndChildren(); assert(node->getHash() == hashes[childIndex]); - SpinBitlock sl(lock_, childIndex); + packed_spinlock sl(lock_, childIndex); std::lock_guard lock(sl); if (children[childIndex]) diff --git a/src/test/app/AccountDelete_test.cpp b/src/test/app/AccountDelete_test.cpp index db2ac5799..73a0ccbf9 100644 --- a/src/test/app/AccountDelete_test.cpp +++ b/src/test/app/AccountDelete_test.cpp @@ -515,16 +515,16 @@ public: // All it takes is a large enough XRP payment to resurrect // becky's account. Try too small a payment. - env(pay(alice, becky, XRP(19)), ter(tecNO_DST_INSUF_XRP)); + env(pay(alice, becky, XRP(9)), ter(tecNO_DST_INSUF_XRP)); env.close(); // Actually resurrect becky's account. - env(pay(alice, becky, XRP(20))); + env(pay(alice, becky, XRP(10))); env.close(); // becky's account root should be back. BEAST_EXPECT(env.closed()->exists(beckyAcctKey)); - BEAST_EXPECT(env.balance(becky) == XRP(20)); + BEAST_EXPECT(env.balance(becky) == XRP(10)); // becky's resurrected account can be the destination of alice's // PayChannel. @@ -541,7 +541,7 @@ public: env(payChanClaim()); env.close(); - BEAST_EXPECT(env.balance(becky) == XRP(20) + payChanXRP); + BEAST_EXPECT(env.balance(becky) == XRP(10) + payChanXRP); } void diff --git a/src/test/app/AccountTxPaging_test.cpp b/src/test/app/AccountTxPaging_test.cpp index 332ef2131..d3969e279 100644 --- a/src/test/app/AccountTxPaging_test.cpp +++ b/src/test/app/AccountTxPaging_test.cpp @@ -263,1906 +263,11 @@ class AccountTxPaging_test : public beast::unit_test::suite } } - class GrpcAccountTxClient : public test::GRPCTestClientBase - { - public: - org::xrpl::rpc::v1::GetAccountTransactionHistoryRequest request; - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse reply; - - explicit GrpcAccountTxClient(std::string const& port) - : GRPCTestClientBase(port) - { - } - - void - AccountTx() - { - status = - stub_->GetAccountTransactionHistory(&context, request, &reply); - } - }; - - bool - checkTransaction( - org::xrpl::rpc::v1::GetTransactionResponse const& tx, - int sequence, - int ledger) - { - return ( - tx.transaction().sequence().value() == sequence && - tx.ledger_index() == ledger); - } - - std::pair< - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse, - grpc::Status> - nextBinary( - std::string grpcPort, - test::jtx::Env& env, - std::string const& account = "", - int ledger_min = -1, - int ledger_max = -1, - int limit = -1, - bool forward = false, - org::xrpl::rpc::v1::Marker* marker = nullptr) - { - GrpcAccountTxClient client{grpcPort}; - auto& request = client.request; - if (account != "") - request.mutable_account()->set_address(account); - if (ledger_min != -1) - request.mutable_ledger_range()->set_ledger_index_min(ledger_min); - if (ledger_max != -1) - request.mutable_ledger_range()->set_ledger_index_max(ledger_max); - request.set_forward(forward); - request.set_binary(true); - if (limit != -1) - request.set_limit(limit); - if (marker) - { - *request.mutable_marker() = *marker; - } - - client.AccountTx(); - return {client.reply, client.status}; - } - - std::pair< - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse, - grpc::Status> - next( - std::string grpcPort, - test::jtx::Env& env, - std::string const& account = "", - int ledger_min = -1, - int ledger_max = -1, - int limit = -1, - bool forward = false, - org::xrpl::rpc::v1::Marker* marker = nullptr) - { - GrpcAccountTxClient client{grpcPort}; - auto& request = client.request; - if (account != "") - request.mutable_account()->set_address(account); - if (ledger_min != -1) - request.mutable_ledger_range()->set_ledger_index_min(ledger_min); - if (ledger_max != -1) - request.mutable_ledger_range()->set_ledger_index_max(ledger_max); - request.set_forward(forward); - if (limit != -1) - request.set_limit(limit); - if (marker) - { - *request.mutable_marker() = *marker; - } - - client.AccountTx(); - return {client.reply, client.status}; - } - - std::pair< - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse, - grpc::Status> - nextWithSeq( - std::string grpcPort, - test::jtx::Env& env, - std::string const& account = "", - int ledger_seq = -1, - int limit = -1, - bool forward = false, - org::xrpl::rpc::v1::Marker* marker = nullptr) - { - GrpcAccountTxClient client{grpcPort}; - auto& request = client.request; - if (account != "") - request.mutable_account()->set_address(account); - if (ledger_seq != -1) - request.mutable_ledger_specifier()->set_sequence(ledger_seq); - request.set_forward(forward); - if (limit != -1) - request.set_limit(limit); - if (marker) - { - *request.mutable_marker() = *marker; - } - - client.AccountTx(); - return {client.reply, client.status}; - } - - std::pair< - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse, - grpc::Status> - nextWithHash( - std::string grpcPort, - test::jtx::Env& env, - std::string const& account = "", - uint256 const& hash = beast::zero, - int limit = -1, - bool forward = false, - org::xrpl::rpc::v1::Marker* marker = nullptr) - { - GrpcAccountTxClient client{grpcPort}; - auto& request = client.request; - if (account != "") - request.mutable_account()->set_address(account); - if (hash != beast::zero) - request.mutable_ledger_specifier()->set_hash( - hash.data(), hash.size()); - request.set_forward(forward); - if (limit != -1) - request.set_limit(limit); - if (marker) - { - *request.mutable_marker() = *marker; - } - - client.AccountTx(); - return {client.reply, client.status}; - } - - void - testAccountTxParametersGrpc() - { - testcase("Test Account_tx Grpc"); - - using namespace test::jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - - Account A1{"A1"}; - env.fund(XRP(10000), A1); - env.close(); - - // Ledger 3 has the two txs associated with funding the account - // All other ledgers have no txs - - auto hasTxs = [](auto res) { - return res.second.error_code() == 0 && - (res.first.transactions().size() == 2) && - //(res.transactions()[0u].transaction().has_account_set()) && - (res.first.transactions()[1u].transaction().has_payment()); - }; - auto noTxs = [](auto res) { - return res.second.error_code() == 0 && - (res.first.transactions().size() == 0); - }; - - auto isErr = [](auto res, auto expect) { - return res.second.error_code() == expect; - }; - - BEAST_EXPECT( - isErr(next(grpcPort, env, ""), grpc::StatusCode::INVALID_ARGUMENT)); - - BEAST_EXPECT(isErr( - next(grpcPort, env, "0xDEADBEEF"), - grpc::StatusCode::INVALID_ARGUMENT)); - - BEAST_EXPECT(hasTxs(next(grpcPort, env, A1.human()))); - - // Ledger min/max index - { - BEAST_EXPECT(hasTxs(next(grpcPort, env, A1.human()))); - - BEAST_EXPECT(hasTxs(next(grpcPort, env, A1.human(), 0, 100))); - - BEAST_EXPECT(noTxs(next(grpcPort, env, A1.human(), 1, 2))); - - BEAST_EXPECT(isErr( - next(grpcPort, env, A1.human(), 2, 1), - grpc::StatusCode::INVALID_ARGUMENT)); - } - - // Ledger index min only - { - BEAST_EXPECT(hasTxs(next(grpcPort, env, A1.human(), -1))); - - BEAST_EXPECT(hasTxs(next(grpcPort, env, A1.human(), 1))); - - BEAST_EXPECT(isErr( - next(grpcPort, env, A1.human(), env.current()->info().seq), - grpc::StatusCode::INVALID_ARGUMENT)); - } - - // Ledger index max only - { - BEAST_EXPECT(hasTxs(next(grpcPort, env, A1.human(), -1, -1))); - - BEAST_EXPECT(hasTxs(next( - grpcPort, env, A1.human(), -1, env.current()->info().seq))); - - BEAST_EXPECT(hasTxs( - next(grpcPort, env, A1.human(), -1, env.closed()->info().seq))); - - BEAST_EXPECT(noTxs(next( - grpcPort, env, A1.human(), -1, env.closed()->info().seq - 1))); - } - // Ledger Sequence - { - BEAST_EXPECT(hasTxs(nextWithSeq( - grpcPort, env, A1.human(), env.closed()->info().seq))); - - BEAST_EXPECT(noTxs(nextWithSeq( - grpcPort, env, A1.human(), env.closed()->info().seq - 1))); - - BEAST_EXPECT(isErr( - nextWithSeq( - grpcPort, env, A1.human(), env.current()->info().seq), - grpc::StatusCode::INVALID_ARGUMENT)); - - BEAST_EXPECT(isErr( - nextWithSeq( - grpcPort, env, A1.human(), env.current()->info().seq + 1), - grpc::StatusCode::NOT_FOUND)); - } - - // Ledger Hash - { - BEAST_EXPECT(hasTxs(nextWithHash( - grpcPort, env, A1.human(), env.closed()->info().hash))); - - BEAST_EXPECT(noTxs(nextWithHash( - grpcPort, env, A1.human(), env.closed()->info().parentHash))); - } - } - - struct TxCheck - { - uint32_t sequence; - uint32_t ledgerIndex; - std::string hash; - std::function - checkTxn; - }; - - void - testAccountTxContentsGrpc() - { - testcase("Test AccountTx context grpc"); - // Get results for all transaction types that can be associated - // with an account. Start by generating all transaction types. - using namespace test::jtx; - using namespace std::chrono_literals; - - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - // Set time to this value (or greater) to get delivered_amount in meta - env.timeKeeper().set(NetClock::time_point{446000001s}); - Account const alice{"alice"}; - Account const alie{"alie"}; - Account const gw{"gw"}; - auto const USD{gw["USD"]}; - - std::vector> txns; - - env.fund(XRP(1000000), alice, gw); - env.close(); - - // AccountSet - env(noop(alice)); - - txns.emplace_back(env.tx()); - // Payment - env(pay(alice, gw, XRP(100)), stag(42), dtag(24), last_ledger_seq(20)); - - txns.emplace_back(env.tx()); - // Regular key set - env(regkey(alice, alie)); - env.close(); - - txns.emplace_back(env.tx()); - // Trust and Offers - env(trust(alice, USD(200)), sig(alie)); - - txns.emplace_back(env.tx()); - std::uint32_t const offerSeq{env.seq(alice)}; - env(offer(alice, USD(50), XRP(150)), sig(alie)); - - txns.emplace_back(env.tx()); - env.close(); - - env(offer_cancel(alice, offerSeq), sig(alie)); - env.close(); - - txns.emplace_back(env.tx()); - - // SignerListSet - env(signers(alice, 1, {{"bogie", 1}, {"demon", 1}, {gw, 1}}), - sig(alie)); - - txns.emplace_back(env.tx()); - // Escrow - { - // Create an escrow. Requires either a CancelAfter or FinishAfter. - auto escrow = [](Account const& account, - Account const& to, - STAmount const& amount) { - Json::Value escro; - escro[jss::TransactionType] = jss::EscrowCreate; - escro[jss::Flags] = tfUniversal; - escro[jss::Account] = account.human(); - escro[jss::Destination] = to.human(); - escro[jss::Amount] = amount.getJson(JsonOptions::none); - return escro; - }; - - NetClock::time_point const nextTime{env.now() + 2s}; - - Json::Value escrowWithFinish{escrow(alice, alice, XRP(500))}; - escrowWithFinish[sfFinishAfter.jsonName] = - nextTime.time_since_epoch().count(); - - std::uint32_t const escrowFinishSeq{env.seq(alice)}; - env(escrowWithFinish, sig(alie)); - - txns.emplace_back(env.tx()); - Json::Value escrowWithCancel{escrow(alice, alice, XRP(500))}; - escrowWithCancel[sfFinishAfter.jsonName] = - nextTime.time_since_epoch().count(); - escrowWithCancel[sfCancelAfter.jsonName] = - nextTime.time_since_epoch().count() + 1; - - std::uint32_t const escrowCancelSeq{env.seq(alice)}; - env(escrowWithCancel, sig(alie)); - env.close(); - - txns.emplace_back(env.tx()); - { - Json::Value escrowFinish; - escrowFinish[jss::TransactionType] = jss::EscrowFinish; - escrowFinish[jss::Flags] = tfUniversal; - escrowFinish[jss::Account] = alice.human(); - escrowFinish[sfOwner.jsonName] = alice.human(); - escrowFinish[sfOfferSequence.jsonName] = escrowFinishSeq; - env(escrowFinish, sig(alie)); - - txns.emplace_back(env.tx()); - } - { - Json::Value escrowCancel; - escrowCancel[jss::TransactionType] = jss::EscrowCancel; - escrowCancel[jss::Flags] = tfUniversal; - escrowCancel[jss::Account] = alice.human(); - escrowCancel[sfOwner.jsonName] = alice.human(); - escrowCancel[sfOfferSequence.jsonName] = escrowCancelSeq; - env(escrowCancel, sig(alie)); - - txns.emplace_back(env.tx()); - } - env.close(); - } - - // PayChan - { - std::uint32_t payChanSeq{env.seq(alice)}; - Json::Value payChanCreate; - payChanCreate[jss::TransactionType] = jss::PaymentChannelCreate; - payChanCreate[jss::Flags] = tfUniversal; - payChanCreate[jss::Account] = alice.human(); - payChanCreate[jss::Destination] = gw.human(); - payChanCreate[jss::Amount] = - XRP(500).value().getJson(JsonOptions::none); - payChanCreate[sfSettleDelay.jsonName] = - NetClock::duration{100s}.count(); - payChanCreate[sfPublicKey.jsonName] = strHex(alice.pk().slice()); - env(payChanCreate, sig(alie)); - env.close(); - - txns.emplace_back(env.tx()); - std::string const payChanIndex{ - strHex(keylet::payChan(alice, gw, payChanSeq).key)}; - - { - Json::Value payChanFund; - payChanFund[jss::TransactionType] = jss::PaymentChannelFund; - payChanFund[jss::Flags] = tfUniversal; - payChanFund[jss::Account] = alice.human(); - payChanFund[sfChannel.jsonName] = payChanIndex; - payChanFund[jss::Amount] = - XRP(200).value().getJson(JsonOptions::none); - env(payChanFund, sig(alie)); - env.close(); - - txns.emplace_back(env.tx()); - } - { - Json::Value payChanClaim; - payChanClaim[jss::TransactionType] = jss::PaymentChannelClaim; - payChanClaim[jss::Flags] = tfClose; - payChanClaim[jss::Account] = gw.human(); - payChanClaim[sfChannel.jsonName] = payChanIndex; - payChanClaim[sfPublicKey.jsonName] = strHex(alice.pk().slice()); - env(payChanClaim); - env.close(); - - txns.emplace_back(env.tx()); - } - } - - // Check - { - auto const aliceCheckId = keylet::check(alice, env.seq(alice)).key; - env(check::create(alice, gw, XRP(300)), sig(alie)); - - auto txn = env.tx(); - auto const gwCheckId = keylet::check(gw, env.seq(gw)).key; - env(check::create(gw, alice, XRP(200))); - env.close(); - - // need to switch the order of the previous 2 txns, since they are - // in the same ledger and account_tx returns them in a different - // order - txns.emplace_back(env.tx()); - txns.emplace_back(txn); - env(check::cash(alice, gwCheckId, XRP(200)), sig(alie)); - - txns.emplace_back(env.tx()); - env(check::cancel(alice, aliceCheckId), sig(alie)); - - txns.emplace_back(env.tx()); - env.close(); - } - - // Deposit preauthorization. - env(deposit::auth(alice, gw), sig(alie)); - env.close(); - - txns.emplace_back(env.tx()); - // Multi Sig with memo - auto const baseFee = env.current()->fees().base; - env(noop(alice), - msig(gw), - fee(2 * baseFee), - memo("data", "format", "type")); - env.close(); - - txns.emplace_back(env.tx()); - if (!BEAST_EXPECT(txns.size() == 20)) - return; - // Setup is done. Look at the transactions returned by account_tx. - - static const TxCheck txCheck[]{ - {21, - 15, - strHex(txns[txns.size() - 1]->getTransactionID()), - [this, &txns](auto res) { - auto txnJson = - txns[txns.size() - 1]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_account_set()) && - BEAST_EXPECT(res.has_fee()) && - BEAST_EXPECT(res.fee().drops() == 20) && - BEAST_EXPECT(res.memos_size() == 1) && - BEAST_EXPECT(res.memos(0).has_memo_data()) && - BEAST_EXPECT(res.memos(0).memo_data().value() == "data") && - BEAST_EXPECT(res.memos(0).has_memo_format()) && - BEAST_EXPECT( - res.memos(0).memo_format().value() == "format") && - BEAST_EXPECT(res.memos(0).has_memo_type()) && - BEAST_EXPECT(res.memos(0).memo_type().value() == "type") && - BEAST_EXPECT(res.has_signing_public_key()) && - BEAST_EXPECT(res.signing_public_key().value() == "") && - BEAST_EXPECT(res.signers_size() == 1) && - BEAST_EXPECT(res.signers(0).has_account()) && - BEAST_EXPECT( - res.signers(0).account().value().address() == - txnJson["Signers"][0u]["Signer"]["Account"]) && - BEAST_EXPECT(res.signers(0).has_transaction_signature()) && - BEAST_EXPECT( - strHex(res.signers(0) - .transaction_signature() - .value()) == - txnJson["Signers"][0u]["Signer"]["TxnSignature"]) && - BEAST_EXPECT(res.signers(0).has_signing_public_key()) && - BEAST_EXPECT( - strHex( - res.signers(0).signing_public_key().value()) == - txnJson["Signers"][0u]["Signer"]["SigningPubKey"]); - }}, - {20, - 14, - strHex(txns[txns.size() - 2]->getTransactionID()), - [&txns, this](auto res) { - return BEAST_EXPECT(res.has_deposit_preauth()) && - BEAST_EXPECT( - res.deposit_preauth() - .authorize() - .value() - .address() == - // TODO do them all like this - txns[txns.size() - 2]->getJson( - JsonOptions::none)["Authorize"]); - }}, - {19, - 13, - strHex(txns[txns.size() - 3]->getTransactionID()), - [&txns, this](auto res) { - return BEAST_EXPECT(res.has_check_cancel()) && - BEAST_EXPECT( - strHex(res.check_cancel().check_id().value()) == - - txns[txns.size() - 3]->getJson( - JsonOptions::none)["CheckID"]); - }}, - {18, - 13, - strHex(txns[txns.size() - 4]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 4]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_check_cash()) && - BEAST_EXPECT( - strHex(res.check_cash().check_id().value()) == - txnJson["CheckID"]) && - BEAST_EXPECT(res.check_cash() - .amount() - .value() - .has_xrp_amount()) && - BEAST_EXPECT( - res.check_cash() - .amount() - .value() - .xrp_amount() - .drops() == txnJson["Amount"].asUInt()); - }}, - {17, - 12, - strHex(txns[txns.size() - 5]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 5]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_check_create()) && - BEAST_EXPECT( - res.check_create() - .destination() - .value() - .address() == txnJson["Destination"]) && - BEAST_EXPECT(res.check_create() - .send_max() - .value() - .has_xrp_amount()) && - BEAST_EXPECT( - res.check_create() - .send_max() - .value() - .xrp_amount() - .drops() == txnJson["SendMax"].asUInt()); - }}, - {5, - 12, - strHex(txns[txns.size() - 6]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 6]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_check_create()) && - BEAST_EXPECT( - res.check_create() - .destination() - .value() - .address() == txnJson["Destination"]) && - BEAST_EXPECT(res.check_create() - .send_max() - .value() - .has_xrp_amount()) && - BEAST_EXPECT( - res.check_create() - .send_max() - .value() - .xrp_amount() - .drops() == - - txnJson["SendMax"].asUInt()); - }}, - {4, - 11, - strHex(txns[txns.size() - 7]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 7]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_payment_channel_claim()) && - BEAST_EXPECT( - strHex(res.payment_channel_claim() - .channel() - .value()) == txnJson["Channel"]) && - BEAST_EXPECT( - strHex(res.payment_channel_claim() - .public_key() - .value()) == txnJson["PublicKey"]); - }}, - {16, - 10, - strHex(txns[txns.size() - 8]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 8]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_payment_channel_fund()) && - BEAST_EXPECT( - strHex( - res.payment_channel_fund().channel().value()) == - txnJson["Channel"]) && - BEAST_EXPECT(res.payment_channel_fund() - .amount() - .value() - .has_xrp_amount()) && - BEAST_EXPECT( - res.payment_channel_fund() - .amount() - .value() - .xrp_amount() - .drops() == txnJson["Amount"].asUInt()); - }}, - {15, - 9, - strHex(txns[txns.size() - 9]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 9]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_payment_channel_create()) && - BEAST_EXPECT(res.payment_channel_create() - .amount() - .value() - .has_xrp_amount()) && - BEAST_EXPECT( - res.payment_channel_create() - .amount() - .value() - .xrp_amount() - .drops() == txnJson["Amount"].asUInt()) && - BEAST_EXPECT( - res.payment_channel_create() - .destination() - .value() - .address() == txnJson["Destination"]) && - BEAST_EXPECT( - res.payment_channel_create() - .settle_delay() - .value() == txnJson["SettleDelay"].asUInt()) && - BEAST_EXPECT( - strHex(res.payment_channel_create() - .public_key() - .value()) == txnJson["PublicKey"]); - }}, - {14, - 8, - strHex(txns[txns.size() - 10]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 10]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_escrow_cancel()) && - BEAST_EXPECT( - res.escrow_cancel().owner().value().address() == - txnJson["Owner"]) && - BEAST_EXPECT( - res.escrow_cancel().offer_sequence().value() == - txnJson["OfferSequence"].asUInt() - - ); - }}, - {13, - 8, - strHex(txns[txns.size() - 11]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 11]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_escrow_finish()) && - BEAST_EXPECT( - res.escrow_finish().owner().value().address() == - txnJson["Owner"]) && - BEAST_EXPECT( - res.escrow_finish().offer_sequence().value() == - txnJson["OfferSequence"].asUInt() - - ); - }}, - {12, - 7, - strHex(txns[txns.size() - 12]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 12]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_escrow_create()) && - BEAST_EXPECT(res.escrow_create() - .amount() - .value() - .has_xrp_amount()) && - BEAST_EXPECT( - res.escrow_create() - .amount() - .value() - .xrp_amount() - .drops() == txnJson["Amount"].asUInt()) && - BEAST_EXPECT( - res.escrow_create() - .destination() - .value() - .address() == txnJson["Destination"]) && - BEAST_EXPECT( - res.escrow_create().cancel_after().value() == - txnJson["CancelAfter"].asUInt()) && - BEAST_EXPECT( - res.escrow_create().finish_after().value() == - txnJson["FinishAfter"].asUInt()); - }}, - {11, - 7, - strHex(txns[txns.size() - 13]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 13]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_escrow_create()) && - BEAST_EXPECT(res.escrow_create() - .amount() - .value() - .has_xrp_amount()) && - BEAST_EXPECT( - res.escrow_create() - .amount() - .value() - .xrp_amount() - .drops() == txnJson["Amount"].asUInt()) && - BEAST_EXPECT( - res.escrow_create() - .destination() - .value() - .address() == txnJson["Destination"]) && - BEAST_EXPECT( - res.escrow_create().finish_after().value() == - txnJson["FinishAfter"].asUInt()); - }}, - {10, - 7, - strHex(txns[txns.size() - 14]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 14]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_signer_list_set()) && - BEAST_EXPECT( - res.signer_list_set().signer_quorum().value() == - txnJson["SignerQuorum"].asUInt()) && - BEAST_EXPECT( - res.signer_list_set().signer_entries().size() == - 3) && - BEAST_EXPECT( - res.signer_list_set() - .signer_entries()[0] - .account() - .value() - .address() == - txnJson["SignerEntries"][0u]["SignerEntry"] - ["Account"]) && - BEAST_EXPECT( - res.signer_list_set() - .signer_entries()[0] - .signer_weight() - .value() == - txnJson["SignerEntries"][0u]["SignerEntry"] - ["SignerWeight"] - .asUInt()) && - BEAST_EXPECT( - res.signer_list_set() - .signer_entries()[1] - .account() - .value() - .address() == - txnJson["SignerEntries"][1u]["SignerEntry"] - ["Account"]) && - BEAST_EXPECT( - res.signer_list_set() - .signer_entries()[1] - .signer_weight() - .value() == - txnJson["SignerEntries"][1u]["SignerEntry"] - ["SignerWeight"] - .asUInt()) && - BEAST_EXPECT( - res.signer_list_set() - .signer_entries()[2] - .account() - .value() - .address() == - txnJson["SignerEntries"][2u]["SignerEntry"] - ["Account"]) && - BEAST_EXPECT( - res.signer_list_set() - .signer_entries()[2] - .signer_weight() - .value() == - txnJson["SignerEntries"][2u]["SignerEntry"] - ["SignerWeight"] - .asUInt()); - }}, - {9, - 6, - strHex(txns[txns.size() - 15]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 15]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_offer_cancel()) && - BEAST_EXPECT( - res.offer_cancel().offer_sequence().value() == - txnJson["OfferSequence"].asUInt()); - }}, - {8, - 5, - strHex(txns[txns.size() - 16]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 16]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_offer_create()) && - BEAST_EXPECT(res.offer_create() - .taker_gets() - .value() - .has_xrp_amount()) && - BEAST_EXPECT( - res.offer_create() - .taker_gets() - .value() - .xrp_amount() - .drops() == txnJson["TakerGets"].asUInt()) && - BEAST_EXPECT(res.offer_create() - .taker_pays() - .value() - .has_issued_currency_amount()) && - BEAST_EXPECT( - res.offer_create() - .taker_pays() - .value() - .issued_currency_amount() - .currency() - .name() == txnJson["TakerPays"]["currency"]) && - BEAST_EXPECT( - res.offer_create() - .taker_pays() - .value() - .issued_currency_amount() - .value() == txnJson["TakerPays"]["value"]) && - BEAST_EXPECT( - res.offer_create() - .taker_pays() - .value() - .issued_currency_amount() - .issuer() - .address() == txnJson["TakerPays"]["issuer"]); - }}, - {7, - 5, - strHex(txns[txns.size() - 17]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 17]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_trust_set()) && - BEAST_EXPECT(res.trust_set() - .limit_amount() - .value() - .has_issued_currency_amount()) && - BEAST_EXPECT( - res.trust_set() - .limit_amount() - .value() - .issued_currency_amount() - .currency() - .name() == - txnJson["LimitAmount"]["currency"]) && - BEAST_EXPECT( - res.trust_set() - .limit_amount() - .value() - .issued_currency_amount() - .value() == txnJson["LimitAmount"]["value"]) && - BEAST_EXPECT( - res.trust_set() - .limit_amount() - .value() - .issued_currency_amount() - .issuer() - .address() == txnJson["LimitAmount"]["issuer"]); - }}, - {6, - 4, - strHex(txns[txns.size() - 18]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 18]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_set_regular_key()) && - BEAST_EXPECT( - res.set_regular_key() - .regular_key() - .value() - .address() == txnJson["RegularKey"]); - }}, - {5, - 4, - strHex(txns[txns.size() - 19]->getTransactionID()), - [&txns, this](auto res) { - auto txnJson = - txns[txns.size() - 19]->getJson(JsonOptions::none); - return BEAST_EXPECT(res.has_payment()) && - BEAST_EXPECT( - res.payment().amount().value().has_xrp_amount()) && - BEAST_EXPECT( - res.payment() - .amount() - .value() - .xrp_amount() - .drops() == txnJson["Amount"].asUInt()) && - BEAST_EXPECT( - res.payment().destination().value().address() == - txnJson["Destination"]) && - BEAST_EXPECT(res.has_source_tag()) && - BEAST_EXPECT( - res.source_tag().value() == - txnJson["SourceTag"].asUInt()) && - BEAST_EXPECT(res.payment().has_destination_tag()) && - BEAST_EXPECT( - res.payment().destination_tag().value() == - txnJson["DestinationTag"].asUInt()) && - BEAST_EXPECT(res.has_last_ledger_sequence()) && - BEAST_EXPECT( - res.last_ledger_sequence().value() == - txnJson["LastLedgerSequence"].asUInt()) && - BEAST_EXPECT(res.has_transaction_signature()) && - BEAST_EXPECT(res.has_account()) && - BEAST_EXPECT( - res.account().value().address() == - txnJson["Account"]) && - BEAST_EXPECT(res.has_flags()) && - BEAST_EXPECT( - res.flags().value() == txnJson["Flags"].asUInt()); - }}, - {4, - 4, - strHex(txns[txns.size() - 20]->getTransactionID()), - [this](auto res) { return BEAST_EXPECT(res.has_account_set()); }}, - {3, - 3, - "9CE54C3B934E473A995B477E92EC229F99CED5B62BF4D2ACE4DC42719103AE2F", - [this](auto res) { - return BEAST_EXPECT(res.has_account_set()) && - BEAST_EXPECT(res.account_set().set_flag().value() == 8); - }}, - {1, - 3, - "2B5054734FA43C6C7B54F61944FAD6178ACD5D0272B39BA7FCD32A5D3932FBFF", - [&alice, this](auto res) { - return BEAST_EXPECT(res.has_payment()) && - BEAST_EXPECT( - res.payment().amount().value().has_xrp_amount()) && - BEAST_EXPECT( - res.payment() - .amount() - .value() - .xrp_amount() - .drops() == 1000000000010) && - BEAST_EXPECT( - res.payment().destination().value().address() == - alice.human()); - }}}; - - using MetaCheck = - std::function; - static const MetaCheck txMetaCheck[]{ - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](org::xrpl::rpc::v1::AffectedNode const& - entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 3) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DEPOSIT_PREAUTH; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 1) && - BEAST_EXPECT(meta.affected_nodes_size() == 5) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_CHECK; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 2); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 5) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_CHECK; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 2); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 1) && - BEAST_EXPECT(meta.affected_nodes_size() == 5) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_CHECK; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 2); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 5) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_CHECK; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 2); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 5) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_PAY_CHANNEL; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 2); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 2) && - - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_PAY_CHANNEL; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 5) && - - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_PAY_CHANNEL; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 2); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 1) && - BEAST_EXPECT(meta.affected_nodes_size() == 3) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ESCROW; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 3) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ESCROW; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 2) && - BEAST_EXPECT(meta.affected_nodes_size() == 3) && - - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ESCROW; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 1) && - BEAST_EXPECT(meta.affected_nodes_size() == 3) && - - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ESCROW; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 3) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_SIGNER_LIST; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 4) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_OFFER; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 1) && - BEAST_EXPECT(meta.affected_nodes_size() == 4) && - - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_OFFER; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 5) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_DIRECTORY_NODE; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_RIPPLE_STATE; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 2) && - BEAST_EXPECT(meta.affected_nodes_size() == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 1) && - BEAST_EXPECT(meta.affected_nodes_size() == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 2); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 2) && - BEAST_EXPECT(meta.affected_nodes_size() == 1) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 1); - }}, - {[this](auto meta) { - return BEAST_EXPECT(meta.transaction_index() == 0) && - BEAST_EXPECT(meta.affected_nodes_size() == 2) && - BEAST_EXPECT( - std::count_if( - meta.affected_nodes().begin(), - meta.affected_nodes().end(), - [](auto entry) { - return entry.ledger_entry_type() == - org::xrpl::rpc::v1::LedgerEntryType:: - LEDGER_ENTRY_TYPE_ACCOUNT_ROOT; - }) == 2); - }}}; - - auto doCheck = [this](auto txn, auto txCheck) { - return BEAST_EXPECT(txn.has_transaction()) && - BEAST_EXPECT(txn.validated()) && - BEAST_EXPECT(strHex(txn.hash()) == txCheck.hash) && - BEAST_EXPECT(txn.ledger_index() == txCheck.ledgerIndex) && - BEAST_EXPECT( - txn.transaction().sequence().value() == - txCheck.sequence) && - txCheck.checkTxn(txn.transaction()); - }; - - auto doMetaCheck = [this](auto txn, auto txMetaCheck) { - return BEAST_EXPECT(txn.has_meta()) && - BEAST_EXPECT(txn.meta().has_transaction_result()) && - BEAST_EXPECT( - txn.meta().transaction_result().result_type() == - org::xrpl::rpc::v1::TransactionResult:: - RESULT_TYPE_TES) && - BEAST_EXPECT( - txn.meta().transaction_result().result() == - "tesSUCCESS") && - txMetaCheck(txn.meta()); - }; - - auto [res, status] = next(grpcPort, env, alice.human()); - - if (!BEAST_EXPECT(status.error_code() == 0)) - return; - - if (!BEAST_EXPECT(res.transactions().size() == std::size(txCheck))) - return; - for (int i = 0; i < res.transactions().size(); ++i) - { - BEAST_EXPECT(doCheck(res.transactions()[i], txCheck[i])); - BEAST_EXPECT(doMetaCheck(res.transactions()[i], txMetaCheck[i])); - } - - // test binary representation - std::tie(res, status) = nextBinary(grpcPort, env, alice.human()); - - // txns vector does not contain the first two transactions returned by - // account_tx - if (!BEAST_EXPECT(res.transactions().size() == txns.size() + 2)) - return; - - std::reverse(txns.begin(), txns.end()); - for (int i = 0; i < txns.size(); ++i) - { - auto toByteString = [](auto data) { - const char* bytes = reinterpret_cast(data.data()); - return std::string(bytes, data.size()); - }; - - auto tx = txns[i]; - Serializer s = tx->getSerializer(); - std::string bin = toByteString(s); - - BEAST_EXPECT(res.transactions(i).transaction_binary() == bin); - } - } - - void - testAccountTxPagingGrpc() - { - testcase("Test Account_tx Grpc"); - - using namespace test::jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - - Account A1{"A1"}; - Account A2{"A2"}; - Account A3{"A3"}; - - env.fund(XRP(10000), A1, A2, A3); - env.close(); - - env.trust(A3["USD"](1000), A1); - env.trust(A2["USD"](1000), A1); - env.trust(A3["USD"](1000), A2); - env.close(); - - for (auto i = 0; i < 5; ++i) - { - env(pay(A2, A1, A2["USD"](2))); - env(pay(A3, A1, A3["USD"](2))); - env(offer(A1, XRP(11), A1["USD"](1))); - env(offer(A2, XRP(10), A2["USD"](1))); - env(offer(A3, XRP(9), A3["USD"](1))); - env.close(); - } - - /* The sequence/ledger for A3 are as follows: - * seq ledger_index - * 3 ----> 3 - * 1 ----> 3 - * 2 ----> 4 - * 2 ----> 4 - * 2 ----> 5 - * 3 ----> 5 - * 4 ----> 6 - * 5 ----> 6 - * 6 ----> 7 - * 7 ----> 7 - * 8 ----> 8 - * 9 ----> 8 - * 10 ----> 9 - * 11 ----> 9 - */ - - // page through the results in several ways. - { - // limit = 2, 3 batches giving the first 6 txs - auto [res, status] = next(grpcPort, env, A3.human(), 2, 5, 2, true); - - auto txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 2)) - return; - - BEAST_EXPECT(checkTransaction(txs[0u], 3, 3)); - BEAST_EXPECT(checkTransaction(txs[1u], 3, 3)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, env, A3.human(), 2, 5, 2, true, res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 2)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 4, 4)); - BEAST_EXPECT(checkTransaction(txs[1u], 4, 4)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, env, A3.human(), 2, 5, 2, true, res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 2)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 4, 5)); - BEAST_EXPECT(checkTransaction(txs[1u], 5, 5)); - BEAST_EXPECT(!res.has_marker()); - return; - } - - { - // limit 1, 3 requests giving the first 3 txs - auto [res, status] = next(grpcPort, env, A3.human(), 3, 9, 1, true); - auto txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 1)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 3, 3)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, env, A3.human(), 3, 9, 1, true, res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 1)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 3, 3)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, env, A3.human(), 3, 9, 1, true, res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 1)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 4, 4)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - // continue with limit 3, to end of all txs - std::tie(res, status) = next( - grpcPort, env, A3.human(), 3, 9, 3, true, res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 3)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 4, 4)); - BEAST_EXPECT(checkTransaction(txs[1u], 4, 5)); - BEAST_EXPECT(checkTransaction(txs[2u], 5, 5)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, env, A3.human(), 3, 9, 3, true, res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 3)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 6, 6)); - BEAST_EXPECT(checkTransaction(txs[1u], 7, 6)); - BEAST_EXPECT(checkTransaction(txs[2u], 8, 7)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, env, A3.human(), 3, 9, 3, true, res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 3)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 9, 7)); - BEAST_EXPECT(checkTransaction(txs[1u], 10, 8)); - BEAST_EXPECT(checkTransaction(txs[2u], 11, 8)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, env, A3.human(), 3, 9, 3, true, res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 2)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 12, 9)); - BEAST_EXPECT(checkTransaction(txs[1u], 13, 9)); - BEAST_EXPECT(!res.has_marker()); - } - - { - // limit 2, descending, 2 batches giving last 4 txs - auto [res, status] = - next(grpcPort, env, A3.human(), 3, 9, 2, false); - auto txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 2)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 13, 9)); - BEAST_EXPECT(checkTransaction(txs[1u], 12, 9)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, - env, - A3.human(), - 3, - 9, - 2, - false, - res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 2)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 11, 8)); - BEAST_EXPECT(checkTransaction(txs[1u], 10, 8)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - // continue with limit 3 until all txs have been seen - std::tie(res, status) = next( - grpcPort, - env, - A3.human(), - 3, - 9, - 3, - false, - res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 3)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 9, 7)); - BEAST_EXPECT(checkTransaction(txs[1u], 8, 7)); - BEAST_EXPECT(checkTransaction(txs[2u], 7, 6)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, - env, - A3.human(), - 3, - 9, - 3, - false, - res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 3)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 6, 6)); - BEAST_EXPECT(checkTransaction(txs[1u], 5, 5)); - BEAST_EXPECT(checkTransaction(txs[2u], 4, 5)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, - env, - A3.human(), - 3, - 9, - 3, - false, - res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 3)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 4, 4)); - BEAST_EXPECT(checkTransaction(txs[1u], 4, 4)); - BEAST_EXPECT(checkTransaction(txs[2u], 3, 3)); - if (!BEAST_EXPECT(res.has_marker())) - return; - - std::tie(res, status) = next( - grpcPort, - env, - A3.human(), - 3, - 9, - 3, - false, - res.mutable_marker()); - txs = res.transactions(); - if (!BEAST_EXPECT(txs.size() == 1)) - return; - BEAST_EXPECT(checkTransaction(txs[0u], 3, 3)); - BEAST_EXPECT(!res.has_marker()); - } - } - public: void run() override { testAccountTxPaging(); - testAccountTxPagingGrpc(); - testAccountTxParametersGrpc(); - testAccountTxContentsGrpc(); } }; diff --git a/src/test/app/FeeVote_test.cpp b/src/test/app/FeeVote_test.cpp index a2dd76fa4..4c2acf629 100644 --- a/src/test/app/FeeVote_test.cpp +++ b/src/test/app/FeeVote_test.cpp @@ -34,8 +34,8 @@ class FeeVote_test : public beast::unit_test::suite Section config; auto setup = setup_FeeVote(config); BEAST_EXPECT(setup.reference_fee == 10); - BEAST_EXPECT(setup.account_reserve == 20 * DROPS_PER_XRP); - BEAST_EXPECT(setup.owner_reserve == 5 * DROPS_PER_XRP); + BEAST_EXPECT(setup.account_reserve == 10 * DROPS_PER_XRP); + BEAST_EXPECT(setup.owner_reserve == 2 * DROPS_PER_XRP); } { Section config; @@ -57,8 +57,8 @@ class FeeVote_test : public beast::unit_test::suite // Illegal values are ignored, and the defaults left unchanged auto setup = setup_FeeVote(config); BEAST_EXPECT(setup.reference_fee == 10); - BEAST_EXPECT(setup.account_reserve == 20 * DROPS_PER_XRP); - BEAST_EXPECT(setup.owner_reserve == 5 * DROPS_PER_XRP); + BEAST_EXPECT(setup.account_reserve == 10 * DROPS_PER_XRP); + BEAST_EXPECT(setup.owner_reserve == 2 * DROPS_PER_XRP); } { Section config; @@ -87,8 +87,8 @@ class FeeVote_test : public beast::unit_test::suite // Illegal values are ignored, and the defaults left unchanged auto setup = setup_FeeVote(config); BEAST_EXPECT(setup.reference_fee == 10); - BEAST_EXPECT(setup.account_reserve == 20 * DROPS_PER_XRP); - BEAST_EXPECT(setup.owner_reserve == 5 * DROPS_PER_XRP); + BEAST_EXPECT(setup.account_reserve == 10 * DROPS_PER_XRP); + BEAST_EXPECT(setup.owner_reserve == 2 * DROPS_PER_XRP); } } diff --git a/src/test/app/LedgerLoad_test.cpp b/src/test/app/LedgerLoad_test.cpp index 7175df345..d78d25ea0 100644 --- a/src/test/app/LedgerLoad_test.cpp +++ b/src/test/app/LedgerLoad_test.cpp @@ -21,11 +21,12 @@ #include #include #include +#include +#include + #include #include #include -#include -#include namespace ripple { @@ -111,7 +112,9 @@ class LedgerLoad_test : public beast::unit_test::suite Env env( *this, envconfig( - ledgerConfig, sd.dbPath, sd.ledgerFile, Config::LOAD_FILE)); + ledgerConfig, sd.dbPath, sd.ledgerFile, Config::LOAD_FILE), + nullptr, + beast::severities::kDisabled); auto jrb = env.rpc("ledger", "current", "full")[jss::result]; BEAST_EXPECT( sd.ledger[jss::ledger][jss::accountState].size() == @@ -129,7 +132,9 @@ class LedgerLoad_test : public beast::unit_test::suite except([&] { Env env( *this, - envconfig(ledgerConfig, sd.dbPath, "", Config::LOAD_FILE)); + envconfig(ledgerConfig, sd.dbPath, "", Config::LOAD_FILE), + nullptr, + beast::severities::kDisabled); }); // file does not exist @@ -137,10 +142,9 @@ class LedgerLoad_test : public beast::unit_test::suite Env env( *this, envconfig( - ledgerConfig, - sd.dbPath, - "badfile.json", - Config::LOAD_FILE)); + ledgerConfig, sd.dbPath, "badfile.json", Config::LOAD_FILE), + nullptr, + beast::severities::kDisabled); }); // make a corrupted version of the ledger file (last 10 bytes removed). @@ -168,7 +172,9 @@ class LedgerLoad_test : public beast::unit_test::suite ledgerConfig, sd.dbPath, ledgerFileCorrupt.string(), - Config::LOAD_FILE)); + Config::LOAD_FILE), + nullptr, + beast::severities::kDisabled); }); } @@ -183,7 +189,9 @@ class LedgerLoad_test : public beast::unit_test::suite boost::erase_all(ledgerHash, "\""); Env env( *this, - envconfig(ledgerConfig, sd.dbPath, ledgerHash, Config::LOAD)); + envconfig(ledgerConfig, sd.dbPath, ledgerHash, Config::LOAD), + nullptr, + beast::severities::kDisabled); auto jrb = env.rpc("ledger", "current", "full")[jss::result]; BEAST_EXPECT(jrb[jss::ledger][jss::accountState].size() == 97); BEAST_EXPECT( @@ -199,7 +207,10 @@ class LedgerLoad_test : public beast::unit_test::suite // create a new env with the ledger "latest" specified for startup Env env( - *this, envconfig(ledgerConfig, sd.dbPath, "latest", Config::LOAD)); + *this, + envconfig(ledgerConfig, sd.dbPath, "latest", Config::LOAD), + nullptr, + beast::severities::kDisabled); auto jrb = env.rpc("ledger", "current", "full")[jss::result]; BEAST_EXPECT( sd.ledger[jss::ledger][jss::accountState].size() == @@ -213,7 +224,11 @@ class LedgerLoad_test : public beast::unit_test::suite using namespace test::jtx; // create a new env with specific ledger index at startup - Env env(*this, envconfig(ledgerConfig, sd.dbPath, "43", Config::LOAD)); + Env env( + *this, + envconfig(ledgerConfig, sd.dbPath, "43", Config::LOAD), + nullptr, + beast::severities::kDisabled); auto jrb = env.rpc("ledger", "current", "full")[jss::result]; BEAST_EXPECT( sd.ledger[jss::ledger][jss::accountState].size() == diff --git a/src/test/app/LedgerReplay_test.cpp b/src/test/app/LedgerReplay_test.cpp index 549495d40..cff94ee04 100644 --- a/src/test/app/LedgerReplay_test.cpp +++ b/src/test/app/LedgerReplay_test.cpp @@ -465,7 +465,7 @@ struct LedgerServer assert(param.initLedgers > 0); createAccounts(param.initAccounts); createLedgerHistory(); - app.logs().threshold(beast::severities::Severity::kWarning); + app.logs().threshold(beast::severities::kWarning); } /** @@ -567,7 +567,10 @@ public: PeerSetBehavior behavior = PeerSetBehavior::Good, InboundLedgersBehavior inboundBhvr = InboundLedgersBehavior::Good, PeerFeature peerFeature = PeerFeature::LedgerReplayEnabled) - : env(suite, jtx::envconfig(jtx::port_increment, 3)) + : env(suite, + jtx::envconfig(jtx::port_increment, 3), + nullptr, + beast::severities::kDisabled) , app(env.app()) , ledgerMaster(env.app().getLedgerMaster()) , inboundLedgers( diff --git a/src/test/app/Manifest_test.cpp b/src/test/app/Manifest_test.cpp index 47b16d948..db66e09e5 100644 --- a/src/test/app/Manifest_test.cpp +++ b/src/test/app/Manifest_test.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/test/app/MultiSign_test.cpp b/src/test/app/MultiSign_test.cpp index 6e9278f51..433cf2f7c 100644 --- a/src/test/app/MultiSign_test.cpp +++ b/src/test/app/MultiSign_test.cpp @@ -34,6 +34,30 @@ class MultiSign_test : public beast::unit_test::suite jtx::Account const phase{"phase", KeyType::ed25519}; jtx::Account const shade{"shade", KeyType::secp256k1}; jtx::Account const spook{"spook", KeyType::ed25519}; + jtx::Account const acc10{"acc10", KeyType::ed25519}; + jtx::Account const acc11{"acc11", KeyType::ed25519}; + jtx::Account const acc12{"acc12", KeyType::ed25519}; + jtx::Account const acc13{"acc13", KeyType::ed25519}; + jtx::Account const acc14{"acc14", KeyType::ed25519}; + jtx::Account const acc15{"acc15", KeyType::ed25519}; + jtx::Account const acc16{"acc16", KeyType::ed25519}; + jtx::Account const acc17{"acc17", KeyType::ed25519}; + jtx::Account const acc18{"acc18", KeyType::ed25519}; + jtx::Account const acc19{"acc19", KeyType::ed25519}; + jtx::Account const acc20{"acc20", KeyType::ed25519}; + jtx::Account const acc21{"acc21", KeyType::ed25519}; + jtx::Account const acc22{"acc22", KeyType::ed25519}; + jtx::Account const acc23{"acc23", KeyType::ed25519}; + jtx::Account const acc24{"acc24", KeyType::ed25519}; + jtx::Account const acc25{"acc25", KeyType::ed25519}; + jtx::Account const acc26{"acc26", KeyType::ed25519}; + jtx::Account const acc27{"acc27", KeyType::ed25519}; + jtx::Account const acc28{"acc28", KeyType::ed25519}; + jtx::Account const acc29{"acc29", KeyType::ed25519}; + jtx::Account const acc30{"acc30", KeyType::ed25519}; + jtx::Account const acc31{"acc31", KeyType::ed25519}; + jtx::Account const acc32{"acc32", KeyType::ed25519}; + jtx::Account const acc33{"acc33", KeyType::ed25519}; public: void @@ -159,22 +183,30 @@ public: {spook, 1}}), ter(temBAD_QUORUM)); - // Make a signer list that's too big. Should fail. + // clang-format off + // Make a signer list that's too big. Should fail. (Even with + // ExpandedSignerList) Account const spare("spare", KeyType::secp256k1); env(signers( alice, 1, - {{bogie, 1}, - {demon, 1}, - {ghost, 1}, - {haunt, 1}, - {jinni, 1}, - {phase, 1}, - {shade, 1}, - {spook, 1}, - {spare, 1}}), + features[featureExpandedSignerList] + ? std::vector{{bogie, 1}, {demon, 1}, {ghost, 1}, + {haunt, 1}, {jinni, 1}, {phase, 1}, + {shade, 1}, {spook, 1}, {spare, 1}, + {acc10, 1}, {acc11, 1}, {acc12, 1}, + {acc13, 1}, {acc14, 1}, {acc15, 1}, + {acc16, 1}, {acc17, 1}, {acc18, 1}, + {acc19, 1}, {acc20, 1}, {acc21, 1}, + {acc22, 1}, {acc23, 1}, {acc24, 1}, + {acc25, 1}, {acc26, 1}, {acc27, 1}, + {acc28, 1}, {acc29, 1}, {acc30, 1}, + {acc31, 1}, {acc32, 1}, {acc33, 1}} + : std::vector{{bogie, 1}, {demon, 1}, {ghost, 1}, + {haunt, 1}, {jinni, 1}, {phase, 1}, + {shade, 1}, {spook, 1}, {spare, 1}}), ter(temMALFORMED)); - + // clang-format on env.close(); env.require(owners(alice, 0)); } @@ -1149,20 +1181,56 @@ public: "fails local checks: Invalid Signers array size."); } { - // Multisign 9 times should fail. + // Multisign 9 (!ExpandedSignerList) | 33 (ExpandedSignerList) times + // should fail. JTx tx = env.jt( noop(alice), fee(2 * baseFee), - msig( - bogie, - bogie, - bogie, - bogie, - bogie, - bogie, - bogie, - bogie, - bogie)); + + features[featureExpandedSignerList] ? msig( + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie) + : msig( + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie, + bogie)); STTx local = *(tx.stx); auto const info = submitSTTx(local); BEAST_EXPECT( @@ -1517,6 +1585,82 @@ public: BEAST_EXPECT(env.seq(alice) == aliceSeq); } + void + test_signersWithTags(FeatureBitset features) + { + if (!features[featureExpandedSignerList]) + return; + + testcase("Signers With Tags"); + + using namespace jtx; + Env env{*this, features}; + Account const alice{"alice", KeyType::ed25519}; + env.fund(XRP(1000), alice); + env.close(); + uint8_t tag1[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}; + + uint8_t tag2[] = + "hello world some ascii 32b long"; // including 1 byte for NUL + + uint256 bogie_tag = ripple::base_uint<256>::fromVoid(tag1); + uint256 demon_tag = ripple::base_uint<256>::fromVoid(tag2); + + // Attach phantom signers to alice and use them for a transaction. + env(signers(alice, 1, {{bogie, 1, bogie_tag}, {demon, 1, demon_tag}})); + env.close(); + env.require(owners(alice, features[featureMultiSignReserve] ? 1 : 4)); + + // This should work. + auto const baseFee = env.current()->fees().base; + std::uint32_t aliceSeq = env.seq(alice); + env(noop(alice), msig(bogie, demon), fee(3 * baseFee)); + env.close(); + BEAST_EXPECT(env.seq(alice) == aliceSeq + 1); + + // Either signer alone should work. + aliceSeq = env.seq(alice); + env(noop(alice), msig(bogie), fee(2 * baseFee)); + env.close(); + BEAST_EXPECT(env.seq(alice) == aliceSeq + 1); + + aliceSeq = env.seq(alice); + env(noop(alice), msig(demon), fee(2 * baseFee)); + env.close(); + BEAST_EXPECT(env.seq(alice) == aliceSeq + 1); + + // Duplicate signers should fail. + aliceSeq = env.seq(alice); + env(noop(alice), msig(demon, demon), fee(3 * baseFee), ter(temINVALID)); + env.close(); + BEAST_EXPECT(env.seq(alice) == aliceSeq); + + // A non-signer should fail. + aliceSeq = env.seq(alice); + env(noop(alice), + msig(bogie, spook), + fee(3 * baseFee), + ter(tefBAD_SIGNATURE)); + env.close(); + BEAST_EXPECT(env.seq(alice) == aliceSeq); + + // Don't meet the quorum. Should fail. + env(signers(alice, 2, {{bogie, 1}, {demon, 1}})); + aliceSeq = env.seq(alice); + env(noop(alice), msig(bogie), fee(2 * baseFee), ter(tefBAD_QUORUM)); + env.close(); + BEAST_EXPECT(env.seq(alice) == aliceSeq); + + // Meet the quorum. Should succeed. + aliceSeq = env.seq(alice); + env(noop(alice), msig(bogie, demon), fee(3 * baseFee)); + env.close(); + BEAST_EXPECT(env.seq(alice) == aliceSeq + 1); + } + void testAll(FeatureBitset features) { @@ -1537,6 +1681,7 @@ public: test_multisigningMultisigner(features); test_signForHash(features); test_signersWithTickets(features); + test_signersWithTags(features); } void @@ -1545,10 +1690,13 @@ public: using namespace jtx; auto const all = supported_amendments(); - // The reserve required on a signer list changes based on. - // featureMultiSignReserve. Test both with and without. - testAll(all - featureMultiSignReserve); - testAll(all | featureMultiSignReserve); + // The reserve required on a signer list changes based on + // featureMultiSignReserve. Limits on the number of signers + // changes based on featureExpandedSignerList. Test both with and + // without. + testAll(all - featureMultiSignReserve - featureExpandedSignerList); + testAll(all - featureExpandedSignerList); + testAll(all); test_amendmentTransition(); } }; diff --git a/src/test/app/NFTokenBurn_test.cpp b/src/test/app/NFTokenBurn_test.cpp index e40f4c839..00124731c 100644 --- a/src/test/app/NFTokenBurn_test.cpp +++ b/src/test/app/NFTokenBurn_test.cpp @@ -595,8 +595,11 @@ public: run() override { using namespace test::jtx; - auto const sa = supported_amendments(); - testWithFeats(sa); + FeatureBitset const all{supported_amendments()}; + FeatureBitset const fixNFTDir{fixNFTokenDirV1}; + + testWithFeats(all - fixNFTDir); + testWithFeats(all); } }; diff --git a/src/test/app/NFTokenDir_test.cpp b/src/test/app/NFTokenDir_test.cpp index c19a8d079..d50bd1584 100644 --- a/src/test/app/NFTokenDir_test.cpp +++ b/src/test/app/NFTokenDir_test.cpp @@ -17,6 +17,7 @@ */ //============================================================================== +#include #include #include #include @@ -95,6 +96,63 @@ class NFTokenDir_test : public beast::unit_test::suite } } + void + testConsecutiveNFTs(FeatureBitset features) + { + // It should be possible to store many consecutive NFTs. + testcase("Sequential NFTs"); + + using namespace test::jtx; + Env env{*this, features}; + + // A single minter tends not to mint numerically sequential NFTokens + // because the taxon cipher mixes things up. We can override the + // cipher, however, and mint many sequential NFTokens with no gaps + // between them. + // + // Here we'll simply mint 100 sequential NFTs. Then we'll create + // offers for them to verify that the ledger can find them. + + Account const issuer{"issuer"}; + Account const buyer{"buyer"}; + env.fund(XRP(10000), buyer, issuer); + env.close(); + + // Mint 100 sequential NFTs. Tweak the taxon so zero is always stored. + // That's what makes them sequential. + constexpr std::size_t nftCount = 100; + std::vector nftIDs; + nftIDs.reserve(nftCount); + for (int i = 0; i < nftCount; ++i) + { + std::uint32_t taxon = + toUInt32(nft::cipheredTaxon(i, nft::toTaxon(0))); + nftIDs.emplace_back( + token::getNextID(env, issuer, taxon, tfTransferable)); + env(token::mint(issuer, taxon), txflags(tfTransferable)); + env.close(); + } + + // Create an offer for each of the NFTs. This verifies that the ledger + // can find all of the minted NFTs. + std::vector offers; + for (uint256 const& nftID : nftIDs) + { + offers.emplace_back(keylet::nftoffer(issuer, env.seq(issuer)).key); + env(token::createOffer(issuer, nftID, XRP(0)), + txflags((tfSellNFToken))); + env.close(); + } + + // Buyer accepts all of the offers in reverse order. + std::reverse(offers.begin(), offers.end()); + for (uint256 const& offer : offers) + { + env(token::acceptSellOffer(buyer, offer)); + env.close(); + } + } + void testLopsidedSplits(FeatureBitset features) { @@ -307,19 +365,231 @@ class NFTokenDir_test : public beast::unit_test::suite "sp6JS7f14BuwFY8Mw6xCigaMwC6Dp", // 32. 0x309b67ed }; - // FUTURE TEST - // These seeds fill the last 17 entries of the initial page with - // equivalent NFTs. The split should keep these together. - - // FUTURE TEST - // These seeds fill the first entries of the initial page with - // equivalent NFTs. The split should keep these together. - // Run the test cases. exerciseLopsided(splitAndAddToHi); exerciseLopsided(splitAndAddToLo); } + void + testFixNFTokenDirV1(FeatureBitset features) + { + // Exercise a fix for an off-by-one in the creation of an NFTokenPage + // index. + testcase("fixNFTokenDirV1"); + + using namespace test::jtx; + + // When a single NFT page exceeds 32 entries, the code is inclined + // to split that page into two equal pieces. The new page is lower + // than the original. There was an off-by-one in the selection of + // the index for the new page. This test recreates the problem. + + // Lambda that exercises the split. + auto exerciseFixNFTokenDirV1 = + [this, + &features](std::initializer_list seeds) { + Env env{ + *this, + envconfig(), + features, + nullptr, + beast::severities::kDisabled}; + + // Eventually all of the NFTokens will be owned by buyer. + Account const buyer{"buyer"}; + env.fund(XRP(10000), buyer); + env.close(); + + // Create accounts for all of the seeds and fund those accounts. + std::vector accounts; + accounts.reserve(seeds.size()); + for (std::string_view const& seed : seeds) + { + Account const& account = accounts.emplace_back( + Account::base58Seed, std::string(seed)); + env.fund(XRP(10000), account); + env.close(); + } + + // All of the accounts create one NFT and and offer that NFT to + // buyer. + std::vector nftIDs; + std::vector offers; + offers.reserve(accounts.size()); + for (Account const& account : accounts) + { + // Mint the NFT. + uint256 const& nftID = nftIDs.emplace_back( + token::getNextID(env, account, 0, tfTransferable)); + env(token::mint(account, 0), txflags(tfTransferable)); + env.close(); + + // Create an offer to give the NFT to buyer for free. + offers.emplace_back( + keylet::nftoffer(account, env.seq(account)).key); + env(token::createOffer(account, nftID, XRP(0)), + token::destination(buyer), + txflags((tfSellNFToken))); + } + env.close(); + + // buyer accepts all of the but the last. The last offer + // causes the page to split. + for (std::size_t i = 0; i < offers.size() - 1; ++i) + { + env(token::acceptSellOffer(buyer, offers[i])); + env.close(); + } + + // Here is the last offer. Without the fix accepting this + // offer causes tecINVARIANT_FAILED. With the fix the offer + // accept succeeds. + if (!features[fixNFTokenDirV1]) + { + env(token::acceptSellOffer(buyer, offers.back()), + ter(tecINVARIANT_FAILED)); + env.close(); + return; + } + env(token::acceptSellOffer(buyer, offers.back())); + env.close(); + + // This can be a good time to look at the NFT pages. + // printNFTPages(env, noisy); + + // Verify that all NFTs are owned by buyer and findable in the + // ledger by having buyer create sell offers for all of their + // NFTs. Attempting to sell an offer that the ledger can't find + // generates a non-tesSUCCESS error code. + for (uint256 const& nftID : nftIDs) + { + uint256 const offerID = + keylet::nftoffer(buyer, env.seq(buyer)).key; + env(token::createOffer(buyer, nftID, XRP(100)), + txflags(tfSellNFToken)); + env.close(); + + env(token::cancelOffer(buyer, {offerID})); + } + + // Verify that all the NFTs are owned by buyer. + Json::Value buyerNFTs = [&env, &buyer]() { + Json::Value params; + params[jss::account] = buyer.human(); + params[jss::type] = "state"; + return env.rpc("json", "account_nfts", to_string(params)); + }(); + + BEAST_EXPECT( + buyerNFTs[jss::result][jss::account_nfts].size() == + nftIDs.size()); + for (Json::Value const& ownedNFT : + buyerNFTs[jss::result][jss::account_nfts]) + { + uint256 ownedID; + BEAST_EXPECT(ownedID.parseHex( + ownedNFT[sfNFTokenID.jsonName].asString())); + auto const foundIter = + std::find(nftIDs.begin(), nftIDs.end(), ownedID); + + // Assuming we find the NFT, erase it so we know it's been + // found and can't be found again. + if (BEAST_EXPECT(foundIter != nftIDs.end())) + nftIDs.erase(foundIter); + } + + // All NFTs should now be accounted for, so nftIDs should be + // empty. + BEAST_EXPECT(nftIDs.empty()); + }; + + // These seeds fill the last 17 entries of the initial page with + // equivalent NFTs. The split should keep these together. + static std::initializer_list const seventeenHi{ + // These 16 need to be kept together by the implementation. + "sp6JS7f14BuwFY8Mw5EYu5z86hKDL", // 0. 0x399187e9 + "sp6JS7f14BuwFY8Mw5PUAMwc5ygd7", // 1. 0x399187e9 + "sp6JS7f14BuwFY8Mw5R3xUBcLSeTs", // 2. 0x399187e9 + "sp6JS7f14BuwFY8Mw5W6oS5sdC3oF", // 3. 0x399187e9 + "sp6JS7f14BuwFY8Mw5pYc3D9iuLcw", // 4. 0x399187e9 + "sp6JS7f14BuwFY8Mw5pfGVnhcdp3b", // 5. 0x399187e9 + "sp6JS7f14BuwFY8Mw6jS6RdEqXqrN", // 6. 0x399187e9 + "sp6JS7f14BuwFY8Mw6krt6AKbvRXW", // 7. 0x399187e9 + "sp6JS7f14BuwFY8Mw6mnVBQq7cAN2", // 8. 0x399187e9 + "sp6JS7f14BuwFY8Mw8ECJxPjmkufQ", // 9. 0x399187e9 + "sp6JS7f14BuwFY8Mw8asgzcceGWYm", // 10. 0x399187e9 + "sp6JS7f14BuwFY8MwF6J3FXnPCgL8", // 11. 0x399187e9 + "sp6JS7f14BuwFY8MwFEud2w5czv5q", // 12. 0x399187e9 + "sp6JS7f14BuwFY8MwFNxKVqJnx8P5", // 13. 0x399187e9 + "sp6JS7f14BuwFY8MwFnTCXg3eRidL", // 14. 0x399187e9 + "sp6JS7f14BuwFY8Mwj47hv1vrDge6", // 15. 0x399187e9 + + // These 17 need to be kept together by the implementation. + "sp6JS7f14BuwFY8MwjJCwYr9zSfAv", // 16. 0xabb11898 + "sp6JS7f14BuwFY8MwjYa5yLkgCLuT", // 17. 0xabb11898 + "sp6JS7f14BuwFY8MwjenxuJ3TH2Bc", // 18. 0xabb11898 + "sp6JS7f14BuwFY8MwjriN7Ui11NzB", // 19. 0xabb11898 + "sp6JS7f14BuwFY8Mwk3AuoJNSEo34", // 20. 0xabb11898 + "sp6JS7f14BuwFY8MwkT36hnRv8hTo", // 21. 0xabb11898 + "sp6JS7f14BuwFY8MwkTQixEXfi1Cr", // 22. 0xabb11898 + "sp6JS7f14BuwFY8MwkYJaZM1yTJBF", // 23. 0xabb11898 + "sp6JS7f14BuwFY8Mwkc4k1uo85qp2", // 24. 0xabb11898 + "sp6JS7f14BuwFY8Mwkf7cFhF1uuxx", // 25. 0xabb11898 + "sp6JS7f14BuwFY8MwmCK2un99wb4e", // 26. 0xabb11898 + "sp6JS7f14BuwFY8MwmETztNHYu2Bx", // 27. 0xabb11898 + "sp6JS7f14BuwFY8MwmJws9UwRASfR", // 28. 0xabb11898 + "sp6JS7f14BuwFY8MwoH5PQkGK8tEb", // 29. 0xabb11898 + "sp6JS7f14BuwFY8MwoVXtP2yCzjJV", // 30. 0xabb11898 + "sp6JS7f14BuwFY8MwobxRXA9vsTeX", // 31. 0xabb11898 + "sp6JS7f14BuwFY8Mwos3pc5Gb3ihU", // 32. 0xabb11898 + }; + + // These seeds fill the first entries of the initial page with + // equivalent NFTs. The split should keep these together. + static std::initializer_list const seventeenLo{ + // These 17 need to be kept together by the implementation. + "sp6JS7f14BuwFY8Mw5EYu5z86hKDL", // 0. 0x399187e9 + "sp6JS7f14BuwFY8Mw5PUAMwc5ygd7", // 1. 0x399187e9 + "sp6JS7f14BuwFY8Mw5R3xUBcLSeTs", // 2. 0x399187e9 + "sp6JS7f14BuwFY8Mw5W6oS5sdC3oF", // 3. 0x399187e9 + "sp6JS7f14BuwFY8Mw5pYc3D9iuLcw", // 4. 0x399187e9 + "sp6JS7f14BuwFY8Mw5pfGVnhcdp3b", // 5. 0x399187e9 + "sp6JS7f14BuwFY8Mw6jS6RdEqXqrN", // 6. 0x399187e9 + "sp6JS7f14BuwFY8Mw6krt6AKbvRXW", // 7. 0x399187e9 + "sp6JS7f14BuwFY8Mw6mnVBQq7cAN2", // 8. 0x399187e9 + "sp6JS7f14BuwFY8Mw8ECJxPjmkufQ", // 9. 0x399187e9 + "sp6JS7f14BuwFY8Mw8asgzcceGWYm", // 10. 0x399187e9 + "sp6JS7f14BuwFY8MwF6J3FXnPCgL8", // 11. 0x399187e9 + "sp6JS7f14BuwFY8MwFEud2w5czv5q", // 12. 0x399187e9 + "sp6JS7f14BuwFY8MwFNxKVqJnx8P5", // 13. 0x399187e9 + "sp6JS7f14BuwFY8MwFnTCXg3eRidL", // 14. 0x399187e9 + "sp6JS7f14BuwFY8Mwj47hv1vrDge6", // 15. 0x399187e9 + "sp6JS7f14BuwFY8Mwj6TYekeeyukh", // 16. 0x399187e9 + + // These 16 need to be kept together by the implementation. + "sp6JS7f14BuwFY8MwjYa5yLkgCLuT", // 17. 0xabb11898 + "sp6JS7f14BuwFY8MwjenxuJ3TH2Bc", // 18. 0xabb11898 + "sp6JS7f14BuwFY8MwjriN7Ui11NzB", // 19. 0xabb11898 + "sp6JS7f14BuwFY8Mwk3AuoJNSEo34", // 20. 0xabb11898 + "sp6JS7f14BuwFY8MwkT36hnRv8hTo", // 21. 0xabb11898 + "sp6JS7f14BuwFY8MwkTQixEXfi1Cr", // 22. 0xabb11898 + "sp6JS7f14BuwFY8MwkYJaZM1yTJBF", // 23. 0xabb11898 + "sp6JS7f14BuwFY8Mwkc4k1uo85qp2", // 24. 0xabb11898 + "sp6JS7f14BuwFY8Mwkf7cFhF1uuxx", // 25. 0xabb11898 + "sp6JS7f14BuwFY8MwmCK2un99wb4e", // 26. 0xabb11898 + "sp6JS7f14BuwFY8MwmETztNHYu2Bx", // 27. 0xabb11898 + "sp6JS7f14BuwFY8MwmJws9UwRASfR", // 28. 0xabb11898 + "sp6JS7f14BuwFY8MwoH5PQkGK8tEb", // 29. 0xabb11898 + "sp6JS7f14BuwFY8MwoVXtP2yCzjJV", // 30. 0xabb11898 + "sp6JS7f14BuwFY8MwobxRXA9vsTeX", // 31. 0xabb11898 + "sp6JS7f14BuwFY8Mwos3pc5Gb3ihU", // 32. 0xabb11898 + }; + + // Run the test cases. + exerciseFixNFTokenDirV1(seventeenHi); + exerciseFixNFTokenDirV1(seventeenLo); + } + void testTooManyEquivalent(FeatureBitset features) { @@ -338,23 +608,40 @@ class NFTokenDir_test : public beast::unit_test::suite // Here are 33 seeds that produce identical low 32-bits in their // corresponding AccountIDs. - // - // NOTE: We've not yet identified 33 AccountIDs that meet the - // requirements. At the moment 12 is the best we can do. We'll fill - // in the full count when they are available. static std::initializer_list const seeds{ - "sp6JS7f14BuwFY8Mw5G5vCrbxB3TZ", - "sp6JS7f14BuwFY8Mw5H6qyXhorcip", - "sp6JS7f14BuwFY8Mw5suWxsBQRqLx", - "sp6JS7f14BuwFY8Mw66gtwamvGgSg", - "sp6JS7f14BuwFY8Mw66iNV4PPcmyt", - "sp6JS7f14BuwFY8Mw68Qz2P58ybfE", - "sp6JS7f14BuwFY8Mw6AYtLXKzi2Bo", - "sp6JS7f14BuwFY8Mw6boCES4j62P2", - "sp6JS7f14BuwFY8Mw6kv7QDDv7wjw", - "sp6JS7f14BuwFY8Mw6mHXMvpBjjwg", - "sp6JS7f14BuwFY8Mw6qfGbznyYvVp", - "sp6JS7f14BuwFY8Mw6zg6qHKDfSoU", + "sp6JS7f14BuwFY8Mw5FnqmbciPvH6", // 0. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw5MBGbyMSsXLp", // 1. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw5S4PnDyBdKKm", // 2. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw6kcXpM2enE35", // 3. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw6tuuSMMwyJ44", // 4. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw8E8JWLQ1P8pt", // 5. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw8WwdgWkCHhEx", // 6. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw8XDUYvU6oGhQ", // 7. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw8ceVGL4M1zLQ", // 8. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw8fdSwLCZWDFd", // 9. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mw8zuF6Fg65i1E", // 10. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwF2k7bihVfqes", // 11. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwF6X24WXGn557", // 12. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwFMpn7strjekg", // 13. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwFSdy9sYVrwJs", // 14. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwFdMcLy9UkrXn", // 15. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwFdbwFm1AAboa", // 16. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwFdr5AhKThVtU", // 17. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwjFc3Q9YatvAw", // 18. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwjRXcNs1ozEXn", // 19. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwkQGUKL7v1FBt", // 20. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mwkamsoxx1wECt", // 21. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mwm3hus1dG6U8y", // 22. 0x9a8ebed3 + "sp6JS7f14BuwFY8Mwm589M8vMRpXF", // 23. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwmJTRJ4Fqz1A3", // 24. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwmRfy8fer4QbL", // 25. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwmkkFx1HtgWRx", // 26. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwmwP9JFdKa4PS", // 27. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwoXWJLB3ciHfo", // 28. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwoYc1gTtT2mWL", // 29. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwogXtHH7FNVoo", // 30. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwoqYoA9P8gf3r", // 31. 0x9a8ebed3 + "sp6JS7f14BuwFY8MwoujwMJofGnsA", // 32. 0x9a8ebed3 }; // Create accounts for all of the seeds and fund those accounts. @@ -396,15 +683,25 @@ class NFTokenDir_test : public beast::unit_test::suite BEAST_EXPECT(expectLowBits == (nftID & nft::pageMask)); } - // buyer accepts all of the offers. + // Remove one NFT and offer from the vectors. This offer is the one + // that will overflow the page. + nftIDs.pop_back(); + uint256 const offerForPageOverflow = offers.back(); + offers.pop_back(); + + // buyer accepts all of the offers but one. for (uint256 const& offer : offers) { env(token::acceptSellOffer(buyer, offer)); env.close(); } - // Verify that all NFTs are owned by buyer and findable in the - // ledger by having buyer create sell offers for all of their NFTs. + // buyer accepts the last offer which causes a page overflow. + env(token::acceptSellOffer(buyer, offerForPageOverflow), + ter(tecNO_SUITABLE_NFTOKEN_PAGE)); + + // Verify that all expected NFTs are owned by buyer and findable in + // the ledger by having buyer create sell offers for all of their NFTs. // Attempting to sell an offer that the ledger can't find generates // a non-tesSUCCESS error code. for (uint256 const& nftID : nftIDs) @@ -444,13 +741,332 @@ class NFTokenDir_test : public beast::unit_test::suite // All NFTs should now be accounted for, so nftIDs should be empty. BEAST_EXPECT(nftIDs.empty()); + + // Show that Without fixNFTokenDirV1 no more NFTs can be added to + // buyer. Also show that fixNFTokenDirV1 fixes the problem. + TER const expect = features[fixNFTokenDirV1] + ? static_cast(tesSUCCESS) + : static_cast(tecNO_SUITABLE_NFTOKEN_PAGE); + env(token::mint(buyer, 0), txflags(tfTransferable), ter(expect)); + env.close(); + } + + void + testConsecutivePacking(FeatureBitset features) + { + // We'll make a worst case scenario for NFT packing: + // + // 1. 33 accounts with identical low-32 bits mint 7 consecutive NFTs. + // 2. The taxon is manipulated to always be stored as zero. + // 3. A single account buys all 7x32 of the 33 NFTs. + // + // All of the NFTs should be acquired by the buyer. + // + // Lastly, none of the remaining NFTs should be acquirable by the + // buyer. They would cause page overflow. + + // This test collapses in a heap if fixNFTokenDirV1 is not enabled. + // If it is enabled just return so we skip the test. + if (!features[fixNFTokenDirV1]) + return; + + testcase("NFToken consecutive packing"); + + using namespace test::jtx; + + Env env{*this, features}; + + // Eventually all of the NFTokens will be owned by buyer. + Account const buyer{"buyer"}; + env.fund(XRP(10000), buyer); + env.close(); + + // Here are 33 seeds that produce identical low 32-bits in their + // corresponding AccountIDs. + static std::initializer_list const seeds{ + "sp6JS7f14BuwFY8Mw56vZeiBuhePx", // 0. 0x115d0525 + "sp6JS7f14BuwFY8Mw5BodF9tGuTUe", // 1. 0x115d0525 + "sp6JS7f14BuwFY8Mw5EnhC1cg84J7", // 2. 0x115d0525 + "sp6JS7f14BuwFY8Mw5P913Cunr2BK", // 3. 0x115d0525 + "sp6JS7f14BuwFY8Mw5Pru7eLo1XzT", // 4. 0x115d0525 + "sp6JS7f14BuwFY8Mw61SLUC8UX2m8", // 5. 0x115d0525 + "sp6JS7f14BuwFY8Mw6AsBF9TpeMpq", // 6. 0x115d0525 + "sp6JS7f14BuwFY8Mw84XqrBZkU2vE", // 7. 0x115d0525 + "sp6JS7f14BuwFY8Mw89oSU6dBk3KB", // 8. 0x115d0525 + "sp6JS7f14BuwFY8Mw89qUKCyDmyzj", // 9. 0x115d0525 + "sp6JS7f14BuwFY8Mw8GfqQ9VRZ8tm", // 10. 0x115d0525 + "sp6JS7f14BuwFY8Mw8LtW3VqrqMks", // 11. 0x115d0525 + "sp6JS7f14BuwFY8Mw8ZrAkJc2sHew", // 12. 0x115d0525 + "sp6JS7f14BuwFY8Mw8jpkYSNrD3ah", // 13. 0x115d0525 + "sp6JS7f14BuwFY8MwF2mshd786m3V", // 14. 0x115d0525 + "sp6JS7f14BuwFY8MwFHfXq9x5NbPY", // 15. 0x115d0525 + "sp6JS7f14BuwFY8MwFrjWq5LAB8NT", // 16. 0x115d0525 + "sp6JS7f14BuwFY8Mwj4asgSh6hQZd", // 17. 0x115d0525 + "sp6JS7f14BuwFY8Mwj7ipFfqBSRrE", // 18. 0x115d0525 + "sp6JS7f14BuwFY8MwjHqtcvGav8uW", // 19. 0x115d0525 + "sp6JS7f14BuwFY8MwjLp4sk5fmzki", // 20. 0x115d0525 + "sp6JS7f14BuwFY8MwjioHuYb3Ytkx", // 21. 0x115d0525 + "sp6JS7f14BuwFY8MwkRjHPXWi7fGN", // 22. 0x115d0525 + "sp6JS7f14BuwFY8MwkdVdPV3LjNN1", // 23. 0x115d0525 + "sp6JS7f14BuwFY8MwkxUtVY5AXZFk", // 24. 0x115d0525 + "sp6JS7f14BuwFY8Mwm4jQzdfTbY9F", // 25. 0x115d0525 + "sp6JS7f14BuwFY8MwmCucYAqNp4iF", // 26. 0x115d0525 + "sp6JS7f14BuwFY8Mwo2bgdFtxBzpF", // 27. 0x115d0525 + "sp6JS7f14BuwFY8MwoGwD7v4U6qBh", // 28. 0x115d0525 + "sp6JS7f14BuwFY8MwoUczqFADMoXi", // 29. 0x115d0525 + "sp6JS7f14BuwFY8MwoY1xZeGd3gAr", // 30. 0x115d0525 + "sp6JS7f14BuwFY8MwomVCbfkv4kYZ", // 31. 0x115d0525 + "sp6JS7f14BuwFY8MwoqbrPSr4z13F", // 32. 0x115d0525 + }; + + // Create accounts for all of the seeds and fund those accounts. + std::vector accounts; + accounts.reserve(seeds.size()); + for (std::string_view const& seed : seeds) + { + Account const& account = + accounts.emplace_back(Account::base58Seed, std::string(seed)); + env.fund(XRP(10000), account); + env.close(); + } + + // All of the accounts create seven consecutive NFTs and and offer + // those NFTs to buyer. + std::array, 7> nftIDsByPage; + for (auto& vec : nftIDsByPage) + vec.reserve(accounts.size()); + std::array, 7> offers; + for (auto& vec : offers) + vec.reserve(accounts.size()); + for (std::size_t i = 0; i < nftIDsByPage.size(); ++i) + { + for (Account const& account : accounts) + { + // Mint the NFT. Tweak the taxon so zero is always stored. + std::uint32_t taxon = + toUInt32(nft::cipheredTaxon(i, nft::toTaxon(0))); + + uint256 const& nftID = nftIDsByPage[i].emplace_back( + token::getNextID(env, account, taxon, tfTransferable)); + env(token::mint(account, taxon), txflags(tfTransferable)); + env.close(); + + // Create an offer to give the NFT to buyer for free. + offers[i].emplace_back( + keylet::nftoffer(account, env.seq(account)).key); + env(token::createOffer(account, nftID, XRP(0)), + token::destination(buyer), + txflags((tfSellNFToken))); + } + } + env.close(); + + // Verify that the low 96 bits of all generated NFTs of the same + // sequence is identical. + for (auto const& vec : nftIDsByPage) + { + uint256 const expectLowBits = vec.front() & nft::pageMask; + for (uint256 const& nftID : vec) + { + BEAST_EXPECT(expectLowBits == (nftID & nft::pageMask)); + } + } + + // Remove one NFT and offer from each of the vectors. These offers + // are the ones that will overflow the page. + std::vector overflowNFTs; + overflowNFTs.reserve(nftIDsByPage.size()); + std::vector overflowOffers; + overflowOffers.reserve(nftIDsByPage.size()); + + for (std::size_t i = 0; i < nftIDsByPage.size(); ++i) + { + overflowNFTs.push_back(nftIDsByPage[i].back()); + nftIDsByPage[i].pop_back(); + BEAST_EXPECT(nftIDsByPage[i].size() == seeds.size() - 1); + + overflowOffers.push_back(offers[i].back()); + offers[i].pop_back(); + BEAST_EXPECT(offers[i].size() == seeds.size() - 1); + } + + // buyer accepts all of the offers that won't cause an overflow. + // Fill the center and outsides first to exercise different boundary + // cases. + for (int i : std::initializer_list{3, 6, 0, 1, 2, 5, 4}) + { + for (uint256 const& offer : offers[i]) + { + env(token::acceptSellOffer(buyer, offer)); + env.close(); + } + } + + // buyer accepts the seven offers that would cause page overflows if + // the transaction succeeded. + for (uint256 const& offer : overflowOffers) + { + env(token::acceptSellOffer(buyer, offer), + ter(tecNO_SUITABLE_NFTOKEN_PAGE)); + env.close(); + } + + // Verify that all expected NFTs are owned by buyer and findable in + // the ledger by having buyer create sell offers for all of their NFTs. + // Attempting to sell an offer that the ledger can't find generates + // a non-tesSUCCESS error code. + for (auto const& vec : nftIDsByPage) + { + for (uint256 const& nftID : vec) + { + env(token::createOffer(buyer, nftID, XRP(100)), + txflags(tfSellNFToken)); + env.close(); + } + } + + // See what the account_objects command does with "nft_offer". + { + Json::Value ownedNftOffers(Json::arrayValue); + std::string marker; + do + { + Json::Value buyerOffers = [&env, &buyer, &marker]() { + Json::Value params; + params[jss::account] = buyer.human(); + params[jss::type] = jss::nft_offer; + + if (!marker.empty()) + params[jss::marker] = marker; + return env.rpc( + "json", "account_objects", to_string(params)); + }(); + + marker.clear(); + if (buyerOffers.isMember(jss::result)) + { + Json::Value& result = buyerOffers[jss::result]; + + if (result.isMember(jss::marker)) + marker = result[jss::marker].asString(); + + if (result.isMember(jss::account_objects)) + { + Json::Value& someOffers = result[jss::account_objects]; + for (std::size_t i = 0; i < someOffers.size(); ++i) + ownedNftOffers.append(someOffers[i]); + } + } + } while (!marker.empty()); + + // Verify there are as many offers are there are NFTs. + { + std::size_t totalOwnedNFTs = 0; + for (auto const& vec : nftIDsByPage) + totalOwnedNFTs += vec.size(); + BEAST_EXPECT(ownedNftOffers.size() == totalOwnedNFTs); + } + + // Cancel all the offers. + { + std::vector cancelOffers; + cancelOffers.reserve(ownedNftOffers.size()); + + for (auto const& offer : ownedNftOffers) + { + if (offer.isMember(jss::index)) + { + uint256 offerIndex; + if (offerIndex.parseHex(offer[jss::index].asString())) + cancelOffers.push_back(offerIndex); + } + } + env(token::cancelOffer(buyer, cancelOffers)); + env.close(); + } + + // account_objects should no longer return any "nft_offer"s. + Json::Value remainingOffers = [&env, &buyer]() { + Json::Value params; + params[jss::account] = buyer.human(); + params[jss::type] = jss::nft_offer; + + return env.rpc("json", "account_objects", to_string(params)); + }(); + BEAST_EXPECT( + remainingOffers.isMember(jss::result) && + remainingOffers[jss::result].isMember(jss::account_objects) && + remainingOffers[jss::result][jss::account_objects].size() == 0); + } + + // Verify that the ledger reports all of the NFTs owned by buyer. + // Use the account_nfts rpc call to get the values. + Json::Value ownedNFTs(Json::arrayValue); + std::string marker; + do + { + Json::Value buyerNFTs = [&env, &buyer, &marker]() { + Json::Value params; + params[jss::account] = buyer.human(); + params[jss::type] = "state"; + + if (!marker.empty()) + params[jss::marker] = marker; + return env.rpc("json", "account_nfts", to_string(params)); + }(); + + marker.clear(); + if (buyerNFTs.isMember(jss::result)) + { + Json::Value& result = buyerNFTs[jss::result]; + + if (result.isMember(jss::marker)) + marker = result[jss::marker].asString(); + + if (result.isMember(jss::account_nfts)) + { + Json::Value& someNFTs = result[jss::account_nfts]; + for (std::size_t i = 0; i < someNFTs.size(); ++i) + ownedNFTs.append(someNFTs[i]); + } + } + } while (!marker.empty()); + + // Copy all of the nftIDs into a set to make validation easier. + std::set allNftIDs; + for (auto& vec : nftIDsByPage) + allNftIDs.insert(vec.begin(), vec.end()); + + BEAST_EXPECT(ownedNFTs.size() == allNftIDs.size()); + + for (Json::Value const& ownedNFT : ownedNFTs) + { + if (ownedNFT.isMember(sfNFTokenID.jsonName)) + { + uint256 ownedID; + BEAST_EXPECT(ownedID.parseHex( + ownedNFT[sfNFTokenID.jsonName].asString())); + auto const foundIter = allNftIDs.find(ownedID); + + // Assuming we find the NFT, erase it so we know it's been found + // and can't be found again. + if (BEAST_EXPECT(foundIter != allNftIDs.end())) + allNftIDs.erase(foundIter); + } + } + + // All NFTs should now be accounted for, so allNftIDs should be empty. + BEAST_EXPECT(allNftIDs.empty()); } void testWithFeats(FeatureBitset features) { + testConsecutiveNFTs(features); testLopsidedSplits(features); + testFixNFTokenDirV1(features); testTooManyEquivalent(features); + testConsecutivePacking(features); } public: @@ -458,11 +1074,526 @@ public: run() override { using namespace test::jtx; - auto const sa = supported_amendments(); - testWithFeats(sa); + FeatureBitset const all{supported_amendments()}; + FeatureBitset const fixNFTDir{ + fixNFTokenDirV1, featureNonFungibleTokensV1_1}; + + testWithFeats(all - fixNFTDir); + testWithFeats(all); } }; BEAST_DEFINE_TESTSUITE_PRIO(NFTokenDir, tx, ripple, 1); } // namespace ripple + +// Seed that produces an account with the low-32 bits == 0xFFFFFFFF in +// case it is needed for future testing: +// +// sp6JS7f14BuwFY8MwFe95Vpi9Znjs +// + +// Sets of related accounts. +// +// Identifying the seeds of accounts that generate account IDs with the +// same low 32 bits takes a while. However several sets of accounts with +// that relationship have been located. In case these sets of accounts are +// needed for future testing scenarios they are recorded below. +#if 0 +34 account seeds that produce account IDs with low 32-bits 0x399187e9: + sp6JS7f14BuwFY8Mw5EYu5z86hKDL + sp6JS7f14BuwFY8Mw5PUAMwc5ygd7 + sp6JS7f14BuwFY8Mw5R3xUBcLSeTs + sp6JS7f14BuwFY8Mw5W6oS5sdC3oF + sp6JS7f14BuwFY8Mw5pYc3D9iuLcw + sp6JS7f14BuwFY8Mw5pfGVnhcdp3b + sp6JS7f14BuwFY8Mw6jS6RdEqXqrN + sp6JS7f14BuwFY8Mw6krt6AKbvRXW + sp6JS7f14BuwFY8Mw6mnVBQq7cAN2 + sp6JS7f14BuwFY8Mw8ECJxPjmkufQ + sp6JS7f14BuwFY8Mw8asgzcceGWYm + sp6JS7f14BuwFY8MwF6J3FXnPCgL8 + sp6JS7f14BuwFY8MwFEud2w5czv5q + sp6JS7f14BuwFY8MwFNxKVqJnx8P5 + sp6JS7f14BuwFY8MwFnTCXg3eRidL + sp6JS7f14BuwFY8Mwj47hv1vrDge6 + sp6JS7f14BuwFY8Mwj6TYekeeyukh + sp6JS7f14BuwFY8MwjFjsRDerz7jb + sp6JS7f14BuwFY8Mwjrj9mHTLBrcX + sp6JS7f14BuwFY8MwkKcJi3zMzAea + sp6JS7f14BuwFY8MwkYTDdnYRm9z4 + sp6JS7f14BuwFY8Mwkq8ei4D8uPNd + sp6JS7f14BuwFY8Mwm2pFruxbnJRd + sp6JS7f14BuwFY8MwmJV2ZnAjpC2g + sp6JS7f14BuwFY8MwmTFMPHQHfVYF + sp6JS7f14BuwFY8MwmkG2jXEgqiud + sp6JS7f14BuwFY8Mwms3xEh5tMDTw + sp6JS7f14BuwFY8MwmtipW4D8giZ9 + sp6JS7f14BuwFY8MwoRQBZm4KUUeE + sp6JS7f14BuwFY8MwoVey94QpXcrc + sp6JS7f14BuwFY8MwoZiuUoUTo3VG + sp6JS7f14BuwFY8MwonFFDLT4bHAZ + sp6JS7f14BuwFY8MwooGphD4hefBQ + sp6JS7f14BuwFY8MwoxDp3dmX6q5N + +34 account seeds that produce account IDs with low 32-bits 0x473f2c9a: + sp6JS7f14BuwFY8Mw53ktgqmv5Bmz + sp6JS7f14BuwFY8Mw5KPb2Kz7APFX + sp6JS7f14BuwFY8Mw5Xx4A6HRTPEE + sp6JS7f14BuwFY8Mw5y6qZFNAo358 + sp6JS7f14BuwFY8Mw6kdaBg1QrZfn + sp6JS7f14BuwFY8Mw8QmTfLMAZ5K1 + sp6JS7f14BuwFY8Mw8cbRRVcCEELr + sp6JS7f14BuwFY8Mw8gQvJebmxvDG + sp6JS7f14BuwFY8Mw8qPQurwu3P7Y + sp6JS7f14BuwFY8MwFS4PEVKmuPy5 + sp6JS7f14BuwFY8MwFUQM1rAsQ8tS + sp6JS7f14BuwFY8MwjJBZCkuwsRnM + sp6JS7f14BuwFY8MwjTdS8vZhX5E9 + sp6JS7f14BuwFY8MwjhSmWCbNhd25 + sp6JS7f14BuwFY8MwjwkpqwZsDBw9 + sp6JS7f14BuwFY8MwjyET4p6eqd5J + sp6JS7f14BuwFY8MwkMNAe4JhnG7E + sp6JS7f14BuwFY8MwkRRpnT93UWWS + sp6JS7f14BuwFY8MwkY9CvB22RvUe + sp6JS7f14BuwFY8Mwkhw9VxXqmTr7 + sp6JS7f14BuwFY8MwkmgaTat7eFa7 + sp6JS7f14BuwFY8Mwkq5SxGGv1oLH + sp6JS7f14BuwFY8MwmCBM5p5bTg6y + sp6JS7f14BuwFY8MwmmmXaVah64dB + sp6JS7f14BuwFY8Mwo7R7Cn614v9V + sp6JS7f14BuwFY8MwoCAG1na7GR2M + sp6JS7f14BuwFY8MwoDuPvJS4gG7C + sp6JS7f14BuwFY8MwoMMowSyPQLfy + sp6JS7f14BuwFY8MwoRqDiwTNsTBm + sp6JS7f14BuwFY8MwoWbBWtjpB7pg + sp6JS7f14BuwFY8Mwoi1AEeELGecF + sp6JS7f14BuwFY8MwopGP6Lo5byuj + sp6JS7f14BuwFY8MwoufkXGHp2VW8 + sp6JS7f14BuwFY8MwowGeagFQY32k + +34 account seeds that produce account IDs with low 32-bits 0x4d59f0d1: + sp6JS7f14BuwFY8Mw5CsNgH64zxK7 + sp6JS7f14BuwFY8Mw5Dg4wi2E344h + sp6JS7f14BuwFY8Mw5ErV949Zh2PX + sp6JS7f14BuwFY8Mw5p4nsQvEUE1s + sp6JS7f14BuwFY8Mw8LGnkbaP68Gn + sp6JS7f14BuwFY8Mw8aq6RCBc3iHo + sp6JS7f14BuwFY8Mw8bkWaGoKYT6e + sp6JS7f14BuwFY8Mw8qrCuXnzAXVj + sp6JS7f14BuwFY8MwFDKcPAHPHJTm + sp6JS7f14BuwFY8MwFUXJs4unfgNu + sp6JS7f14BuwFY8MwFj9Yv5LjshD9 + sp6JS7f14BuwFY8Mwj3H73nmq5UaC + sp6JS7f14BuwFY8MwjHSYShis1Yhk + sp6JS7f14BuwFY8MwjpfE1HVo8UP1 + sp6JS7f14BuwFY8Mwk6JE1SXUuiNc + sp6JS7f14BuwFY8MwkASgxEjEnFmU + sp6JS7f14BuwFY8MwkGNY8kg7R6RK + sp6JS7f14BuwFY8MwkHinNZ8SYBQu + sp6JS7f14BuwFY8MwkXLCW1hbhGya + sp6JS7f14BuwFY8MwkZ7mWrYK9YtU + sp6JS7f14BuwFY8MwkdFSqNB5DbKL + sp6JS7f14BuwFY8Mwm3jdBaCAx8H6 + sp6JS7f14BuwFY8Mwm3rk5hEwDRtY + sp6JS7f14BuwFY8Mwm77a2ULuwxu4 + sp6JS7f14BuwFY8MwmJpY7braKLaN + sp6JS7f14BuwFY8MwmKHQjG4XiZ6g + sp6JS7f14BuwFY8Mwmmv8Y3wyUDzs + sp6JS7f14BuwFY8MwmucFe1WgqtwG + sp6JS7f14BuwFY8Mwo1EjdU1bznZR + sp6JS7f14BuwFY8MwoJiqankkU5uR + sp6JS7f14BuwFY8MwoLnvQ6zdqbKw + sp6JS7f14BuwFY8MwoUGeJ319eu48 + sp6JS7f14BuwFY8MwoYf135tQjHP4 + sp6JS7f14BuwFY8MwogeF6M6SAyid + +34 account seeds that produce account IDs with low 32-bits 0xabb11898: + sp6JS7f14BuwFY8Mw5DgiYaNVSb1G + sp6JS7f14BuwFY8Mw5k6e94TMvuox + sp6JS7f14BuwFY8Mw5tTSN7KzYxiT + sp6JS7f14BuwFY8Mw61XV6m33utif + sp6JS7f14BuwFY8Mw87jKfrjiENCb + sp6JS7f14BuwFY8Mw8AFtxxFiRtJG + sp6JS7f14BuwFY8Mw8cosAVExzbeE + sp6JS7f14BuwFY8Mw8fmkQ63zE8WQ + sp6JS7f14BuwFY8Mw8iYSsxNbDN6D + sp6JS7f14BuwFY8Mw8wTZdGRJyyM1 + sp6JS7f14BuwFY8Mw8z7xEh3qBGr7 + sp6JS7f14BuwFY8MwFL5gpKQWZj7g + sp6JS7f14BuwFY8MwFPeZchXQnRZ5 + sp6JS7f14BuwFY8MwFSPxWSJVoU29 + sp6JS7f14BuwFY8MwFYyVkqX8kvRm + sp6JS7f14BuwFY8MwFcbVikUEwJvk + sp6JS7f14BuwFY8MwjF7NcZk1NctK + sp6JS7f14BuwFY8MwjJCwYr9zSfAv + sp6JS7f14BuwFY8MwjYa5yLkgCLuT + sp6JS7f14BuwFY8MwjenxuJ3TH2Bc + sp6JS7f14BuwFY8MwjriN7Ui11NzB + sp6JS7f14BuwFY8Mwk3AuoJNSEo34 + sp6JS7f14BuwFY8MwkT36hnRv8hTo + sp6JS7f14BuwFY8MwkTQixEXfi1Cr + sp6JS7f14BuwFY8MwkYJaZM1yTJBF + sp6JS7f14BuwFY8Mwkc4k1uo85qp2 + sp6JS7f14BuwFY8Mwkf7cFhF1uuxx + sp6JS7f14BuwFY8MwmCK2un99wb4e + sp6JS7f14BuwFY8MwmETztNHYu2Bx + sp6JS7f14BuwFY8MwmJws9UwRASfR + sp6JS7f14BuwFY8MwoH5PQkGK8tEb + sp6JS7f14BuwFY8MwoVXtP2yCzjJV + sp6JS7f14BuwFY8MwobxRXA9vsTeX + sp6JS7f14BuwFY8Mwos3pc5Gb3ihU + +34 account seeds that produce account IDs with low 32-bits 0xce627322: + sp6JS7f14BuwFY8Mw5Ck6i83pGNh3 + sp6JS7f14BuwFY8Mw5FKuwTxjAdH1 + sp6JS7f14BuwFY8Mw5FVKkEn6TkLH + sp6JS7f14BuwFY8Mw5NbQwLwHDd5v + sp6JS7f14BuwFY8Mw5X1dbz3msZaZ + sp6JS7f14BuwFY8Mw6qv6qaXNeP74 + sp6JS7f14BuwFY8Mw81SXagUeutCw + sp6JS7f14BuwFY8Mw84Ph7Qa8kwwk + sp6JS7f14BuwFY8Mw8Hp4gFyU3Qko + sp6JS7f14BuwFY8Mw8Kt8bAKredSx + sp6JS7f14BuwFY8Mw8XHK3VKRQ7v7 + sp6JS7f14BuwFY8Mw8eGyWxZGHY6v + sp6JS7f14BuwFY8Mw8iU5CLyHVcD2 + sp6JS7f14BuwFY8Mw8u3Zr26Ar914 + sp6JS7f14BuwFY8MwF2Kcdxtjzjv8 + sp6JS7f14BuwFY8MwFLmPWb6rbxNg + sp6JS7f14BuwFY8MwFUu8s7UVuxuJ + sp6JS7f14BuwFY8MwFYBaatwHxAJ8 + sp6JS7f14BuwFY8Mwjg6hFkeHwoqG + sp6JS7f14BuwFY8MwjjycJojy2ufk + sp6JS7f14BuwFY8MwkEWoxcSKGPXv + sp6JS7f14BuwFY8MwkMe7wLkEUsQT + sp6JS7f14BuwFY8MwkvyKLaPUc4FS + sp6JS7f14BuwFY8Mwm8doqXPKZmVQ + sp6JS7f14BuwFY8Mwm9r3No8yQ8Tx + sp6JS7f14BuwFY8Mwm9w6dks68W9B + sp6JS7f14BuwFY8MwmMPrv9sCdbpS + sp6JS7f14BuwFY8MwmPAvs3fcQNja + sp6JS7f14BuwFY8MwmS5jasapfcnJ + sp6JS7f14BuwFY8MwmU2L3qJEhnuA + sp6JS7f14BuwFY8MwoAQYmiBnW7fM + sp6JS7f14BuwFY8MwoBkkkXrPmkKF + sp6JS7f14BuwFY8MwonfmxPo6tkvC + sp6JS7f14BuwFY8MwouZFwhiNcYq6 + +34 account seeds that produce account IDs with low 32-bits 0xe29643e8: + sp6JS7f14BuwFY8Mw5EfAavcXAh2k + sp6JS7f14BuwFY8Mw5LhFjLkFSCVF + sp6JS7f14BuwFY8Mw5bRfEv5HgdBh + sp6JS7f14BuwFY8Mw5d6sPcKzypKN + sp6JS7f14BuwFY8Mw5rcqDtk1fACP + sp6JS7f14BuwFY8Mw5xkxRq1Notzv + sp6JS7f14BuwFY8Mw66fbkdw5WYmt + sp6JS7f14BuwFY8Mw6diEG8sZ7Fx7 + sp6JS7f14BuwFY8Mw6v2r1QhG7xc1 + sp6JS7f14BuwFY8Mw6zP6DHCTx2Fd + sp6JS7f14BuwFY8Mw8B3n39JKuFkk + sp6JS7f14BuwFY8Mw8FmBvqYw7uqn + sp6JS7f14BuwFY8Mw8KEaftb1eRwu + sp6JS7f14BuwFY8Mw8WJ1qKkegj9N + sp6JS7f14BuwFY8Mw8r8cAZEkq2BS + sp6JS7f14BuwFY8MwFKPxxwF65gZh + sp6JS7f14BuwFY8MwFKhaF8APcN5H + sp6JS7f14BuwFY8MwFN2buJn4BgYC + sp6JS7f14BuwFY8MwFUTe175MjP3x + sp6JS7f14BuwFY8MwFZhmRDb53NNb + sp6JS7f14BuwFY8MwFa2Azn5nU2WS + sp6JS7f14BuwFY8MwjNNt91hwgkn7 + sp6JS7f14BuwFY8MwjdiYt6ChACe7 + sp6JS7f14BuwFY8Mwk5qFVQ48Mmr9 + sp6JS7f14BuwFY8MwkGvCj7pNf1zG + sp6JS7f14BuwFY8MwkY9UcN2D2Fzs + sp6JS7f14BuwFY8MwkpGvSk9G9RyT + sp6JS7f14BuwFY8MwmGQ7nJf1eEzV + sp6JS7f14BuwFY8MwmQLjGsYdyAmV + sp6JS7f14BuwFY8MwmZ8usztKvikT + sp6JS7f14BuwFY8MwobyMLC2hQdFR + sp6JS7f14BuwFY8MwoiRtwUecZeJ5 + sp6JS7f14BuwFY8MwojHjKsUzj1KJ + sp6JS7f14BuwFY8Mwop29anGAjidU + +33 account seeds that produce account IDs with low 32-bits 0x115d0525: + sp6JS7f14BuwFY8Mw56vZeiBuhePx + sp6JS7f14BuwFY8Mw5BodF9tGuTUe + sp6JS7f14BuwFY8Mw5EnhC1cg84J7 + sp6JS7f14BuwFY8Mw5P913Cunr2BK + sp6JS7f14BuwFY8Mw5Pru7eLo1XzT + sp6JS7f14BuwFY8Mw61SLUC8UX2m8 + sp6JS7f14BuwFY8Mw6AsBF9TpeMpq + sp6JS7f14BuwFY8Mw84XqrBZkU2vE + sp6JS7f14BuwFY8Mw89oSU6dBk3KB + sp6JS7f14BuwFY8Mw89qUKCyDmyzj + sp6JS7f14BuwFY8Mw8GfqQ9VRZ8tm + sp6JS7f14BuwFY8Mw8LtW3VqrqMks + sp6JS7f14BuwFY8Mw8ZrAkJc2sHew + sp6JS7f14BuwFY8Mw8jpkYSNrD3ah + sp6JS7f14BuwFY8MwF2mshd786m3V + sp6JS7f14BuwFY8MwFHfXq9x5NbPY + sp6JS7f14BuwFY8MwFrjWq5LAB8NT + sp6JS7f14BuwFY8Mwj4asgSh6hQZd + sp6JS7f14BuwFY8Mwj7ipFfqBSRrE + sp6JS7f14BuwFY8MwjHqtcvGav8uW + sp6JS7f14BuwFY8MwjLp4sk5fmzki + sp6JS7f14BuwFY8MwjioHuYb3Ytkx + sp6JS7f14BuwFY8MwkRjHPXWi7fGN + sp6JS7f14BuwFY8MwkdVdPV3LjNN1 + sp6JS7f14BuwFY8MwkxUtVY5AXZFk + sp6JS7f14BuwFY8Mwm4jQzdfTbY9F + sp6JS7f14BuwFY8MwmCucYAqNp4iF + sp6JS7f14BuwFY8Mwo2bgdFtxBzpF + sp6JS7f14BuwFY8MwoGwD7v4U6qBh + sp6JS7f14BuwFY8MwoUczqFADMoXi + sp6JS7f14BuwFY8MwoY1xZeGd3gAr + sp6JS7f14BuwFY8MwomVCbfkv4kYZ + sp6JS7f14BuwFY8MwoqbrPSr4z13F + +33 account seeds that produce account IDs with low 32-bits 0x304033aa: + sp6JS7f14BuwFY8Mw5DaUP9agF5e1 + sp6JS7f14BuwFY8Mw5ohbtmPN4yGN + sp6JS7f14BuwFY8Mw5rRsA5fcoTAQ + sp6JS7f14BuwFY8Mw6zpYHMY3m6KT + sp6JS7f14BuwFY8Mw86BzQq4sTnoW + sp6JS7f14BuwFY8Mw8CCpnfvmGdV7 + sp6JS7f14BuwFY8Mw8DRjUDaBcFco + sp6JS7f14BuwFY8Mw8cL7GPo3zZN7 + sp6JS7f14BuwFY8Mw8y6aeYVtH6qt + sp6JS7f14BuwFY8MwFZR3PtVTCdUH + sp6JS7f14BuwFY8MwFcdcdbgz7m3s + sp6JS7f14BuwFY8MwjdnJDiUxEBRR + sp6JS7f14BuwFY8MwjhxWgSntqrFe + sp6JS7f14BuwFY8MwjrSHEhZ8CUM1 + sp6JS7f14BuwFY8MwjzkEeSTc9ZYf + sp6JS7f14BuwFY8MwkBZSk9JhaeCB + sp6JS7f14BuwFY8MwkGfwNY4i2iiU + sp6JS7f14BuwFY8MwknjtZd2oU2Ff + sp6JS7f14BuwFY8Mwkszsqd3ok9NE + sp6JS7f14BuwFY8Mwm58A81MAMvgZ + sp6JS7f14BuwFY8MwmiPTWysuDJCH + sp6JS7f14BuwFY8MwmxhiNeLfD76r + sp6JS7f14BuwFY8Mwo7SPdkwpGrFH + sp6JS7f14BuwFY8MwoANq4F1Sj3qH + sp6JS7f14BuwFY8MwoVjcHufAkd6L + sp6JS7f14BuwFY8MwoVxHBXdaxzhm + sp6JS7f14BuwFY8MwoZ2oTjBNfLpm + sp6JS7f14BuwFY8Mwoc9swzyotFVD + sp6JS7f14BuwFY8MwogMqVRwVEcQ9 + sp6JS7f14BuwFY8MwohMm7WxwnFqH + sp6JS7f14BuwFY8MwopUcpZHuF8BH + sp6JS7f14BuwFY8Mwor6rW6SS7tiB + sp6JS7f14BuwFY8MwoxyaqYz4Ngsb + +33 account seeds that produce account IDs with low 32-bits 0x42d4e09c: + sp6JS7f14BuwFY8Mw58NSZH9EaUxQ + sp6JS7f14BuwFY8Mw5JByk1pgPpL7 + sp6JS7f14BuwFY8Mw5YrJJuXnkHVB + sp6JS7f14BuwFY8Mw5kZe2ZzNSnKR + sp6JS7f14BuwFY8Mw6eXHTsbwi1U7 + sp6JS7f14BuwFY8Mw6gqN7HHDDKSh + sp6JS7f14BuwFY8Mw6zw8L1sSSR53 + sp6JS7f14BuwFY8Mw8E4WqSKKbksy + sp6JS7f14BuwFY8MwF3V9gemqJtND + sp6JS7f14BuwFY8Mwj4j46LHWZuY6 + sp6JS7f14BuwFY8MwjF5i8vh4Ezjy + sp6JS7f14BuwFY8MwjJZpEKgMpUAt + sp6JS7f14BuwFY8MwjWL7LfnzNUuh + sp6JS7f14BuwFY8Mwk7Y1csGuqAhX + sp6JS7f14BuwFY8MwkB1HVH17hN5W + sp6JS7f14BuwFY8MwkBntH7BZZupu + sp6JS7f14BuwFY8MwkEy4rMbNHG9P + sp6JS7f14BuwFY8MwkKz4LYesZeiN + sp6JS7f14BuwFY8MwkUrXyo9gMDPM + sp6JS7f14BuwFY8MwkV2hySsxej1G + sp6JS7f14BuwFY8MwkozhTVN12F9C + sp6JS7f14BuwFY8MwkpkzGB3sFJw5 + sp6JS7f14BuwFY8Mwks3zDZLGrhdn + sp6JS7f14BuwFY8MwktG1KCS7L2wW + sp6JS7f14BuwFY8Mwm1jVFsafwcYx + sp6JS7f14BuwFY8Mwm8hmrU6g5Wd6 + sp6JS7f14BuwFY8MwmFvstfRF7e2f + sp6JS7f14BuwFY8MwmeRohi6m5fs8 + sp6JS7f14BuwFY8MwmmU96RHUaRZL + sp6JS7f14BuwFY8MwoDFzteYqaUh4 + sp6JS7f14BuwFY8MwoPkTf5tDykPF + sp6JS7f14BuwFY8MwoSbMaDtiMoDN + sp6JS7f14BuwFY8MwoVL1vY1CysjR + +33 account seeds that produce account IDs with low 32-bits 0x9a8ebed3: + sp6JS7f14BuwFY8Mw5FnqmbciPvH6 + sp6JS7f14BuwFY8Mw5MBGbyMSsXLp + sp6JS7f14BuwFY8Mw5S4PnDyBdKKm + sp6JS7f14BuwFY8Mw6kcXpM2enE35 + sp6JS7f14BuwFY8Mw6tuuSMMwyJ44 + sp6JS7f14BuwFY8Mw8E8JWLQ1P8pt + sp6JS7f14BuwFY8Mw8WwdgWkCHhEx + sp6JS7f14BuwFY8Mw8XDUYvU6oGhQ + sp6JS7f14BuwFY8Mw8ceVGL4M1zLQ + sp6JS7f14BuwFY8Mw8fdSwLCZWDFd + sp6JS7f14BuwFY8Mw8zuF6Fg65i1E + sp6JS7f14BuwFY8MwF2k7bihVfqes + sp6JS7f14BuwFY8MwF6X24WXGn557 + sp6JS7f14BuwFY8MwFMpn7strjekg + sp6JS7f14BuwFY8MwFSdy9sYVrwJs + sp6JS7f14BuwFY8MwFdMcLy9UkrXn + sp6JS7f14BuwFY8MwFdbwFm1AAboa + sp6JS7f14BuwFY8MwFdr5AhKThVtU + sp6JS7f14BuwFY8MwjFc3Q9YatvAw + sp6JS7f14BuwFY8MwjRXcNs1ozEXn + sp6JS7f14BuwFY8MwkQGUKL7v1FBt + sp6JS7f14BuwFY8Mwkamsoxx1wECt + sp6JS7f14BuwFY8Mwm3hus1dG6U8y + sp6JS7f14BuwFY8Mwm589M8vMRpXF + sp6JS7f14BuwFY8MwmJTRJ4Fqz1A3 + sp6JS7f14BuwFY8MwmRfy8fer4QbL + sp6JS7f14BuwFY8MwmkkFx1HtgWRx + sp6JS7f14BuwFY8MwmwP9JFdKa4PS + sp6JS7f14BuwFY8MwoXWJLB3ciHfo + sp6JS7f14BuwFY8MwoYc1gTtT2mWL + sp6JS7f14BuwFY8MwogXtHH7FNVoo + sp6JS7f14BuwFY8MwoqYoA9P8gf3r + sp6JS7f14BuwFY8MwoujwMJofGnsA + +33 account seeds that produce account IDs with low 32-bits 0xa1dcea4a: + sp6JS7f14BuwFY8Mw5Ccov2N36QTy + sp6JS7f14BuwFY8Mw5CuSemVb5p7w + sp6JS7f14BuwFY8Mw5Ep8wpsTfpSz + sp6JS7f14BuwFY8Mw5WtutJc2H45M + sp6JS7f14BuwFY8Mw6vsDeaSKeUJZ + sp6JS7f14BuwFY8Mw83t5BPWUAzzF + sp6JS7f14BuwFY8Mw8FYGnK35mgkV + sp6JS7f14BuwFY8Mw8huo1x5pfKKJ + sp6JS7f14BuwFY8Mw8mPStxfMDrZa + sp6JS7f14BuwFY8Mw8yC3A7aQJytK + sp6JS7f14BuwFY8MwFCWCDmo9o3t8 + sp6JS7f14BuwFY8MwFjapa4gKxPhR + sp6JS7f14BuwFY8Mwj8CWtG29uw71 + sp6JS7f14BuwFY8MwjHyU5KpEMLVT + sp6JS7f14BuwFY8MwjMZSN7LZuWD8 + sp6JS7f14BuwFY8Mwja2TXJNBhKHU + sp6JS7f14BuwFY8Mwjf3xNTopHKTF + sp6JS7f14BuwFY8Mwjn5RAhedPeuM + sp6JS7f14BuwFY8MwkJdr4d6QoE8K + sp6JS7f14BuwFY8MwkmBryo3SUoLm + sp6JS7f14BuwFY8MwkrPdsc4tR8yw + sp6JS7f14BuwFY8Mwkttjcw2a65Fi + sp6JS7f14BuwFY8Mwm19n3rSaNx5S + sp6JS7f14BuwFY8Mwm3ryr4Xp2aQX + sp6JS7f14BuwFY8MwmBnDmgnJLB6B + sp6JS7f14BuwFY8MwmHgPjzrYjthq + sp6JS7f14BuwFY8MwmeV55DAnWKdd + sp6JS7f14BuwFY8Mwo49hK6BGrauT + sp6JS7f14BuwFY8Mwo56vfKY9aoWu + sp6JS7f14BuwFY8MwoU7tTTXLQTrh + sp6JS7f14BuwFY8MwoXpogSF2KaZB + sp6JS7f14BuwFY8MwoY9JYQAR16pc + sp6JS7f14BuwFY8MwoozLzKNAEXKM + +33 account seeds that produce account IDs with low 32-bits 0xbd2116db: + sp6JS7f14BuwFY8Mw5GrpkmPuA3Bw + sp6JS7f14BuwFY8Mw5r1sLoQJZDc6 + sp6JS7f14BuwFY8Mw68zzRmezLdd6 + sp6JS7f14BuwFY8Mw6jDSyaiF1mRp + sp6JS7f14BuwFY8Mw813wU9u5D6Uh + sp6JS7f14BuwFY8Mw8BBvpf2JFGoJ + sp6JS7f14BuwFY8Mw8F7zXxAiT263 + sp6JS7f14BuwFY8Mw8XG7WuVGHP2N + sp6JS7f14BuwFY8Mw8eyWrcz91cz6 + sp6JS7f14BuwFY8Mw8yNVKFVYyk9u + sp6JS7f14BuwFY8MwF2oA6ePqvZWP + sp6JS7f14BuwFY8MwF9VkcSNh3keq + sp6JS7f14BuwFY8MwFYsMWajgEf2j + sp6JS7f14BuwFY8Mwj3Gu43jYoJ4n + sp6JS7f14BuwFY8MwjJ5iRmYDHrW4 + sp6JS7f14BuwFY8MwjaUSSga93CiM + sp6JS7f14BuwFY8MwjxgLh2FY4Lvt + sp6JS7f14BuwFY8Mwk9hQdNZUgmTB + sp6JS7f14BuwFY8MwkcMXqtFp1sMx + sp6JS7f14BuwFY8MwkzZCDc56jsUB + sp6JS7f14BuwFY8Mwm5Zz7fP24Qym + sp6JS7f14BuwFY8MwmDWqizXSoJRG + sp6JS7f14BuwFY8MwmKHmkNYdMqqi + sp6JS7f14BuwFY8MwmRfAWHxWpGNK + sp6JS7f14BuwFY8MwmjCdXwyhphZ1 + sp6JS7f14BuwFY8MwmmukDAm1w6FL + sp6JS7f14BuwFY8Mwmmz2SzaR9TRH + sp6JS7f14BuwFY8Mwmz2z5mKHXzfn + sp6JS7f14BuwFY8Mwo2xNe5629r5k + sp6JS7f14BuwFY8MwoKy8tZxZrfJw + sp6JS7f14BuwFY8MwoLyQ9aMsq8Dm + sp6JS7f14BuwFY8MwoqqYkewuyZck + sp6JS7f14BuwFY8MwouvvhREVp6Pp + +33 account seeds that produce account IDs with low 32-bits 0xd80df065: + sp6JS7f14BuwFY8Mw5B7ERyhAfgHA + sp6JS7f14BuwFY8Mw5VuW3cF7bm2v + sp6JS7f14BuwFY8Mw5py3t1j7YbFT + sp6JS7f14BuwFY8Mw5qc84SzB6RHr + sp6JS7f14BuwFY8Mw5vGHW1G1hAy8 + sp6JS7f14BuwFY8Mw6gVa8TYukws6 + sp6JS7f14BuwFY8Mw8K9w1RoUAv1w + sp6JS7f14BuwFY8Mw8KvKtB7787CA + sp6JS7f14BuwFY8Mw8Y7WhRbuFzRq + sp6JS7f14BuwFY8Mw8cipw7inRmMn + sp6JS7f14BuwFY8MwFM5fAUNLNB13 + sp6JS7f14BuwFY8MwFSe1zAsht3X3 + sp6JS7f14BuwFY8MwFYNdigqQuHZM + sp6JS7f14BuwFY8MwjWkejj7V4V5Q + sp6JS7f14BuwFY8Mwjd2JGpsjvynq + sp6JS7f14BuwFY8Mwjg1xkducn751 + sp6JS7f14BuwFY8Mwjsp6LnaJvL1W + sp6JS7f14BuwFY8MwjvSbLc9593yH + sp6JS7f14BuwFY8Mwjw2h5wx7U6vZ + sp6JS7f14BuwFY8MwjxKUjtRsmPLH + sp6JS7f14BuwFY8Mwk1Yy8ginDfqv + sp6JS7f14BuwFY8Mwk2HrWhWwZP12 + sp6JS7f14BuwFY8Mwk4SsqiexvpWs + sp6JS7f14BuwFY8Mwk66zCs5ACpE6 + sp6JS7f14BuwFY8MwkCwx6vY97Nwh + sp6JS7f14BuwFY8MwknrbjnhTTWU8 + sp6JS7f14BuwFY8MwkokDy2ShRzQx + sp6JS7f14BuwFY8Mwm3BxnRPNxsuu + sp6JS7f14BuwFY8MwmY9EWdQQsFVr + sp6JS7f14BuwFY8MwmYTWjrDhmk8S + sp6JS7f14BuwFY8Mwo9skXt9Y5BVS + sp6JS7f14BuwFY8MwoZYKZybJ1Crp + sp6JS7f14BuwFY8MwoyXqkhySfSmF + +33 account seeds that produce account IDs with low 32-bits 0xe2e44294: + sp6JS7f14BuwFY8Mw53dmvTgNtBwi + sp6JS7f14BuwFY8Mw5Wrxsqn6WrXW + sp6JS7f14BuwFY8Mw5fGDT31RCXgC + sp6JS7f14BuwFY8Mw5nKRkubwrLWM + sp6JS7f14BuwFY8Mw5nXMajwKjriB + sp6JS7f14BuwFY8Mw5xZybggrC9NG + sp6JS7f14BuwFY8Mw5xea8f6dBMV5 + sp6JS7f14BuwFY8Mw5zDGofAHy5Lb + sp6JS7f14BuwFY8Mw6eado41rQNVG + sp6JS7f14BuwFY8Mw6yqKXQsQJPuU + sp6JS7f14BuwFY8Mw83MSN4FDzSGH + sp6JS7f14BuwFY8Mw8B3pUbzQqHe2 + sp6JS7f14BuwFY8Mw8WwRLnhBRvfk + sp6JS7f14BuwFY8Mw8hDBpKbpJwJX + sp6JS7f14BuwFY8Mw8jggRSZACe7M + sp6JS7f14BuwFY8Mw8mJRpU3qWbwC + sp6JS7f14BuwFY8MwFDnVozykN21u + sp6JS7f14BuwFY8MwFGGRGY9fctgv + sp6JS7f14BuwFY8MwjKznfChH9DQb + sp6JS7f14BuwFY8MwjbC5GvngRCk6 + sp6JS7f14BuwFY8Mwk3Lb7FPe1629 + sp6JS7f14BuwFY8MwkCeS41BwVrBD + sp6JS7f14BuwFY8MwkDnnvRyuWJ7d + sp6JS7f14BuwFY8MwkbkRNnzDEFpf + sp6JS7f14BuwFY8MwkiNhaVhGNk6v + sp6JS7f14BuwFY8Mwm1X4UJXRZx3p + sp6JS7f14BuwFY8Mwm7da9q5vfq7J + sp6JS7f14BuwFY8MwmPLqfBPrHw5H + sp6JS7f14BuwFY8MwmbJpxvVjEwm2 + sp6JS7f14BuwFY8MwoAVeA7ka37cD + sp6JS7f14BuwFY8MwoTFFTAwFKmVM + sp6JS7f14BuwFY8MwoYsne51VpDE3 + sp6JS7f14BuwFY8MwohLVnU1VTk5h + +#endif // 0 diff --git a/src/test/app/NFToken_test.cpp b/src/test/app/NFToken_test.cpp index 40dfe2fe3..2fb27f8a3 100644 --- a/src/test/app/NFToken_test.cpp +++ b/src/test/app/NFToken_test.cpp @@ -96,7 +96,10 @@ class NFToken_test : public beast::unit_test::suite { // If the NFT amendment is not enabled, you should not be able // to create or burn NFTs. - Env env{*this, features - featureNonFungibleTokensV1}; + Env env{ + *this, + features - featureNonFungibleTokensV1 - + featureNonFungibleTokensV1_1}; Account const& master = env.master; BEAST_EXPECT(ownerCount(env, master) == 0); @@ -482,15 +485,14 @@ class NFToken_test : public beast::unit_test::suite if (replacement->getFieldU32(sfMintedNFTokens) != 1) return false; // Unexpected test conditions. - // Now replace the sfMintedNFTokens with its maximum value. - (*replacement)[sfMintedNFTokens] = - std::numeric_limits::max(); + // Now replace sfMintedNFTokens with the largest valid value. + (*replacement)[sfMintedNFTokens] = 0xFFFF'FFFE; view.rawReplace(replacement); return true; }); - // alice should not be able to mint any tokens because she has already - // minted the maximum allowed by a single account. + // See whether alice is at the boundary that causes an error. + env(token::mint(alice, 0u), ter(tesSUCCESS)); env(token::mint(alice, 0u), ter(tecMAX_SEQUENCE_REACHED)); } @@ -1128,6 +1130,12 @@ class NFToken_test : public beast::unit_test::suite //---------------------------------------------------------------------- // preclaim + // The buy offer must be non-zero. + env(token::acceptBuyOffer(buyer, beast::zero), + ter(tecOBJECT_NOT_FOUND)); + env.close(); + BEAST_EXPECT(ownerCount(env, buyer) == 0); + // The buy offer must be present in the ledger. uint256 const missingOfferIndex = keylet::nftoffer(alice, 1).key; env(token::acceptBuyOffer(buyer, missingOfferIndex), @@ -1140,6 +1148,12 @@ class NFToken_test : public beast::unit_test::suite env.close(); BEAST_EXPECT(ownerCount(env, buyer) == 0); + // The sell offer must be non-zero. + env(token::acceptSellOffer(buyer, beast::zero), + ter(tecOBJECT_NOT_FOUND)); + env.close(); + BEAST_EXPECT(ownerCount(env, buyer) == 0); + // The sell offer must be present in the ledger. env(token::acceptSellOffer(buyer, missingOfferIndex), ter(tecOBJECT_NOT_FOUND)); @@ -1572,7 +1586,6 @@ class NFToken_test : public beast::unit_test::suite using namespace test::jtx; - Env env{*this, features}; Account const alice{"alice"}; Account const becky{"becky"}; Account const cheri{"cheri"}; @@ -1581,155 +1594,179 @@ class NFToken_test : public beast::unit_test::suite IOU const gwCAD(gw["CAD"]); IOU const gwEUR(gw["EUR"]); - env.fund(XRP(1000), alice, becky, cheri, gw); - env.close(); - - // Set trust lines so becky and cheri can use gw's currency. - env(trust(becky, gwAUD(1000))); - env(trust(cheri, gwAUD(1000))); - env(trust(becky, gwCAD(1000))); - env(trust(cheri, gwCAD(1000))); - env(trust(becky, gwEUR(1000))); - env(trust(cheri, gwEUR(1000))); - env.close(); - env(pay(gw, becky, gwAUD(500))); - env(pay(gw, becky, gwCAD(500))); - env(pay(gw, becky, gwEUR(500))); - env(pay(gw, cheri, gwAUD(500))); - env(pay(gw, cheri, gwCAD(500))); - env.close(); - - // An nft without flagCreateTrustLines but with a non-zero transfer - // fee will not allow creating offers that use IOUs for payment. - for (std::uint32_t xferFee : {0, 1}) + // The behavior of this test changes dramatically based on the + // presence (or absence) of the fixRemoveNFTokenAutoTrustLine + // amendment. So we test both cases here. + for (auto const& tweakedFeatures : + {features - fixRemoveNFTokenAutoTrustLine, + features | fixRemoveNFTokenAutoTrustLine}) { - uint256 const nftNoAutoTrustID{ - token::getNextID(env, alice, 0u, tfTransferable, xferFee)}; - env(token::mint(alice, 0u), - token::xferFee(xferFee), - txflags(tfTransferable)); + Env env{*this, tweakedFeatures}; + env.fund(XRP(1000), alice, becky, cheri, gw); env.close(); - // becky buys the nft for 1 drop. - uint256 const beckyBuyOfferIndex = - keylet::nftoffer(becky, env.seq(becky)).key; - env(token::createOffer(becky, nftNoAutoTrustID, drops(1)), - token::owner(alice)); + // Set trust lines so becky and cheri can use gw's currency. + env(trust(becky, gwAUD(1000))); + env(trust(cheri, gwAUD(1000))); + env(trust(becky, gwCAD(1000))); + env(trust(cheri, gwCAD(1000))); + env(trust(becky, gwEUR(1000))); + env(trust(cheri, gwEUR(1000))); env.close(); - env(token::acceptBuyOffer(alice, beckyBuyOfferIndex)); + env(pay(gw, becky, gwAUD(500))); + env(pay(gw, becky, gwCAD(500))); + env(pay(gw, becky, gwEUR(500))); + env(pay(gw, cheri, gwAUD(500))); + env(pay(gw, cheri, gwCAD(500))); env.close(); - // becky attempts to sell the nft for AUD. - TER const createOfferTER = - xferFee ? TER(tecNO_LINE) : TER(tesSUCCESS); - uint256 const beckyOfferIndex = - keylet::nftoffer(becky, env.seq(becky)).key; - env(token::createOffer(becky, nftNoAutoTrustID, gwAUD(100)), - txflags(tfSellNFToken), - ter(createOfferTER)); - env.close(); + // An nft without flagCreateTrustLines but with a non-zero transfer + // fee will not allow creating offers that use IOUs for payment. + for (std::uint32_t xferFee : {0, 1}) + { + uint256 const nftNoAutoTrustID{ + token::getNextID(env, alice, 0u, tfTransferable, xferFee)}; + env(token::mint(alice, 0u), + token::xferFee(xferFee), + txflags(tfTransferable)); + env.close(); - // cheri offers to buy the nft for CAD. - uint256 const cheriOfferIndex = - keylet::nftoffer(cheri, env.seq(cheri)).key; - env(token::createOffer(cheri, nftNoAutoTrustID, gwCAD(100)), - token::owner(becky), - ter(createOfferTER)); - env.close(); + // becky buys the nft for 1 drop. + uint256 const beckyBuyOfferIndex = + keylet::nftoffer(becky, env.seq(becky)).key; + env(token::createOffer(becky, nftNoAutoTrustID, drops(1)), + token::owner(alice)); + env.close(); + env(token::acceptBuyOffer(alice, beckyBuyOfferIndex)); + env.close(); - // To keep things tidy, cancel the offers. - env(token::cancelOffer(becky, {beckyOfferIndex})); - env(token::cancelOffer(cheri, {cheriOfferIndex})); - env.close(); - } - // An nft with flagCreateTrustLines but with a non-zero transfer - // fee allows transfers using IOUs for payment. - { - std::uint16_t transferFee = 10000; // 10% + // becky attempts to sell the nft for AUD. + TER const createOfferTER = + xferFee ? TER(tecNO_LINE) : TER(tesSUCCESS); + uint256 const beckyOfferIndex = + keylet::nftoffer(becky, env.seq(becky)).key; + env(token::createOffer(becky, nftNoAutoTrustID, gwAUD(100)), + txflags(tfSellNFToken), + ter(createOfferTER)); + env.close(); - uint256 const nftAutoTrustID{token::getNextID( - env, alice, 0u, tfTransferable | tfTrustLine, transferFee)}; - env(token::mint(alice, 0u), - token::xferFee(transferFee), - txflags(tfTransferable | tfTrustLine)); - env.close(); + // cheri offers to buy the nft for CAD. + uint256 const cheriOfferIndex = + keylet::nftoffer(cheri, env.seq(cheri)).key; + env(token::createOffer(cheri, nftNoAutoTrustID, gwCAD(100)), + token::owner(becky), + ter(createOfferTER)); + env.close(); - // becky buys the nft for 1 drop. - uint256 const beckyBuyOfferIndex = - keylet::nftoffer(becky, env.seq(becky)).key; - env(token::createOffer(becky, nftAutoTrustID, drops(1)), - token::owner(alice)); - env.close(); - env(token::acceptBuyOffer(alice, beckyBuyOfferIndex)); - env.close(); + // To keep things tidy, cancel the offers. + env(token::cancelOffer(becky, {beckyOfferIndex})); + env(token::cancelOffer(cheri, {cheriOfferIndex})); + env.close(); + } + // An nft with flagCreateTrustLines but with a non-zero transfer + // fee allows transfers using IOUs for payment. + { + std::uint16_t transferFee = 10000; // 10% - // becky sells the nft for AUD. - uint256 const beckySellOfferIndex = - keylet::nftoffer(becky, env.seq(becky)).key; - env(token::createOffer(becky, nftAutoTrustID, gwAUD(100)), - txflags(tfSellNFToken)); - env.close(); - env(token::acceptSellOffer(cheri, beckySellOfferIndex)); - env.close(); + uint256 const nftAutoTrustID{token::getNextID( + env, alice, 0u, tfTransferable | tfTrustLine, transferFee)}; - // alice should now have a trust line for gwAUD. - BEAST_EXPECT(env.balance(alice, gwAUD) == gwAUD(10)); + // If the fixRemoveNFTokenAutoTrustLine amendment is active + // then this transaction fails. + { + TER const mintTER = + tweakedFeatures[fixRemoveNFTokenAutoTrustLine] + ? static_cast(temINVALID_FLAG) + : static_cast(tesSUCCESS); - // becky buys the nft back for CAD. - uint256 const beckyBuyBackOfferIndex = - keylet::nftoffer(becky, env.seq(becky)).key; - env(token::createOffer(becky, nftAutoTrustID, gwCAD(50)), - token::owner(cheri)); - env.close(); - env(token::acceptBuyOffer(cheri, beckyBuyBackOfferIndex)); - env.close(); + env(token::mint(alice, 0u), + token::xferFee(transferFee), + txflags(tfTransferable | tfTrustLine), + ter(mintTER)); + env.close(); - // alice should now have a trust line for gwAUD and gwCAD. - BEAST_EXPECT(env.balance(alice, gwAUD) == gwAUD(10)); - BEAST_EXPECT(env.balance(alice, gwCAD) == gwCAD(5)); - } - // Now that alice has trust lines already established, an nft without - // flagCreateTrustLines will work for preestablished trust lines. - { - std::uint16_t transferFee = 5000; // 5% - uint256 const nftNoAutoTrustID{ - token::getNextID(env, alice, 0u, tfTransferable, transferFee)}; - env(token::mint(alice, 0u), - token::xferFee(transferFee), - txflags(tfTransferable)); - env.close(); + // If fixRemoveNFTokenAutoTrustLine is active the rest + // of this test falls on its face. + if (tweakedFeatures[fixRemoveNFTokenAutoTrustLine]) + break; + } + // becky buys the nft for 1 drop. + uint256 const beckyBuyOfferIndex = + keylet::nftoffer(becky, env.seq(becky)).key; + env(token::createOffer(becky, nftAutoTrustID, drops(1)), + token::owner(alice)); + env.close(); + env(token::acceptBuyOffer(alice, beckyBuyOfferIndex)); + env.close(); - // alice sells the nft using AUD. - uint256 const aliceSellOfferIndex = - keylet::nftoffer(alice, env.seq(alice)).key; - env(token::createOffer(alice, nftNoAutoTrustID, gwAUD(200)), - txflags(tfSellNFToken)); - env.close(); - env(token::acceptSellOffer(cheri, aliceSellOfferIndex)); - env.close(); + // becky sells the nft for AUD. + uint256 const beckySellOfferIndex = + keylet::nftoffer(becky, env.seq(becky)).key; + env(token::createOffer(becky, nftAutoTrustID, gwAUD(100)), + txflags(tfSellNFToken)); + env.close(); + env(token::acceptSellOffer(cheri, beckySellOfferIndex)); + env.close(); - // alice should now have AUD(210): - // o 200 for this sale and - // o 10 for the previous sale's fee. - BEAST_EXPECT(env.balance(alice, gwAUD) == gwAUD(210)); + // alice should now have a trust line for gwAUD. + BEAST_EXPECT(env.balance(alice, gwAUD) == gwAUD(10)); - // cheri can't sell the NFT for EUR, but can for CAD. - env(token::createOffer(cheri, nftNoAutoTrustID, gwEUR(50)), - txflags(tfSellNFToken), - ter(tecNO_LINE)); - env.close(); - uint256 const cheriSellOfferIndex = - keylet::nftoffer(cheri, env.seq(cheri)).key; - env(token::createOffer(cheri, nftNoAutoTrustID, gwCAD(100)), - txflags(tfSellNFToken)); - env.close(); - env(token::acceptSellOffer(becky, cheriSellOfferIndex)); - env.close(); + // becky buys the nft back for CAD. + uint256 const beckyBuyBackOfferIndex = + keylet::nftoffer(becky, env.seq(becky)).key; + env(token::createOffer(becky, nftAutoTrustID, gwCAD(50)), + token::owner(cheri)); + env.close(); + env(token::acceptBuyOffer(cheri, beckyBuyBackOfferIndex)); + env.close(); - // alice should now have CAD(10): - // o 5 from this sale's fee and - // o 5 for the previous sale's fee. - BEAST_EXPECT(env.balance(alice, gwCAD) == gwCAD(10)); + // alice should now have a trust line for gwAUD and gwCAD. + BEAST_EXPECT(env.balance(alice, gwAUD) == gwAUD(10)); + BEAST_EXPECT(env.balance(alice, gwCAD) == gwCAD(5)); + } + // Now that alice has trust lines preestablished, an nft without + // flagCreateTrustLines will work for preestablished trust lines. + { + std::uint16_t transferFee = 5000; // 5% + uint256 const nftNoAutoTrustID{token::getNextID( + env, alice, 0u, tfTransferable, transferFee)}; + env(token::mint(alice, 0u), + token::xferFee(transferFee), + txflags(tfTransferable)); + env.close(); + + // alice sells the nft using AUD. + uint256 const aliceSellOfferIndex = + keylet::nftoffer(alice, env.seq(alice)).key; + env(token::createOffer(alice, nftNoAutoTrustID, gwAUD(200)), + txflags(tfSellNFToken)); + env.close(); + env(token::acceptSellOffer(cheri, aliceSellOfferIndex)); + env.close(); + + // alice should now have AUD(210): + // o 200 for this sale and + // o 10 for the previous sale's fee. + BEAST_EXPECT(env.balance(alice, gwAUD) == gwAUD(210)); + + // cheri can't sell the NFT for EUR, but can for CAD. + env(token::createOffer(cheri, nftNoAutoTrustID, gwEUR(50)), + txflags(tfSellNFToken), + ter(tecNO_LINE)); + env.close(); + uint256 const cheriSellOfferIndex = + keylet::nftoffer(cheri, env.seq(cheri)).key; + env(token::createOffer(cheri, nftNoAutoTrustID, gwCAD(100)), + txflags(tfSellNFToken)); + env.close(); + env(token::acceptSellOffer(becky, cheriSellOfferIndex)); + env.close(); + + // alice should now have CAD(10): + // o 5 from this sale's fee and + // o 5 for the previous sale's fee. + BEAST_EXPECT(env.balance(alice, gwCAD) == gwCAD(10)); + } } } @@ -2667,7 +2704,7 @@ class NFToken_test : public beast::unit_test::suite env.close(); // Test how adding a Destination field to an offer affects permissions - // for cancelling offers. + // for canceling offers. { uint256 const offerMinterToIssuer = keylet::nftoffer(minter, env.seq(minter)).key; @@ -2681,14 +2718,20 @@ class NFToken_test : public beast::unit_test::suite token::destination(buyer), txflags(tfSellNFToken)); - // buy offers cannot contain a Destination, so this attempt fails. + uint256 const offerIssuerToMinter = + keylet::nftoffer(issuer, env.seq(issuer)).key; env(token::createOffer(issuer, nftokenID, drops(1)), token::owner(minter), - token::destination(minter), - ter(temMALFORMED)); + token::destination(minter)); + + uint256 const offerIssuerToBuyer = + keylet::nftoffer(issuer, env.seq(issuer)).key; + env(token::createOffer(issuer, nftokenID, drops(1)), + token::owner(minter), + token::destination(buyer)); env.close(); - BEAST_EXPECT(ownerCount(env, issuer) == 0); + BEAST_EXPECT(ownerCount(env, issuer) == 2); BEAST_EXPECT(ownerCount(env, minter) == 3); BEAST_EXPECT(ownerCount(env, buyer) == 0); @@ -2703,8 +2746,12 @@ class NFToken_test : public beast::unit_test::suite ter(tecNO_PERMISSION)); env(token::cancelOffer(buyer, {offerMinterToIssuer}), ter(tecNO_PERMISSION)); + env(token::cancelOffer(buyer, {offerIssuerToMinter}), + ter(tecNO_PERMISSION)); + env(token::cancelOffer(minter, {offerIssuerToBuyer}), + ter(tecNO_PERMISSION)); env.close(); - BEAST_EXPECT(ownerCount(env, issuer) == 0); + BEAST_EXPECT(ownerCount(env, issuer) == 2); BEAST_EXPECT(ownerCount(env, minter) == 3); BEAST_EXPECT(ownerCount(env, buyer) == 0); @@ -2712,6 +2759,8 @@ class NFToken_test : public beast::unit_test::suite // cancel the offers. env(token::cancelOffer(buyer, {offerMinterToBuyer})); env(token::cancelOffer(minter, {offerMinterToIssuer})); + env(token::cancelOffer(buyer, {offerIssuerToBuyer})); + env(token::cancelOffer(issuer, {offerIssuerToMinter})); env.close(); BEAST_EXPECT(ownerCount(env, issuer) == 0); BEAST_EXPECT(ownerCount(env, minter) == 1); @@ -2721,7 +2770,7 @@ class NFToken_test : public beast::unit_test::suite // Test how adding a Destination field to a sell offer affects // accepting that offer. { - uint256 const offerMinterToBuyer = + uint256 const offerMinterSellsToBuyer = keylet::nftoffer(minter, env.seq(minter)).key; env(token::createOffer(minter, nftokenID, drops(1)), token::destination(buyer), @@ -2733,7 +2782,7 @@ class NFToken_test : public beast::unit_test::suite // issuer cannot accept a sell offer where they are not the // destination. - env(token::acceptSellOffer(issuer, offerMinterToBuyer), + env(token::acceptSellOffer(issuer, offerMinterSellsToBuyer), ter(tecNO_PERMISSION)); env.close(); BEAST_EXPECT(ownerCount(env, issuer) == 0); @@ -2741,36 +2790,61 @@ class NFToken_test : public beast::unit_test::suite BEAST_EXPECT(ownerCount(env, buyer) == 0); // However buyer can accept the sell offer. - env(token::acceptSellOffer(buyer, offerMinterToBuyer)); + env(token::acceptSellOffer(buyer, offerMinterSellsToBuyer)); env.close(); BEAST_EXPECT(ownerCount(env, issuer) == 0); BEAST_EXPECT(ownerCount(env, minter) == 0); BEAST_EXPECT(ownerCount(env, buyer) == 1); } - // You can't add a Destination field to a buy offer. + // Test how adding a Destination field to a buy offer affects + // accepting that offer. { - env(token::createOffer(minter, nftokenID, drops(1)), - token::owner(buyer), - token::destination(buyer), - ter(temMALFORMED)); - env.close(); - BEAST_EXPECT(ownerCount(env, issuer) == 0); - BEAST_EXPECT(ownerCount(env, minter) == 0); - BEAST_EXPECT(ownerCount(env, buyer) == 1); - - // However without the Destination the buy offer works fine. - uint256 const offerMinterToBuyer = + uint256 const offerMinterBuysFromBuyer = keylet::nftoffer(minter, env.seq(minter)).key; env(token::createOffer(minter, nftokenID, drops(1)), - token::owner(buyer)); + token::owner(buyer), + token::destination(buyer)); + env.close(); + BEAST_EXPECT(ownerCount(env, issuer) == 0); + BEAST_EXPECT(ownerCount(env, minter) == 1); + BEAST_EXPECT(ownerCount(env, buyer) == 1); + + // issuer cannot accept a buy offer where they are the + // destination. + env(token::acceptBuyOffer(issuer, offerMinterBuysFromBuyer), + ter(tecNO_PERMISSION)); env.close(); BEAST_EXPECT(ownerCount(env, issuer) == 0); BEAST_EXPECT(ownerCount(env, minter) == 1); BEAST_EXPECT(ownerCount(env, buyer) == 1); // Buyer accepts minter's offer. - env(token::acceptBuyOffer(buyer, offerMinterToBuyer)); + env(token::acceptBuyOffer(buyer, offerMinterBuysFromBuyer)); + env.close(); + BEAST_EXPECT(ownerCount(env, issuer) == 0); + BEAST_EXPECT(ownerCount(env, minter) == 1); + BEAST_EXPECT(ownerCount(env, buyer) == 0); + + // If a destination other than the NFToken owner is set, that + // destination must act as a broker. The NFToken owner may not + // simply accept the offer. + uint256 const offerBuyerBuysFromMinter = + keylet::nftoffer(buyer, env.seq(buyer)).key; + env(token::createOffer(buyer, nftokenID, drops(1)), + token::owner(minter), + token::destination(broker)); + env.close(); + BEAST_EXPECT(ownerCount(env, issuer) == 0); + BEAST_EXPECT(ownerCount(env, minter) == 1); + BEAST_EXPECT(ownerCount(env, buyer) == 1); + + env(token::acceptBuyOffer(minter, offerBuyerBuysFromMinter), + ter(tecNO_PERMISSION)); + env.close(); + + // Clean up the unused offer. + env(token::cancelOffer(buyer, {offerBuyerBuysFromMinter})); env.close(); BEAST_EXPECT(ownerCount(env, issuer) == 0); BEAST_EXPECT(ownerCount(env, minter) == 1); @@ -2857,6 +2931,47 @@ class NFToken_test : public beast::unit_test::suite BEAST_EXPECT(ownerCount(env, issuer) == 1); BEAST_EXPECT(ownerCount(env, minter) == 1); BEAST_EXPECT(ownerCount(env, buyer) == 0); + + // Clean out the unconsumed offer. + env(token::cancelOffer(issuer, {offerIssuerToBuyer})); + env.close(); + BEAST_EXPECT(ownerCount(env, issuer) == 0); + BEAST_EXPECT(ownerCount(env, minter) == 1); + BEAST_EXPECT(ownerCount(env, buyer) == 0); + } + + // Show that if a buy and a sell offer both have the same destination, + // then that destination can broker the offers. + { + uint256 const offerMinterToBroker = + keylet::nftoffer(minter, env.seq(minter)).key; + env(token::createOffer(minter, nftokenID, drops(1)), + token::destination(broker), + txflags(tfSellNFToken)); + + uint256 const offerBuyerToBroker = + keylet::nftoffer(buyer, env.seq(buyer)).key; + env(token::createOffer(buyer, nftokenID, drops(1)), + token::owner(minter), + token::destination(broker)); + + // Cannot broker offers when the sell destination is not the buyer + // or the broker. + env(token::brokerOffers( + issuer, offerBuyerToBroker, offerMinterToBroker), + ter(tecNFTOKEN_BUY_SELL_MISMATCH)); + env.close(); + BEAST_EXPECT(ownerCount(env, issuer) == 0); + BEAST_EXPECT(ownerCount(env, minter) == 2); + BEAST_EXPECT(ownerCount(env, buyer) == 1); + + // Broker is successful if they are the destination of both offers. + env(token::brokerOffers( + broker, offerBuyerToBroker, offerMinterToBroker)); + env.close(); + BEAST_EXPECT(ownerCount(env, issuer) == 0); + BEAST_EXPECT(ownerCount(env, minter) == 0); + BEAST_EXPECT(ownerCount(env, buyer) == 1); } } @@ -4069,6 +4184,87 @@ class NFToken_test : public beast::unit_test::suite } } + void + testNFTokenOfferOwner(FeatureBitset features) + { + // Verify the Owner field of an offer behaves as expected. + testcase("NFToken offer owner"); + + using namespace test::jtx; + + Env env{*this, features}; + + Account const issuer{"issuer"}; + Account const buyer1{"buyer1"}; + Account const buyer2{"buyer2"}; + env.fund(XRP(10000), issuer, buyer1, buyer2); + env.close(); + + // issuer creates an NFT. + uint256 const nftId{token::getNextID(env, issuer, 0u, tfTransferable)}; + env(token::mint(issuer, 0u), txflags(tfTransferable)); + env.close(); + + // Prove that issuer now owns nftId. + BEAST_EXPECT(nftCount(env, issuer) == 1); + BEAST_EXPECT(nftCount(env, buyer1) == 0); + BEAST_EXPECT(nftCount(env, buyer2) == 0); + + // Both buyer1 and buyer2 create buy offers for nftId. + uint256 const buyer1OfferIndex = + keylet::nftoffer(buyer1, env.seq(buyer1)).key; + env(token::createOffer(buyer1, nftId, XRP(100)), token::owner(issuer)); + uint256 const buyer2OfferIndex = + keylet::nftoffer(buyer2, env.seq(buyer2)).key; + env(token::createOffer(buyer2, nftId, XRP(100)), token::owner(issuer)); + env.close(); + + // Lambda that counts the number of buy offers for a given NFT. + auto nftBuyOfferCount = [&env](uint256 const& nftId) -> std::size_t { + // We know that in this case not very many offers will be + // returned, so we skip the marker stuff. + Json::Value params; + params[jss::nft_id] = to_string(nftId); + Json::Value buyOffers = + env.rpc("json", "nft_buy_offers", to_string(params)); + + if (buyOffers.isMember(jss::result) && + buyOffers[jss::result].isMember(jss::offers)) + return buyOffers[jss::result][jss::offers].size(); + + return 0; + }; + + // Show there are two buy offers for nftId. + BEAST_EXPECT(nftBuyOfferCount(nftId) == 2); + + // issuer accepts buyer1's offer. + env(token::acceptBuyOffer(issuer, buyer1OfferIndex)); + env.close(); + + // Prove that buyer1 now owns nftId. + BEAST_EXPECT(nftCount(env, issuer) == 0); + BEAST_EXPECT(nftCount(env, buyer1) == 1); + BEAST_EXPECT(nftCount(env, buyer2) == 0); + + // buyer1's offer was consumed, but buyer2's offer is still in the + // ledger. + BEAST_EXPECT(nftBuyOfferCount(nftId) == 1); + + // buyer1 can now accept buyer2's offer, even though buyer2's + // NFTokenCreateOffer transaction specified the NFT Owner as issuer. + env(token::acceptBuyOffer(buyer1, buyer2OfferIndex)); + env.close(); + + // Prove that buyer2 now owns nftId. + BEAST_EXPECT(nftCount(env, issuer) == 0); + BEAST_EXPECT(nftCount(env, buyer1) == 0); + BEAST_EXPECT(nftCount(env, buyer2) == 1); + + // All of the NFTokenOffers are now consumed. + BEAST_EXPECT(nftBuyOfferCount(nftId) == 0); + } + void testNFTokenWithTickets(FeatureBitset features) { @@ -4248,6 +4444,472 @@ class NFToken_test : public beast::unit_test::suite env.close(); } + void + testNftXxxOffers(FeatureBitset features) + { + testcase("nft_buy_offers and nft_sell_offers"); + + // The default limit on returned NFToken offers is 250, so we need + // to produce more than 250 offers of each kind in order to exercise + // the marker. + + // Fortunately there's nothing in the rules that says an account + // can't hold more than one offer for the same NFT. So we only + // need two accounts to generate the necessary offers. + using namespace test::jtx; + + Env env{*this, features}; + + Account const issuer{"issuer"}; + Account const buyer{"buyer"}; + + // A lot of offers requires a lot for reserve. + env.fund(XRP(1000000), issuer, buyer); + env.close(); + + // Create an NFT that we'll make offers for. + uint256 const nftID{token::getNextID(env, issuer, 0u, tfTransferable)}; + env(token::mint(issuer, 0), txflags(tfTransferable)); + env.close(); + + // A lambda that validates nft_XXX_offers query responses. + auto checkOffers = [this, &env, &nftID]( + char const* request, + int expectCount, + int expectMarkerCount, + int line) { + int markerCount = 0; + Json::Value allOffers(Json::arrayValue); + std::string marker; + + // The do/while collects results until no marker is returned. + do + { + Json::Value nftOffers = [&env, &nftID, &request, &marker]() { + Json::Value params; + params[jss::nft_id] = to_string(nftID); + + if (!marker.empty()) + params[jss::marker] = marker; + return env.rpc("json", request, to_string(params)); + }(); + + // If there are no offers for the NFT we get an error + if (expectCount == 0) + { + if (expect( + nftOffers.isMember(jss::result), + "expected \"result\"", + __FILE__, + line)) + { + if (expect( + nftOffers[jss::result].isMember(jss::error), + "expected \"error\"", + __FILE__, + line)) + { + expect( + nftOffers[jss::result][jss::error].asString() == + "objectNotFound", + "expected \"objectNotFound\"", + __FILE__, + line); + } + } + break; + } + + marker.clear(); + if (expect( + nftOffers.isMember(jss::result), + "expected \"result\"", + __FILE__, + line)) + { + Json::Value& result = nftOffers[jss::result]; + + if (result.isMember(jss::marker)) + { + ++markerCount; + marker = result[jss::marker].asString(); + } + + if (expect( + result.isMember(jss::offers), + "expected \"offers\"", + __FILE__, + line)) + { + Json::Value& someOffers = result[jss::offers]; + for (std::size_t i = 0; i < someOffers.size(); ++i) + allOffers.append(someOffers[i]); + } + } + } while (!marker.empty()); + + // Verify the contents of allOffers makes sense. + expect( + allOffers.size() == expectCount, + "Unexpected returned offer count", + __FILE__, + line); + expect( + markerCount == expectMarkerCount, + "Unexpected marker count", + __FILE__, + line); + std::optional globalFlags; + std::set offerIndexes; + std::set amounts; + for (Json::Value const& offer : allOffers) + { + // The flags on all found offers should be the same. + if (!globalFlags) + globalFlags = offer[jss::flags].asInt(); + + expect( + *globalFlags == offer[jss::flags].asInt(), + "Inconsistent flags returned", + __FILE__, + line); + + // The test conditions should produce unique indexes and + // amounts for all offers. + offerIndexes.insert(offer[jss::nft_offer_index].asString()); + amounts.insert(offer[jss::amount].asString()); + } + + expect( + offerIndexes.size() == expectCount, + "Duplicate indexes returned?", + __FILE__, + line); + expect( + amounts.size() == expectCount, + "Duplicate amounts returned?", + __FILE__, + line); + }; + + // There are no sell offers. + checkOffers("nft_sell_offers", 0, false, __LINE__); + + // A lambda that generates sell offers. + STAmount sellPrice = XRP(0); + auto makeSellOffers = + [&env, &issuer, &nftID, &sellPrice](STAmount const& limit) { + // Save a little test time by not closing too often. + int offerCount = 0; + while (sellPrice < limit) + { + sellPrice += XRP(1); + env(token::createOffer(issuer, nftID, sellPrice), + txflags(tfSellNFToken)); + if (++offerCount % 10 == 0) + env.close(); + } + env.close(); + }; + + // There is one sell offer. + makeSellOffers(XRP(1)); + checkOffers("nft_sell_offers", 1, 0, __LINE__); + + // There are 250 sell offers. + makeSellOffers(XRP(250)); + checkOffers("nft_sell_offers", 250, 0, __LINE__); + + // There are 251 sell offers. + makeSellOffers(XRP(251)); + checkOffers("nft_sell_offers", 251, 1, __LINE__); + + // There are 500 sell offers. + makeSellOffers(XRP(500)); + checkOffers("nft_sell_offers", 500, 1, __LINE__); + + // There are 501 sell offers. + makeSellOffers(XRP(501)); + checkOffers("nft_sell_offers", 501, 2, __LINE__); + + // There are no buy offers. + checkOffers("nft_buy_offers", 0, 0, __LINE__); + + // A lambda that generates buy offers. + STAmount buyPrice = XRP(0); + auto makeBuyOffers = + [&env, &buyer, &issuer, &nftID, &buyPrice](STAmount const& limit) { + // Save a little test time by not closing too often. + int offerCount = 0; + while (buyPrice < limit) + { + buyPrice += XRP(1); + env(token::createOffer(buyer, nftID, buyPrice), + token::owner(issuer)); + if (++offerCount % 10 == 0) + env.close(); + } + env.close(); + }; + + // There is one buy offer; + makeBuyOffers(XRP(1)); + checkOffers("nft_buy_offers", 1, 0, __LINE__); + + // There are 250 buy offers. + makeBuyOffers(XRP(250)); + checkOffers("nft_buy_offers", 250, 0, __LINE__); + + // There are 251 buy offers. + makeBuyOffers(XRP(251)); + checkOffers("nft_buy_offers", 251, 1, __LINE__); + + // There are 500 buy offers. + makeBuyOffers(XRP(500)); + checkOffers("nft_buy_offers", 500, 1, __LINE__); + + // There are 501 buy offers. + makeBuyOffers(XRP(501)); + checkOffers("nft_buy_offers", 501, 2, __LINE__); + } + + void + testFixNFTokenNegOffer(FeatureBitset features) + { + // Exercise changes introduced by fixNFTokenNegOffer. + using namespace test::jtx; + + testcase("fixNFTokenNegOffer"); + + Account const issuer{"issuer"}; + Account const buyer{"buyer"}; + Account const gw{"gw"}; + IOU const gwXAU(gw["XAU"]); + + // Test both with and without fixNFTokenNegOffer + for (auto const& tweakedFeatures : + {features - fixNFTokenNegOffer - featureNonFungibleTokensV1_1, + features | fixNFTokenNegOffer}) + { + // There was a bug in the initial NFT implementation that + // allowed offers to be placed with negative amounts. Verify + // that fixNFTokenNegOffer addresses the problem. + Env env{*this, tweakedFeatures}; + + env.fund(XRP(1000000), issuer, buyer, gw); + env.close(); + + env(trust(issuer, gwXAU(2000))); + env(trust(buyer, gwXAU(2000))); + env.close(); + + env(pay(gw, issuer, gwXAU(1000))); + env(pay(gw, buyer, gwXAU(1000))); + env.close(); + + // Create an NFT that we'll make XRP offers for. + uint256 const nftID0{ + token::getNextID(env, issuer, 0u, tfTransferable)}; + env(token::mint(issuer, 0), txflags(tfTransferable)); + env.close(); + + // Create an NFT that we'll make IOU offers for. + uint256 const nftID1{ + token::getNextID(env, issuer, 1u, tfTransferable)}; + env(token::mint(issuer, 1), txflags(tfTransferable)); + env.close(); + + TER const offerCreateTER = tweakedFeatures[fixNFTokenNegOffer] + ? static_cast(temBAD_AMOUNT) + : static_cast(tesSUCCESS); + + // Make offers with negative amounts for the NFTs + uint256 const sellNegXrpOfferIndex = + keylet::nftoffer(issuer, env.seq(issuer)).key; + env(token::createOffer(issuer, nftID0, XRP(-2)), + txflags(tfSellNFToken), + ter(offerCreateTER)); + env.close(); + + uint256 const sellNegIouOfferIndex = + keylet::nftoffer(issuer, env.seq(issuer)).key; + env(token::createOffer(issuer, nftID1, gwXAU(-2)), + txflags(tfSellNFToken), + ter(offerCreateTER)); + env.close(); + + uint256 const buyNegXrpOfferIndex = + keylet::nftoffer(buyer, env.seq(buyer)).key; + env(token::createOffer(buyer, nftID0, XRP(-1)), + token::owner(issuer), + ter(offerCreateTER)); + env.close(); + + uint256 const buyNegIouOfferIndex = + keylet::nftoffer(buyer, env.seq(buyer)).key; + env(token::createOffer(buyer, nftID1, gwXAU(-1)), + token::owner(issuer), + ter(offerCreateTER)); + env.close(); + + { + // Now try to accept the offers. + // 1. If fixNFTokenNegOffer is NOT enabled get tecINTERNAL. + // 2. If fixNFTokenNegOffer IS enabled get tecOBJECT_NOT_FOUND. + TER const offerAcceptTER = tweakedFeatures[fixNFTokenNegOffer] + ? static_cast(tecOBJECT_NOT_FOUND) + : static_cast(tecINTERNAL); + + // Sell offers. + env(token::acceptSellOffer(buyer, sellNegXrpOfferIndex), + ter(offerAcceptTER)); + env.close(); + env(token::acceptSellOffer(buyer, sellNegIouOfferIndex), + ter(offerAcceptTER)); + env.close(); + + // Buy offers. + env(token::acceptBuyOffer(issuer, buyNegXrpOfferIndex), + ter(offerAcceptTER)); + env.close(); + env(token::acceptBuyOffer(issuer, buyNegIouOfferIndex), + ter(offerAcceptTER)); + env.close(); + } + { + // 1. If fixNFTokenNegOffer is NOT enabled get tecSUCCESS. + // 2. If fixNFTokenNegOffer IS enabled get tecOBJECT_NOT_FOUND. + TER const offerAcceptTER = tweakedFeatures[fixNFTokenNegOffer] + ? static_cast(tecOBJECT_NOT_FOUND) + : static_cast(tesSUCCESS); + + // Brokered offers. + env(token::brokerOffers( + gw, buyNegXrpOfferIndex, sellNegXrpOfferIndex), + ter(offerAcceptTER)); + env.close(); + env(token::brokerOffers( + gw, buyNegIouOfferIndex, sellNegIouOfferIndex), + ter(offerAcceptTER)); + env.close(); + } + } + + // Test what happens if NFTokenOffers are created with negative amounts + // and then fixNFTokenNegOffer goes live. What does an acceptOffer do? + { + Env env{ + *this, + features - fixNFTokenNegOffer - featureNonFungibleTokensV1_1}; + + env.fund(XRP(1000000), issuer, buyer, gw); + env.close(); + + env(trust(issuer, gwXAU(2000))); + env(trust(buyer, gwXAU(2000))); + env.close(); + + env(pay(gw, issuer, gwXAU(1000))); + env(pay(gw, buyer, gwXAU(1000))); + env.close(); + + // Create an NFT that we'll make XRP offers for. + uint256 const nftID0{ + token::getNextID(env, issuer, 0u, tfTransferable)}; + env(token::mint(issuer, 0), txflags(tfTransferable)); + env.close(); + + // Create an NFT that we'll make IOU offers for. + uint256 const nftID1{ + token::getNextID(env, issuer, 1u, tfTransferable)}; + env(token::mint(issuer, 1), txflags(tfTransferable)); + env.close(); + + // Make offers with negative amounts for the NFTs + uint256 const sellNegXrpOfferIndex = + keylet::nftoffer(issuer, env.seq(issuer)).key; + env(token::createOffer(issuer, nftID0, XRP(-2)), + txflags(tfSellNFToken)); + env.close(); + + uint256 const sellNegIouOfferIndex = + keylet::nftoffer(issuer, env.seq(issuer)).key; + env(token::createOffer(issuer, nftID1, gwXAU(-2)), + txflags(tfSellNFToken)); + env.close(); + + uint256 const buyNegXrpOfferIndex = + keylet::nftoffer(buyer, env.seq(buyer)).key; + env(token::createOffer(buyer, nftID0, XRP(-1)), + token::owner(issuer)); + env.close(); + + uint256 const buyNegIouOfferIndex = + keylet::nftoffer(buyer, env.seq(buyer)).key; + env(token::createOffer(buyer, nftID1, gwXAU(-1)), + token::owner(issuer)); + env.close(); + + // Now the amendment passes. + env.enableFeature(fixNFTokenNegOffer); + env.close(); + + // All attempts to accept the offers with negative amounts + // should fail with temBAD_OFFER. + env(token::acceptSellOffer(buyer, sellNegXrpOfferIndex), + ter(temBAD_OFFER)); + env.close(); + env(token::acceptSellOffer(buyer, sellNegIouOfferIndex), + ter(temBAD_OFFER)); + env.close(); + + // Buy offers. + env(token::acceptBuyOffer(issuer, buyNegXrpOfferIndex), + ter(temBAD_OFFER)); + env.close(); + env(token::acceptBuyOffer(issuer, buyNegIouOfferIndex), + ter(temBAD_OFFER)); + env.close(); + + // Brokered offers. + env(token::brokerOffers( + gw, buyNegXrpOfferIndex, sellNegXrpOfferIndex), + ter(temBAD_OFFER)); + env.close(); + env(token::brokerOffers( + gw, buyNegIouOfferIndex, sellNegIouOfferIndex), + ter(temBAD_OFFER)); + env.close(); + } + + // Test buy offers with a destination with and without + // fixNFTokenNegOffer. + for (auto const& tweakedFeatures : + {features - fixNFTokenNegOffer - featureNonFungibleTokensV1_1, + features | fixNFTokenNegOffer}) + { + Env env{*this, tweakedFeatures}; + + env.fund(XRP(1000000), issuer, buyer); + + // Create an NFT that we'll make offers for. + uint256 const nftID{ + token::getNextID(env, issuer, 0u, tfTransferable)}; + env(token::mint(issuer, 0), txflags(tfTransferable)); + env.close(); + + TER const offerCreateTER = tweakedFeatures[fixNFTokenNegOffer] + ? static_cast(tesSUCCESS) + : static_cast(temMALFORMED); + + env(token::createOffer(buyer, nftID, drops(1)), + token::owner(issuer), + token::destination(issuer), + ter(offerCreateTER)); + env.close(); + } + } + void testWithFeats(FeatureBitset features) { @@ -4271,8 +4933,11 @@ class NFToken_test : public beast::unit_test::suite testCancelOffers(features); testCancelTooManyOffers(features); testBrokeredAccept(features); + testNFTokenOfferOwner(features); testNFTokenWithTickets(features); testNFTokenDeleteAccount(features); + testNftXxxOffers(features); + testFixNFTokenNegOffer(features); } public: @@ -4280,8 +4945,11 @@ public: run() override { using namespace test::jtx; - auto const sa = supported_amendments(); - testWithFeats(sa); + FeatureBitset const all{supported_amendments()}; + FeatureBitset const fixNFTDir{fixNFTokenDirV1}; + + testWithFeats(all - fixNFTDir); + testWithFeats(all); } }; diff --git a/src/test/app/Offer_test.cpp b/src/test/app/Offer_test.cpp index 200e1c4aa..9f6e165bc 100644 --- a/src/test/app/Offer_test.cpp +++ b/src/test/app/Offer_test.cpp @@ -950,9 +950,14 @@ public: env(pay(gw, alice, USD(1000)), ter(tesSUCCESS)); // No cross: - env(offer(alice, XRP(1000), USD(1000)), - txflags(tfImmediateOrCancel), - ter(tesSUCCESS)); + { + TER const expectedCode = features[featureImmediateOfferKilled] + ? static_cast(tecKILLED) + : static_cast(tesSUCCESS); + env(offer(alice, XRP(1000), USD(1000)), + txflags(tfImmediateOrCancel), + ter(expectedCode)); + } env.require( balance(alice, startBalance - f - f), @@ -5165,11 +5170,12 @@ public: FeatureBitset const flowCross{featureFlowCross}; FeatureBitset const takerDryOffer{fixTakerDryOfferRemoval}; FeatureBitset const rmSmallIncreasedQOffers{fixRmSmallIncreasedQOffers}; + FeatureBitset const immediateOfferKilled{featureImmediateOfferKilled}; - testAll(all - takerDryOffer); - testAll(all - flowCross - takerDryOffer); - testAll(all - flowCross); - testAll(all - rmSmallIncreasedQOffers); + testAll(all - takerDryOffer - immediateOfferKilled); + testAll(all - flowCross - takerDryOffer - immediateOfferKilled); + testAll(all - flowCross - immediateOfferKilled); + testAll(all - rmSmallIncreasedQOffers - immediateOfferKilled); testAll(all); testFalseAssert(); } @@ -5184,11 +5190,12 @@ class Offer_manual_test : public Offer_test FeatureBitset const all{supported_amendments()}; FeatureBitset const flowCross{featureFlowCross}; FeatureBitset const f1513{fix1513}; + FeatureBitset const immediateOfferKilled{featureImmediateOfferKilled}; FeatureBitset const takerDryOffer{fixTakerDryOfferRemoval}; - testAll(all - flowCross - f1513); - testAll(all - flowCross); - testAll(all - f1513); + testAll(all - flowCross - f1513 - immediateOfferKilled); + testAll(all - flowCross - immediateOfferKilled); + testAll(all - immediateOfferKilled); testAll(all); testAll(all - flowCross - takerDryOffer); diff --git a/src/test/app/Path_test.cpp b/src/test/app/Path_test.cpp index b83b0d00d..ef290393d 100644 --- a/src/test/app/Path_test.cpp +++ b/src/test/app/Path_test.cpp @@ -203,7 +203,7 @@ public: wait_for(std::chrono::duration const& rel_time) { std::unique_lock lk(mutex_); - auto b = cv_.wait_for(lk, rel_time, [=] { return signaled_; }); + auto b = cv_.wait_for(lk, rel_time, [this] { return signaled_; }); signaled_ = false; return b; } @@ -1378,6 +1378,69 @@ public: } } + void + noripple_combinations() + { + using namespace jtx; + // This test will create trust lines with various values of the noRipple + // flag. alice <-> george <-> bob george will sort of act like a + // gateway, but use a different name to avoid the usual assumptions + // about gateways. + auto const alice = Account("alice"); + auto const bob = Account("bob"); + auto const george = Account("george"); + auto const USD = george["USD"]; + auto test = [&](std::string casename, + bool aliceRipple, + bool bobRipple, + bool expectPath) { + testcase(casename); + + Env env = pathTestEnv(); + env.fund(XRP(10000), noripple(alice, bob, george)); + env.close(); + // Set the same flags at both ends of the trustline, even though + // only george's matter. + env(trust( + alice, + USD(100), + aliceRipple ? tfClearNoRipple : tfSetNoRipple)); + env(trust( + george, + alice["USD"](100), + aliceRipple ? tfClearNoRipple : tfSetNoRipple)); + env(trust( + bob, USD(100), bobRipple ? tfClearNoRipple : tfSetNoRipple)); + env(trust( + george, + bob["USD"](100), + bobRipple ? tfClearNoRipple : tfSetNoRipple)); + env.close(); + env(pay(george, alice, USD(70))); + env.close(); + + auto [st, sa, da] = + find_paths(env, "alice", "bob", Account("bob")["USD"](5)); + BEAST_EXPECT(equal(da, bob["USD"](5))); + + if (expectPath) + { + BEAST_EXPECT(st.size() == 1); + BEAST_EXPECT(same(st, stpath("george"))); + BEAST_EXPECT(equal(sa, alice["USD"](5))); + } + else + { + BEAST_EXPECT(st.size() == 0); + BEAST_EXPECT(equal(sa, XRP(0))); + } + }; + test("ripple -> ripple", true, true, true); + test("ripple -> no ripple", true, false, true); + test("no ripple -> ripple", false, true, true); + test("no ripple -> no ripple", false, false, false); + } + void run() override { @@ -1401,6 +1464,7 @@ public: trust_auto_clear_trust_auto_clear(); xrp_to_xrp(); receive_max(); + noripple_combinations(); // The following path_find_NN tests are data driven tests // that were originally implemented in js/coffee and migrated diff --git a/src/test/app/SHAMapStore_test.cpp b/src/test/app/SHAMapStore_test.cpp index fd21983a4..010c83a42 100644 --- a/src/test/app/SHAMapStore_test.cpp +++ b/src/test/app/SHAMapStore_test.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include @@ -65,7 +65,7 @@ class SHAMapStore_test : public beast::unit_test::suite auto const seq = json[jss::result][jss::ledger_index].asUInt(); std::optional oinfo = - env.app().getRelationalDBInterface().getLedgerInfoByIndex(seq); + env.app().getRelationalDatabase().getLedgerInfoByIndex(seq); if (!oinfo) return false; const LedgerInfo& info = oinfo.value(); @@ -120,8 +120,7 @@ class SHAMapStore_test : public beast::unit_test::suite ledgerCheck(jtx::Env& env, int const rows, int const first) { const auto [actualRows, actualFirst, actualLast] = - dynamic_cast( - &env.app().getRelationalDBInterface()) + dynamic_cast(&env.app().getRelationalDatabase()) ->getLedgerCountMinMax(); BEAST_EXPECT(actualRows == rows); @@ -133,8 +132,7 @@ class SHAMapStore_test : public beast::unit_test::suite transactionCheck(jtx::Env& env, int const rows) { BEAST_EXPECT( - dynamic_cast( - &env.app().getRelationalDBInterface()) + dynamic_cast(&env.app().getRelationalDatabase()) ->getTransactionCount() == rows); } @@ -142,8 +140,7 @@ class SHAMapStore_test : public beast::unit_test::suite accountTransactionCheck(jtx::Env& env, int const rows) { BEAST_EXPECT( - dynamic_cast( - &env.app().getRelationalDBInterface()) + dynamic_cast(&env.app().getRelationalDatabase()) ->getAccountTransactionCount() == rows); } diff --git a/src/test/app/ValidatorKeys_test.cpp b/src/test/app/ValidatorKeys_test.cpp index 18061c7e4..3943fd858 100644 --- a/src/test/app/ValidatorKeys_test.cpp +++ b/src/test/app/ValidatorKeys_test.cpp @@ -23,8 +23,9 @@ #include #include #include +#include + #include -#include namespace ripple { namespace test { @@ -75,7 +76,14 @@ public: void run() override { - SuiteJournal journal("ValidatorKeys_test", *this); + // We're only using Env for its Journal. That Journal gives better + // coverage in unit tests. + test::jtx::Env env{ + *this, + test::jtx::envconfig(), + nullptr, + beast::severities::kDisabled}; + beast::Journal journal{env.app().journal("ValidatorKeys_test")}; // Keys/ID when using [validation_seed] SecretKey const seedSecretKey = diff --git a/src/test/app/ValidatorList_test.cpp b/src/test/app/ValidatorList_test.cpp index 860b8fc17..fead5563f 100644 --- a/src/test/app/ValidatorList_test.cpp +++ b/src/test/app/ValidatorList_test.cpp @@ -29,9 +29,10 @@ #include #include #include -#include #include +#include + namespace ripple { namespace test { @@ -217,7 +218,8 @@ private: { testcase("Config Load"); - jtx::Env env(*this); + jtx::Env env( + *this, jtx::envconfig(), nullptr, beast::severities::kDisabled); auto& app = env.app(); PublicKey emptyLocalKey; std::vector const emptyCfgKeys; diff --git a/src/test/app/ValidatorSite_test.cpp b/src/test/app/ValidatorSite_test.cpp index db9f5c977..8ec86fead 100644 --- a/src/test/app/ValidatorSite_test.cpp +++ b/src/test/app/ValidatorSite_test.cpp @@ -69,7 +69,7 @@ private: using namespace jtx; - Env env(*this); + Env env(*this, envconfig(), nullptr, beast::severities::kDisabled); auto trustedSites = std::make_unique(env.app(), env.journal); @@ -282,9 +282,6 @@ private: if (u.cfg.failFetch) { using namespace std::chrono; - log << " -- Msg: " - << myStatus[jss::last_refresh_message].asString() - << std::endl; std::stringstream nextRefreshStr{ myStatus[jss::next_refresh_time].asString()}; system_clock::time_point nextRefresh; @@ -357,9 +354,6 @@ private: sink.messages().str().find(u.expectMsg) != std::string::npos, sink.messages().str()); - log << " -- Msg: " - << myStatus[jss::last_refresh_message].asString() - << std::endl; } } } diff --git a/src/test/basics/PerfLog_test.cpp b/src/test/basics/PerfLog_test.cpp index a79dded90..79944e0ed 100644 --- a/src/test/basics/PerfLog_test.cpp +++ b/src/test/basics/PerfLog_test.cpp @@ -24,12 +24,13 @@ #include #include #include +#include + #include #include #include #include #include -#include #include //------------------------------------------------------------------------------ @@ -44,7 +45,11 @@ class PerfLog_test : public beast::unit_test::suite // We're only using Env for its Journal. That Journal gives better // coverage in unit tests. - test::jtx::Env env_{*this}; + test::jtx::Env env_{ + *this, + test::jtx::envconfig(), + nullptr, + beast::severities::kDisabled}; beast::Journal j_{env_.app().journal("PerfLog_test")}; struct Fixture diff --git a/src/test/basics/StringUtilities_test.cpp b/src/test/basics/StringUtilities_test.cpp index fc6d54c63..6146a3dcd 100644 --- a/src/test/basics/StringUtilities_test.cpp +++ b/src/test/basics/StringUtilities_test.cpp @@ -289,6 +289,13 @@ public: BEAST_EXPECT(!parseUrl(pUrl, "nonsense")); BEAST_EXPECT(!parseUrl(pUrl, "://")); BEAST_EXPECT(!parseUrl(pUrl, ":///")); + BEAST_EXPECT( + !parseUrl(pUrl, "scheme://user:pass@domain:65536/abc:321")); + BEAST_EXPECT(!parseUrl(pUrl, "UPPER://domain:23498765/")); + BEAST_EXPECT(!parseUrl(pUrl, "UPPER://domain:0/")); + BEAST_EXPECT(!parseUrl(pUrl, "UPPER://domain:+7/")); + BEAST_EXPECT(!parseUrl(pUrl, "UPPER://domain:-7234/")); + BEAST_EXPECT(!parseUrl(pUrl, "UPPER://domain:@#$56!/")); } { diff --git a/src/test/beast/beast_io_latency_probe_test.cpp b/src/test/beast/beast_io_latency_probe_test.cpp index 9ae6a1341..b2bf67b10 100644 --- a/src/test/beast/beast_io_latency_probe_test.cpp +++ b/src/test/beast/beast_io_latency_probe_test.cpp @@ -200,9 +200,6 @@ class io_latency_probe_test : public beast::unit_test::suite, duration_cast(probe_duration).count()) / static_cast(tt.getMean()); #endif - log << "expected_probe_count_min: " << expected_probe_count_min << "\n"; - log << "expected_probe_count_max: " << expected_probe_count_max << "\n"; - test_sampler io_probe{interval, get_io_service()}; io_probe.start(); MyTimer timer{get_io_service(), probe_duration}; diff --git a/src/test/consensus/NegativeUNL_test.cpp b/src/test/consensus/NegativeUNL_test.cpp index 39028afff..fee790281 100644 --- a/src/test/consensus/NegativeUNL_test.cpp +++ b/src/test/consensus/NegativeUNL_test.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include namespace ripple { @@ -1883,114 +1882,6 @@ class NegativeUNLVoteFilterValidations_test : public beast::unit_test::suite } }; -class NegativeUNLgRPC_test : public beast::unit_test::suite -{ - template - std::string - toByteString(T const& data) - { - const char* bytes = reinterpret_cast(data.data()); - return {bytes, data.size()}; - } - - void - testGRPC() - { - testcase("gRPC test"); - - auto gRpcTest = [this]( - std::uint32_t negUnlSize, - bool hasToDisable, - bool hasToReEnable) -> bool { - NetworkHistory history = { - *this, {20, negUnlSize, hasToDisable, hasToReEnable, {}}}; - if (!history.goodHistory) - return false; - - auto const& negUnlObject = - history.lastLedger()->read(keylet::negativeUNL()); - if (!negUnlSize && !hasToDisable && !hasToReEnable && !negUnlObject) - return true; - if (!negUnlObject) - return false; - - org::xrpl::rpc::v1::NegativeUNL to; - ripple::RPC::convert(to, *negUnlObject); - if (!to.has_flags() || - to.flags().value() != negUnlObject->getFlags()) - return false; - - bool goodSize = to.disabled_validators_size() == negUnlSize && - to.has_validator_to_disable() == hasToDisable && - to.has_validator_to_re_enable() == hasToReEnable; - if (!goodSize) - return false; - - if (negUnlSize) - { - if (!negUnlObject->isFieldPresent(sfDisabledValidators)) - return false; - auto const& nUnlData = - negUnlObject->getFieldArray(sfDisabledValidators); - if (nUnlData.size() != negUnlSize) - return false; - int idx = 0; - for (auto const& n : nUnlData) - { - if (!n.isFieldPresent(sfPublicKey) || - !n.isFieldPresent(sfFirstLedgerSequence)) - return false; - - if (!to.disabled_validators(idx).has_ledger_sequence() || - !to.disabled_validators(idx).has_public_key()) - return false; - - if (to.disabled_validators(idx).public_key().value() != - toByteString(n.getFieldVL(sfPublicKey))) - return false; - - if (to.disabled_validators(idx).ledger_sequence().value() != - n.getFieldU32(sfFirstLedgerSequence)) - return false; - - ++idx; - } - } - - if (hasToDisable) - { - if (!negUnlObject->isFieldPresent(sfValidatorToDisable)) - return false; - if (to.validator_to_disable().value() != - toByteString( - negUnlObject->getFieldVL(sfValidatorToDisable))) - return false; - } - - if (hasToReEnable) - { - if (!negUnlObject->isFieldPresent(sfValidatorToReEnable)) - return false; - if (to.validator_to_re_enable().value() != - toByteString( - negUnlObject->getFieldVL(sfValidatorToReEnable))) - return false; - } - - return true; - }; - - BEAST_EXPECT(gRpcTest(0, false, false)); - BEAST_EXPECT(gRpcTest(2, true, true)); - } - - void - run() override - { - testGRPC(); - } -}; - BEAST_DEFINE_TESTSUITE(NegativeUNL, ledger, ripple); BEAST_DEFINE_TESTSUITE(NegativeUNLNoAmendment, ledger, ripple); @@ -2006,7 +1897,6 @@ BEAST_DEFINE_TESTSUITE_PRIO( 1); BEAST_DEFINE_TESTSUITE(NegativeUNLVoteNewValidator, consensus, ripple); BEAST_DEFINE_TESTSUITE(NegativeUNLVoteFilterValidations, consensus, ripple); -BEAST_DEFINE_TESTSUITE(NegativeUNLgRPC, ledger, ripple); /////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////// diff --git a/src/test/core/ClosureCounter_test.cpp b/src/test/core/ClosureCounter_test.cpp index 478816a89..c4199a0b0 100644 --- a/src/test/core/ClosureCounter_test.cpp +++ b/src/test/core/ClosureCounter_test.cpp @@ -19,9 +19,10 @@ #include #include +#include + #include #include -#include #include namespace ripple { @@ -31,9 +32,14 @@ namespace test { class ClosureCounter_test : public beast::unit_test::suite { - // We're only using Env for its Journal. - jtx::Env env{*this}; - beast::Journal j{env.app().journal("ClosureCounter_test")}; + // We're only using Env for its Journal. That Journal gives better + // coverage in unit tests. + test::jtx::Env env_{ + *this, + jtx::envconfig(), + nullptr, + beast::severities::kDisabled}; + beast::Journal j{env_.app().journal("ClosureCounter_test")}; void testConstruction() diff --git a/src/test/core/Config_test.cpp b/src/test/core/Config_test.cpp index 45afaf6cb..da29fafac 100644 --- a/src/test/core/Config_test.cpp +++ b/src/test/core/Config_test.cpp @@ -154,7 +154,7 @@ public: rmDataDir_ = !exists(dataDir_); config_.setup( file_.string(), - /*bQuiet*/ true, + /* bQuiet */ true, /* bSilent */ false, /* bStandalone */ false); } @@ -190,9 +190,6 @@ public: using namespace boost::filesystem; if (rmDataDir_) rmDir(dataDir_); - else - test_.log << "Skipping rm dir: " << dataDir_.string() - << std::endl; } catch (std::exception& e) { diff --git a/src/test/core/Coroutine_test.cpp b/src/test/core/Coroutine_test.cpp index 8937255a7..6d1e5e333 100644 --- a/src/test/core/Coroutine_test.cpp +++ b/src/test/core/Coroutine_test.cpp @@ -44,7 +44,7 @@ public: wait_for(std::chrono::duration const& rel_time) { std::unique_lock lk(mutex_); - auto b = cv_.wait_for(lk, rel_time, [=] { return signaled_; }); + auto b = cv_.wait_for(lk, rel_time, [this] { return signaled_; }); signaled_ = false; return b; } diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index 3a61b853c..6d3008f73 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -538,7 +538,7 @@ struct Peer ConsensusMode const& mode, Json::Value&& consensusJson) { - schedule(delays.ledgerAccept, [=]() { + schedule(delays.ledgerAccept, [=, this]() { const bool proposing = mode == ConsensusMode::proposing; const bool consensusFail = result.state == ConsensusState::MovedOn; diff --git a/src/test/jtx/Env_test.cpp b/src/test/jtx/Env_test.cpp index b1a1a8125..6f09f49ed 100644 --- a/src/test/jtx/Env_test.cpp +++ b/src/test/jtx/Env_test.cpp @@ -888,10 +888,14 @@ public: testExceptionalShutdown() { except([this] { - jtx::Env env{*this, jtx::envconfig([](std::unique_ptr cfg) { - (*cfg).deprecatedClearSection("port_rpc"); - return cfg; - })}; + jtx::Env env{ + *this, + jtx::envconfig([](std::unique_ptr cfg) { + (*cfg).deprecatedClearSection("port_rpc"); + return cfg; + }), + nullptr, + beast::severities::kDisabled}; }); pass(); } diff --git a/src/test/jtx/impl/Env.cpp b/src/test/jtx/impl/Env.cpp index 3445fd1c9..900b9812d 100644 --- a/src/test/jtx/impl/Env.cpp +++ b/src/test/jtx/impl/Env.cpp @@ -83,7 +83,7 @@ Env::AppBundle::AppBundle( std::move(config), std::move(logs), std::move(timeKeeper_)); app = owned.get(); app->logs().threshold(thresh); - if (!app->setup()) + if (!app->setup({})) Throw("Env::AppBundle: setup failed"); timeKeeper->set(app->getLedgerMaster().getClosedLedger()->info().closeTime); app->start(false /*don't start timers*/); diff --git a/src/test/jtx/impl/multisign.cpp b/src/test/jtx/impl/multisign.cpp index 129f30701..1e1f51417 100644 --- a/src/test/jtx/impl/multisign.cpp +++ b/src/test/jtx/impl/multisign.cpp @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include @@ -46,6 +48,8 @@ signers( auto& je = ja[i][sfSignerEntry.getJsonName()]; je[jss::Account] = e.account.human(); je[sfSignerWeight.getJsonName()] = e.weight; + if (e.tag) + je[sfWalletLocator.getJsonName()] = to_string(*e.tag); } return jv; } diff --git a/src/test/jtx/multisign.h b/src/test/jtx/multisign.h index 91a110352..ab9996e41 100644 --- a/src/test/jtx/multisign.h +++ b/src/test/jtx/multisign.h @@ -21,6 +21,7 @@ #define RIPPLE_TEST_JTX_MULTISIGN_H_INCLUDED #include +#include #include #include #include @@ -35,9 +36,13 @@ struct signer { std::uint32_t weight; Account account; + std::optional tag; - signer(Account account_, std::uint32_t weight_ = 1) - : weight(weight_), account(std::move(account_)) + signer( + Account account_, + std::uint32_t weight_ = 1, + std::optional tag_ = std::nullopt) + : weight(weight_), account(std::move(account_)), tag(std::move(tag_)) { } }; diff --git a/src/test/ledger/View_test.cpp b/src/test/ledger/View_test.cpp index 45c8f007e..bbb1eec8f 100644 --- a/src/test/ledger/View_test.cpp +++ b/src/test/ledger/View_test.cpp @@ -130,6 +130,8 @@ class View_test : public beast::unit_test::suite void testLedger() { + testcase("Ledger"); + using namespace jtx; Env env(*this); Config config; @@ -165,6 +167,8 @@ class View_test : public beast::unit_test::suite void testMeta() { + testcase("Meta"); + using namespace jtx; Env env(*this); wipe(env.app().openLedger()); @@ -196,6 +200,8 @@ class View_test : public beast::unit_test::suite void testMetaSucc() { + testcase("Meta succ"); + using namespace jtx; Env env(*this); wipe(env.app().openLedger()); @@ -260,6 +266,8 @@ class View_test : public beast::unit_test::suite void testStacked() { + testcase("Stacked"); + using namespace jtx; Env env(*this); wipe(env.app().openLedger()); @@ -325,6 +333,8 @@ class View_test : public beast::unit_test::suite void testContext() { + testcase("Context"); + using namespace jtx; using namespace std::chrono; { @@ -387,6 +397,8 @@ class View_test : public beast::unit_test::suite void testUpperAndLowerBound() { + testcase("Upper and lower bound"); + using namespace jtx; Env env(*this); Config config; @@ -654,6 +666,8 @@ class View_test : public beast::unit_test::suite void testSles() { + testcase("Sles"); + using namespace jtx; Env env(*this); Config config; @@ -786,6 +800,8 @@ class View_test : public beast::unit_test::suite void testFlags() { + testcase("Flags"); + using namespace jtx; Env env(*this); @@ -949,6 +965,8 @@ class View_test : public beast::unit_test::suite void testTransferRate() { + testcase("Transfer rate"); + using namespace jtx; Env env(*this); @@ -975,12 +993,14 @@ class View_test : public beast::unit_test::suite // construct and manage two different Env instances at the same // time. So we can use two Env instances to produce mutually // incompatible ledgers. + testcase("Are compatible"); + using namespace jtx; auto const alice = Account("alice"); auto const bob = Account("bob"); // The first Env. - Env eA(*this); + Env eA(*this, envconfig(), nullptr, beast::severities::kDisabled); eA.fund(XRP(10000), alice); eA.close(); @@ -990,9 +1010,13 @@ class View_test : public beast::unit_test::suite eA.close(); auto const rdViewA4 = eA.closed(); - // The two Env's can't share the same ports, so modifiy the config + // The two Env's can't share the same ports, so modify the config // of the second Env to use higher port numbers - Env eB{*this, envconfig(port_increment, 3)}; + Env eB{ + *this, + envconfig(port_increment, 3), + nullptr, + beast::severities::kDisabled}; // Make ledgers that are incompatible with the first ledgers. Note // that bob is funded before alice. @@ -1029,6 +1053,8 @@ class View_test : public beast::unit_test::suite void testRegressions() { + testcase("Regressions"); + using namespace jtx; // Create a ledger with 1 item, put a diff --git a/src/test/net/DatabaseDownloader_test.cpp b/src/test/net/DatabaseDownloader_test.cpp index 749000dc5..d4ed2ebce 100644 --- a/src/test/net/DatabaseDownloader_test.cpp +++ b/src/test/net/DatabaseDownloader_test.cpp @@ -19,7 +19,6 @@ #include #include -#include #include #include #include @@ -29,6 +28,8 @@ namespace ripple { namespace test { +#define REPORT_FAILURE(D) reportFailure(D, __FILE__, __LINE__) + class DatabaseDownloader_test : public beast::unit_test::suite { std::shared_ptr @@ -65,13 +66,10 @@ class DatabaseDownloader_test : public beast::unit_test::suite waitComplete() { std::unique_lock lk(m); - using namespace std::chrono_literals; -#if BOOST_OS_WINDOWS - auto constexpr timeout = 4s; -#else - auto constexpr timeout = 2s; -#endif - auto stat = cv.wait_for(lk, timeout, [this] { return called; }); + + auto stat = cv.wait_for( + lk, std::chrono::seconds(10), [this] { return called; }); + called = false; return stat; }; @@ -103,8 +101,29 @@ class DatabaseDownloader_test : public beast::unit_test::suite { return ptr_.get(); } + + DatabaseDownloader const* + operator->() const + { + return ptr_.get(); + } }; + void + reportFailure(Downloader const& dl, char const* file, int line) + { + std::stringstream ss; + ss << "Failed. LOGS:\n" + << dl.sink_.messages().str() + << "\nDownloadCompleter failure." + "\nDatabaseDownloader session active? " + << std::boolalpha << dl->sessionIsActive() + << "\nDatabaseDownloader is stopping? " << std::boolalpha + << dl->isStopping(); + + fail(ss.str(), file, line); + } + void testDownload(bool verify) { @@ -122,7 +141,7 @@ class DatabaseDownloader_test : public beast::unit_test::suite return cfg; })}; - Downloader downloader{env}; + Downloader dl{env}; // create a TrustedPublisherServer as a simple HTTP // server to request from. Use the /textfile endpoint @@ -133,7 +152,7 @@ class DatabaseDownloader_test : public beast::unit_test::suite *this, "downloads", "data", "", false, false}; // initiate the download and wait for the callback // to be invoked - auto stat = downloader->download( + auto stat = dl->download( server->local_endpoint().address().to_string(), std::to_string(server->local_endpoint().port()), "/textfile", @@ -142,12 +161,12 @@ class DatabaseDownloader_test : public beast::unit_test::suite std::function{std::ref(cb)}); if (!BEAST_EXPECT(stat)) { - log << "Failed. LOGS:\n" + downloader.sink_.messages().str(); + REPORT_FAILURE(dl); return; } if (!BEAST_EXPECT(cb.waitComplete())) { - log << "Failed. LOGS:\n" + downloader.sink_.messages().str(); + REPORT_FAILURE(dl); return; } BEAST_EXPECT(cb.dest == data.file()); @@ -187,7 +206,10 @@ class DatabaseDownloader_test : public beast::unit_test::suite datafile.file(), std::function{ std::ref(cb)})); - BEAST_EXPECT(cb.waitComplete()); + if (!BEAST_EXPECT(cb.waitComplete())) + { + REPORT_FAILURE(dl); + } BEAST_EXPECT(!boost::filesystem::exists(datafile.file())); BEAST_EXPECTS( dl.sink_.messages().str().find("async_resolve") != @@ -211,7 +233,10 @@ class DatabaseDownloader_test : public beast::unit_test::suite 11, datafile.file(), std::function{std::ref(cb)})); - BEAST_EXPECT(cb.waitComplete()); + if (!BEAST_EXPECT(cb.waitComplete())) + { + REPORT_FAILURE(dl); + } BEAST_EXPECT(!boost::filesystem::exists(datafile.file())); BEAST_EXPECTS( dl.sink_.messages().str().find("async_connect") != @@ -231,7 +256,10 @@ class DatabaseDownloader_test : public beast::unit_test::suite 11, datafile.file(), std::function{std::ref(cb)})); - BEAST_EXPECT(cb.waitComplete()); + if (!BEAST_EXPECT(cb.waitComplete())) + { + REPORT_FAILURE(dl); + } BEAST_EXPECT(!boost::filesystem::exists(datafile.file())); BEAST_EXPECTS( dl.sink_.messages().str().find("async_handshake") != @@ -251,7 +279,10 @@ class DatabaseDownloader_test : public beast::unit_test::suite 11, datafile.file(), std::function{std::ref(cb)})); - BEAST_EXPECT(cb.waitComplete()); + if (!BEAST_EXPECT(cb.waitComplete())) + { + REPORT_FAILURE(dl); + } BEAST_EXPECT(!boost::filesystem::exists(datafile.file())); BEAST_EXPECTS( dl.sink_.messages().str().find("Insufficient disk space") != @@ -270,6 +301,8 @@ public: } }; +#undef REPORT_FAILURE + BEAST_DEFINE_TESTSUITE(DatabaseDownloader, net, ripple); } // namespace test } // namespace ripple diff --git a/src/test/nodestore/DatabaseShard_test.cpp b/src/test/nodestore/DatabaseShard_test.cpp index 88bcd4977..5c074b149 100644 --- a/src/test/nodestore/DatabaseShard_test.cpp +++ b/src/test/nodestore/DatabaseShard_test.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include @@ -1762,9 +1762,9 @@ class DatabaseShard_test : public TestBase } void - testRelationalDBInterfaceSqlite(std::uint64_t const seedValue) + testSQLiteDatabase(std::uint64_t const seedValue) { - testcase("Relational DB Interface SQLite"); + testcase("SQLite Database"); using namespace test::jtx; @@ -1782,8 +1782,8 @@ class DatabaseShard_test : public TestBase BEAST_EXPECT(shardStore->getShardInfo()->finalized().empty()); BEAST_EXPECT(shardStore->getShardInfo()->incompleteToString().empty()); - auto rdb = dynamic_cast( - &env.app().getRelationalDBInterface()); + auto rdb = + dynamic_cast(&env.app().getRelationalDatabase()); BEAST_EXPECT(rdb); @@ -1796,7 +1796,7 @@ class DatabaseShard_test : public TestBase return; } - // Close these databases to force the RelationalDBInterfaceSqlite + // Close these databases to force the SQLiteDatabase // to use the shard databases and lookup tables. rdb->closeLedgerDB(); rdb->closeTransactionDB(); @@ -1814,7 +1814,7 @@ class DatabaseShard_test : public TestBase for (auto const& ledger : data.ledgers_) { // Compare each test ledger to the data retrieved - // from the RelationalDBInterfaceSqlite class + // from the SQLiteDatabase class if (shardStore->seqToShardIndex(ledger->seq()) < shardStore->earliestShardIndex() || @@ -1829,8 +1829,7 @@ class DatabaseShard_test : public TestBase for (auto const& transaction : ledger->txs) { // Compare each test transaction to the data - // retrieved from the RelationalDBInterfaceSqlite - // class + // retrieved from the SQLiteDatabase class error_code_i error{rpcSUCCESS}; @@ -1885,7 +1884,7 @@ public: testPrepareWithHistoricalPaths(seedValue()); testOpenShardManagement(seedValue()); testShardInfo(seedValue()); - testRelationalDBInterfaceSqlite(seedValue()); + testSQLiteDatabase(seedValue()); } }; diff --git a/src/test/overlay/ProtocolVersion_test.cpp b/src/test/overlay/ProtocolVersion_test.cpp index 81845345d..a5a26fe74 100644 --- a/src/test/overlay/ProtocolVersion_test.cpp +++ b/src/test/overlay/ProtocolVersion_test.cpp @@ -25,22 +25,21 @@ namespace ripple { class ProtocolVersion_test : public beast::unit_test::suite { private: - template - static std::string - join(FwdIt first, FwdIt last, char const* sep = ",") - { - std::string result; - if (first == last) - return result; - result = to_string(*first++); - while (first != last) - result += sep + to_string(*first++); - return result; - } - void check(std::string const& s, std::string const& answer) { + auto join = [](auto first, auto last) { + std::string result; + if (first != last) + { + result = to_string(*first++); + + while (first != last) + result += "," + to_string(*first++); + } + return result; + }; + auto const result = parseProtocolVersions(s); BEAST_EXPECT(join(result.begin(), result.end()) == answer); } @@ -60,20 +59,21 @@ public: // Empty string check("", ""); + + // clang-format off check( - "RTXP/1.1,RTXP/1.2,RTXP/1.3,XRPL/2.1,XRPL/2.0", + "RTXP/1.1,RTXP/1.2,RTXP/1.3,XRPL/2.1,XRPL/2.0,/XRPL/3.0", "XRPL/2.0,XRPL/2.1"); check( - "RTXP/0.9,RTXP/1.01,XRPL/0.3,XRPL/2.01,XRPL/19.04,Oscar/" - "123,NIKB", + "RTXP/0.9,RTXP/1.01,XRPL/0.3,XRPL/2.01,websocket", ""); check( - "XRPL/2.0,RTXP/1.2,XRPL/2.0,XRPL/19.4,XRPL/7.89,XRPL/" - "A.1,XRPL/2.01", + "XRPL/2.0,XRPL/2.0,XRPL/19.4,XRPL/7.89,XRPL/XRPL/3.0,XRPL/2.01", "XRPL/2.0,XRPL/7.89,XRPL/19.4"); check( "XRPL/2.0,XRPL/3.0,XRPL/4,XRPL/,XRPL,OPT XRPL/2.2,XRPL/5.67", "XRPL/2.0,XRPL/3.0,XRPL/5.67"); + // clang-format on } { @@ -81,13 +81,14 @@ public: BEAST_EXPECT(negotiateProtocolVersion("RTXP/1.2") == std::nullopt); BEAST_EXPECT( - negotiateProtocolVersion("RTXP/1.2, XRPL/2.0") == - make_protocol(2, 0)); + negotiateProtocolVersion("RTXP/1.2, XRPL/2.0, XRPL/2.1") == + make_protocol(2, 1)); BEAST_EXPECT( - negotiateProtocolVersion("XRPL/2.0") == make_protocol(2, 0)); + negotiateProtocolVersion("XRPL/2.2") == make_protocol(2, 2)); BEAST_EXPECT( - negotiateProtocolVersion("RTXP/1.2, XRPL/2.0, XRPL/999.999") == - make_protocol(2, 0)); + negotiateProtocolVersion( + "RTXP/1.2, XRPL/2.2, XRPL/2.3, XRPL/999.999") == + make_protocol(2, 2)); BEAST_EXPECT( negotiateProtocolVersion("XRPL/999.999, WebSocket/1.0") == std::nullopt); diff --git a/src/test/overlay/short_read_test.cpp b/src/test/overlay/short_read_test.cpp index dd649bfd1..434b41008 100644 --- a/src/test/overlay/short_read_test.cpp +++ b/src/test/overlay/short_read_test.cpp @@ -195,8 +195,6 @@ private: { acceptor_.listen(); server_.endpoint_ = acceptor_.local_endpoint(); - test_.log << "[server] up on port: " << server_.endpoint_.port() - << std::endl; } void diff --git a/src/test/protocol/KnownFormatToGRPC_test.cpp b/src/test/protocol/KnownFormatToGRPC_test.cpp deleted file mode 100644 index bf49f2e31..000000000 --- a/src/test/protocol/KnownFormatToGRPC_test.cpp +++ /dev/null @@ -1,975 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include - -#include "org/xrpl/rpc/v1/ledger_objects.pb.h" -#include "org/xrpl/rpc/v1/transaction.pb.h" - -#include -#include -#include -#include - -namespace ripple { - -// This test suite uses the google::protobuf::Descriptor class to do runtime -// reflection on our gRPC stuff. At the time of this writing documentation -// for Descriptor could be found here: -// -// https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.descriptor#Descriptor - -class KnownFormatToGRPC_test : public beast::unit_test::suite -{ -private: - static constexpr auto fieldTYPE_UINT32 = - google::protobuf::FieldDescriptor::Type::TYPE_UINT32; - - static constexpr auto fieldTYPE_UINT64 = - google::protobuf::FieldDescriptor::Type::TYPE_UINT64; - - static constexpr auto fieldTYPE_BYTES = - google::protobuf::FieldDescriptor::Type::TYPE_BYTES; - - static constexpr auto fieldTYPE_STRING = - google::protobuf::FieldDescriptor::Type::TYPE_STRING; - - static constexpr auto fieldTYPE_MESSAGE = - google::protobuf::FieldDescriptor::Type::TYPE_MESSAGE; - - // Format names are CamelCase and FieldDescriptor names are snake_case. - // Convert from CamelCase to snake_case. Do not be fooled by consecutive - // capital letters like in NegativeUNL. - static std::string - formatNameToEntryTypeName(std::string const& fmtName) - { - std::string entryName; - entryName.reserve(fmtName.size()); - bool prevUpper = false; - for (std::size_t i = 0; i < fmtName.size(); i++) - { - char const ch = fmtName[i]; - bool const upper = std::isupper(ch); - if (i > 0 && !prevUpper && upper) - entryName.push_back('_'); - - prevUpper = upper; - entryName.push_back(std::tolower(ch)); - } - return entryName; - }; - - // Create a map of (most) all the SFields in an SOTemplate. This map - // can be used to correlate a gRPC Descriptor to its corresponding SField. - template - static std::map - soTemplateToSFields( - SOTemplate const& soTemplate, - [[maybe_unused]] KeyType fmtId) - { - std::map sFields; - for (SOElement const& element : soTemplate) - { - SField const& sField = element.sField(); - - // Fields that gRPC never includes. - // - // o sfLedgerIndex and - // o sfLedgerEntryType are common to all ledger objects, so - // gRPC includes them at a higher level than the ledger - // object itself. - // - // o sfOperationLimit is an optional field in all transactions, - // but no one knows what it was intended for. - using FieldCode_t = - std::remove_const::type; - static const std::set excludedSFields{ - sfLedgerIndex.fieldCode, - sfLedgerEntryType.fieldCode, - sfOperationLimit.fieldCode}; - - if (excludedSFields.count(sField.fieldCode)) - continue; - - // There are certain fields that gRPC never represents in - // transactions. Exclude those. - // - // o sfPreviousTxnID is obsolete and was replaced by - // sfAccountTxnID some time before November of 2014. - // - // o sfWalletLocator and - // o sfWalletSize have been deprecated for six years or more. - // - // o sfTransactionType is not needed by gRPC, since the typing - // is handled using protobuf message types. - if constexpr (std::is_same_v) - { - static const std::set excludedTxFields{ - sfPreviousTxnID.fieldCode, - sfTransactionType.fieldCode, - sfWalletLocator.fieldCode, - sfWalletSize.fieldCode}; - - if (excludedTxFields.count(sField.fieldCode)) - continue; - } - - // If fmtId is a LedgerEntryType, exclude certain fields. - if constexpr (std::is_same_v) - { - // Fields that gRPC does not include in certain LedgerFormats. - // - // o sfWalletLocator, - // o sfWalletSize, - // o sfExchangeRate, and - // o sfFirstLedgerSequence are all deprecated fields in - // their respective ledger objects. - static const std:: - map> - gRPCOmitFields{ - {ltACCOUNT_ROOT, {&sfWalletLocator, &sfWalletSize}}, - {ltDIR_NODE, {&sfExchangeRate}}, - {ltLEDGER_HASHES, {&sfFirstLedgerSequence}}, - }; - - if (auto const iter = gRPCOmitFields.find(fmtId); - iter != gRPCOmitFields.end()) - { - std::vector const& omits = iter->second; - - // Check for fields that gRPC omits from this type. - if (std::find_if( - omits.begin(), - omits.end(), - [&sField](SField const* const omit) { - return *omit == sField; - }) != omits.end()) - { - // This is one of the fields that gRPC omits. - continue; - } - } - } - - // The SFields and gRPC disagree on the names of some fields. - // Provide a mapping from SField names to gRPC names for the - // known exceptions. - // - // clang-format off - // - // The implementers of the gRPC interface made the decision not - // to abbreviate anything. This accounts for the following - // field name differences: - // - // "AccountTxnID", "AccountTransactionID" - // "PreviousTxnID", "PreviousTransactionID" - // "PreviousTxnLgrSeq", "PreviousTransactionLedgerSequence" - // "SigningPubKey", "SigningPublicKey" - // "TxnSignature", "TransactionSignature" - // - // gRPC adds typing information for Fee, which accounts for - // "Fee", "XRPDropsAmount" - // - // There's one misspelling which accounts for - // "TakerGetsCurrency", "TakerGetsCurreny" - // - // The implementers of the gRPC interface observed that a - // PaymentChannelClaim transaction has a TxnSignature field at the - // upper level and a Signature field at the lever level. They - // felt that was confusing, which is the reason for - // "Signature", "PaymentChannelSignature" - // - static const std::map sFieldToGRPC{ - {"AccountTxnID", "AccountTransactionID"}, - {"Fee", "XRPDropsAmount"}, - {"PreviousTxnID", "PreviousTransactionID"}, - {"PreviousTxnLgrSeq", "PreviousTransactionLedgerSequence"}, - {"Signature", "PaymentChannelSignature"}, - {"SigningPubKey", "SigningPublicKey"}, - {"TakerGetsCurrency", "TakerGetsCurreny"}, - {"TxnSignature", "TransactionSignature"}, - }; - // clang-format on - - auto const iter = sFieldToGRPC.find(sField.getName()); - std::string gRPCName = - iter != sFieldToGRPC.end() ? iter->second : sField.getName(); - - sFields.insert({std::move(gRPCName), &sField}); - } - return sFields; - } - - // Given a Descriptor for a KnownFormat and a map of the SFields of that - // KnownFormat, make sure the fields are aligned. - void - validateDescriptorAgainstSFields( - google::protobuf::Descriptor const* const pbufDescriptor, - google::protobuf::Descriptor const* const commonFields, - std::string const& knownFormatName, - std::map&& sFields) - { - // Create namespace aliases for shorter names. - namespace pbuf = google::protobuf; - - // We'll be running through two sets of pbuf::Descriptors: the ones in - // the OneOf and the common fields. Here is a lambda that factors out - // the common checking code for these two cases. - auto checkFieldDesc = [this, &sFields, &knownFormatName]( - pbuf::FieldDescriptor const* const - fieldDesc) { - // gRPC has different handling for repeated vs non-repeated - // types. So we need to do that too. - std::string name; - if (fieldDesc->is_repeated()) - { - // Repeated-type handling. - - // Munge the fieldDescriptor name so it looks like the - // name in sFields. - name = fieldDesc->camelcase_name(); - name[0] = toupper(name[0]); - - // The ledger gives UNL all caps. Adapt to that. - if (size_t const i = name.find("Unl"); i != std::string::npos) - { - name[i + 1] = 'N'; - name[i + 2] = 'L'; - } - - // The ledger gives the NFT part of NFToken all caps. - // Adapt to that. - if (size_t const i = name.find("Nft"); i != std::string::npos) - { - name[i + 1] = 'F'; - name[i + 2] = 'T'; - } - - if (!sFields.count(name)) - { - fail( - std::string("Repeated Protobuf Descriptor '") + name + - "' expected in KnownFormat '" + knownFormatName + - "' and not found", - __FILE__, - __LINE__); - return; - } - pass(); - - validateRepeatedField(fieldDesc, sFields.at(name)); - } - else - { - // Non-repeated handling. - pbuf::Descriptor const* const entryDesc = - fieldDesc->message_type(); - if (entryDesc == nullptr) - return; - - name = entryDesc->name(); - if (!sFields.count(name)) - { - fail( - std::string("Protobuf Descriptor '") + - entryDesc->name() + "' expected in KnownFormat '" + - knownFormatName + "' and not found", - __FILE__, - __LINE__); - return; - } - pass(); - - validateDescriptor(entryDesc, sFields.at(entryDesc->name())); - } - // Remove the validated field from the map so we can tell if - // there are left over fields at the end of all comparisons. - sFields.erase(name); - }; - - // Compare the SFields to the FieldDescriptor->Descriptors. - for (int i = 0; i < pbufDescriptor->field_count(); ++i) - { - pbuf::FieldDescriptor const* const fieldDesc = - pbufDescriptor->field(i); - if (fieldDesc == nullptr || fieldDesc->type() != fieldTYPE_MESSAGE) - continue; - - checkFieldDesc(fieldDesc); - } - - // Now all of the OneOf-specific fields have been removed from - // sFields. But there may be common fields left in there. Process - // the commonFields next. - if (commonFields) - { - for (int i = 0; i < commonFields->field_count(); ++i) - { - // If the field we picked up is a OneOf, skip it. Common - // fields are never OneOfs. - pbuf::FieldDescriptor const* const fieldDesc = - commonFields->field(i); - - if (fieldDesc == nullptr || - fieldDesc->containing_oneof() != nullptr || - fieldDesc->type() != fieldTYPE_MESSAGE) - continue; - - checkFieldDesc(fieldDesc); - } - } - - // All SFields in the KnownFormat have corresponding gRPC fields - // if the sFields map is now empty. - if (!sFields.empty()) - { - fail( - std::string("Protobuf Descriptor '") + pbufDescriptor->name() + - "' did not account for all fields in KnownFormat '" + - knownFormatName + "'. Left over field: `" + - sFields.begin()->first + "'", - __FILE__, - __LINE__); - return; - } - pass(); - } - - // Compare a protobuf descriptor with multiple oneOfFields to choose from - // to an SField. - void - validateOneOfDescriptor( - google::protobuf::Descriptor const* const entryDesc, - SField const* const sField) - { - // Create namespace aliases for shorter names. - namespace pbuf = google::protobuf; - - // Note that it's not okay to compare names because SFields and - // gRPC do not always agree on the names. - if (entryDesc->field_count() == 0 || entryDesc->oneof_decl_count() != 1) - { - fail( - std::string("Protobuf Descriptor '") + entryDesc->name() + - "' expected to have multiple OneOf fields and nothing else", - __FILE__, - __LINE__); - return; - } - - pbuf::FieldDescriptor const* const fieldDesc = entryDesc->field(0); - if (fieldDesc == nullptr) - { - fail( - std::string("Internal test failure. Unhandled nullptr " - "in FieldDescriptor for '") + - entryDesc->name() + "'", - __FILE__, - __LINE__); - return; - } - - // Special handling for CurrencyAmount - if (sField->fieldType == STI_AMOUNT && - entryDesc->name() == "CurrencyAmount") - { - // SFields of type STI_AMOUNT are represented in gRPC by a - // multi-field CurrencyAmount. We don't really learn anything - // by diving into the interior of CurrencyAmount, so we stop here - // and call it good. - pass(); - return; - } - - fail( - std::string("Unhandled OneOf Protobuf Descriptor '") + - entryDesc->name() + "'", - __FILE__, - __LINE__); - } - - void - validateMultiFieldDescriptor( - google::protobuf::Descriptor const* const entryDesc, - SField const* const sField) - { - // Create namespace aliases for shorter names. - namespace pbuf = google::protobuf; - - if (entryDesc->field_count() <= 1 || entryDesc->oneof_decl_count() != 0) - { - fail( - std::string("Protobuf Descriptor '") + entryDesc->name() + - "' expected to have multiple fields and nothing else", - __FILE__, - __LINE__); - return; - } - - // There are composite fields that the SFields handle differently - // from gRPC. Handle those here. - { - struct FieldContents - { - std::string_view fieldName; - google::protobuf::FieldDescriptor::Type fieldType; - - bool - operator<(FieldContents const& other) const - { - return this->fieldName < other.fieldName; - } - - bool - operator==(FieldContents const& other) const - { - return this->fieldName == other.fieldName && - this->fieldType == other.fieldType; - } - }; - - struct SpecialEntry - { - std::string_view const descriptorName; - SerializedTypeID const sFieldType; - std::set const fields; - }; - - // clang-format off - static const std::array specialEntries{ - SpecialEntry{ - "Currency", STI_UINT160, - { - {"name", fieldTYPE_STRING}, - {"code", fieldTYPE_BYTES} - } - }, - SpecialEntry{ - "Memo", STI_OBJECT, - { - {"memo_data", fieldTYPE_BYTES}, - {"memo_format", fieldTYPE_BYTES}, - {"memo_type", fieldTYPE_BYTES} - } - } - }; - // clang-format on - - // If we're handling a SpecialEntry... - if (auto const iter = std::find_if( - specialEntries.begin(), - specialEntries.end(), - [entryDesc, sField](SpecialEntry const& entry) { - return entryDesc->name() == entry.descriptorName && - sField->fieldType == entry.sFieldType; - }); - iter != specialEntries.end()) - { - // Verify the SField. - if (!BEAST_EXPECT(sField->fieldType == iter->sFieldType)) - return; - - // Verify all of the fields in the entryDesc. - if (!BEAST_EXPECT( - entryDesc->field_count() == iter->fields.size())) - return; - - for (int i = 0; i < entryDesc->field_count(); ++i) - { - pbuf::FieldDescriptor const* const fieldDesc = - entryDesc->field(i); - - FieldContents const contents{ - fieldDesc->name(), fieldDesc->type()}; - - if (!BEAST_EXPECT( - iter->fields.find(contents) != iter->fields.end())) - return; - } - - // This field is good. - pass(); - return; - } - } - - // If the field was not one of the SpecialEntries, we expect it to be - // an InnerObjectFormat. - SOTemplate const* const innerFormat = - InnerObjectFormats::getInstance().findSOTemplateBySField(*sField); - if (innerFormat == nullptr) - { - fail( - "SOTemplate for field '" + sField->getName() + "' not found", - __FILE__, - __LINE__); - return; - } - - // Create a map we can use use to correlate each field in the - // gRPC Descriptor to its corresponding SField. - std::map sFields = - soTemplateToSFields(*innerFormat, 0); - - // Compare the SFields to the FieldDescriptor->Descriptors. - validateDescriptorAgainstSFields( - entryDesc, nullptr, sField->getName(), std::move(sFields)); - } - - // Compare a protobuf descriptor with only one field to an SField. - void - validateOneDescriptor( - google::protobuf::Descriptor const* const entryDesc, - SField const* const sField) - { - // Create namespace aliases for shorter names. - namespace pbuf = google::protobuf; - - // Note that it's not okay to compare names because SFields and - // gRPC do not always agree on the names. - if (entryDesc->field_count() != 1 || entryDesc->oneof_decl_count() != 0) - { - fail( - std::string("Protobuf Descriptor '") + entryDesc->name() + - "' expected to be one field and nothing else", - __FILE__, - __LINE__); - return; - } - - pbuf::FieldDescriptor const* const fieldDesc = entryDesc->field(0); - if (fieldDesc == nullptr) - { - fail( - std::string("Internal test failure. Unhandled nullptr " - "in FieldDescriptor for '") + - entryDesc->name() + "'", - __FILE__, - __LINE__); - return; - } - - // Create a map from SerializedTypeID to pbuf::FieldDescriptor::Type. - // - // This works for most, but not all, types because of divergence - // between the gRPC and LedgerFormat implementations. We deal - // with the special cases later. - // clang-format off - static const std::map - sTypeToFieldDescType{ - {STI_UINT8, fieldTYPE_UINT32}, - {STI_UINT16, fieldTYPE_UINT32}, - {STI_UINT32, fieldTYPE_UINT32}, - - {STI_UINT64, fieldTYPE_UINT64}, - - {STI_ACCOUNT, fieldTYPE_STRING}, - - {STI_AMOUNT, fieldTYPE_BYTES}, - {STI_UINT128, fieldTYPE_BYTES}, - {STI_UINT160, fieldTYPE_BYTES}, - {STI_UINT256, fieldTYPE_BYTES}, - {STI_VL, fieldTYPE_BYTES}, - }; - //clang-format on - - // If the SField and FieldDescriptor::Type correlate we're good. - if (auto const iter = sTypeToFieldDescType.find(sField->fieldType); - iter != sTypeToFieldDescType.end() && - iter->second == fieldDesc->type()) - { - pass(); - return; - } - - // Handle special cases for specific SFields. - static const std::map - sFieldCodeToFieldDescType{ - {sfDomain.fieldCode, fieldTYPE_STRING}, - {sfFee.fieldCode, fieldTYPE_UINT64}, - {sfURI.fieldCode, fieldTYPE_STRING}}; - - if (auto const iter = sFieldCodeToFieldDescType.find(sField->fieldCode); - iter != sFieldCodeToFieldDescType.end() && - iter->second == fieldDesc->type()) - { - pass(); - return; - } - - // Special handling for all Message types. - if (fieldDesc->type() == fieldTYPE_MESSAGE) - { - // We need to recurse to get to the bottom of the field(s) - // in question. - - // Start by identifying which fields we need to be handling. - // clang-format off - static const std::map messageMap{ - {sfAccount.fieldCode, "AccountAddress"}, - {sfAmount.fieldCode, "CurrencyAmount"}, - {sfAuthorize.fieldCode, "AccountAddress"}, - {sfBalance.fieldCode, "CurrencyAmount"}, - {sfDestination.fieldCode, "AccountAddress"}, - {sfFee.fieldCode, "XRPDropsAmount"}, - {sfHighLimit.fieldCode, "CurrencyAmount"}, - {sfLowLimit.fieldCode, "CurrencyAmount"}, - {sfOwner.fieldCode, "AccountAddress"}, - {sfRegularKey.fieldCode, "AccountAddress"}, - {sfSendMax.fieldCode, "CurrencyAmount"}, - {sfTakerGets.fieldCode, "CurrencyAmount"}, - {sfTakerGetsCurrency.fieldCode, "Currency"}, - {sfTakerPays.fieldCode, "CurrencyAmount"}, - {sfTakerPaysCurrency.fieldCode, "Currency"}, - }; - // clang-format on - if (messageMap.count(sField->fieldCode)) - { - pbuf::Descriptor const* const entry2Desc = - fieldDesc->message_type(); - - if (entry2Desc == nullptr) - { - fail( - std::string("Unexpected gRPC. ") + fieldDesc->name() + - " MESSAGE with null Descriptor", - __FILE__, - __LINE__); - return; - } - - // The Descriptor name should match the messageMap name. - if (messageMap.at(sField->fieldCode) != entry2Desc->name()) - { - fail( - std::string( - "Internal test error. Mismatch between SField '") + - sField->getName() + "' and gRPC Descriptor name '" + - entry2Desc->name() + "'", - __FILE__, - __LINE__); - return; - } - pass(); - - // Recurse to the next lower Descriptor. - validateDescriptor(entry2Desc, sField); - } - return; - } - - fail( - std::string("Internal test error. Unhandled FieldDescriptor '") + - entryDesc->name() + "' has type `" + fieldDesc->type_name() + - "` and label " + std::to_string(fieldDesc->label()), - __FILE__, - __LINE__); - } - - // Compare a repeated protobuf FieldDescriptor to an SField. - void - validateRepeatedField( - google::protobuf::FieldDescriptor const* const fieldDesc, - SField const* const sField) - { - // Create namespace aliases for shorter names. - namespace pbuf = google::protobuf; - - pbuf::Descriptor const* const entryDesc = fieldDesc->message_type(); - if (entryDesc == nullptr) - { - fail( - std::string("Expected Descriptor for repeated type ") + - sField->getName(), - __FILE__, - __LINE__); - return; - } - - // The following repeated types provide no further structure for their - // in-ledger representation. We just have to trust that the gRPC - // representation is reasonable for what the ledger implements. - static const std::set noFurtherDetail{ - {sfPaths.getName()}, - }; - - if (noFurtherDetail.count(sField->getName())) - { - // There is no Format representation for further details of this - // repeated type. We've done the best we can. - pass(); - return; - } - - // All of the repeated types that the test currently supports. - static const std::map repeatsWhat{ - {sfAmendments.getName(), &sfAmendment}, - {sfDisabledValidators.getName(), &sfDisabledValidator}, - {sfHashes.getName(), &sfLedgerHash}, - {sfIndexes.getName(), &sfLedgerIndex}, - {sfMajorities.getName(), &sfMajority}, - {sfMemos.getName(), &sfMemo}, - {sfNFTokens.getName(), &sfNFToken}, - {sfSignerEntries.getName(), &sfSignerEntry}, - {sfSigners.getName(), &sfSigner}, - {sfNFTokenOffers.getName(), &sfLedgerIndex}}; - - if (!repeatsWhat.count(sField->getName())) - { - fail( - std::string("Unexpected repeated type ") + fieldDesc->name(), - __FILE__, - __LINE__); - return; - } - pass(); - - // Process the type contained by the repeated type. - validateDescriptor(entryDesc, repeatsWhat.at(sField->getName())); - } - - // Determine which of the Descriptor validators to dispatch to. - void - validateDescriptor( - google::protobuf::Descriptor const* const entryDesc, - SField const* const sField) - { - if (entryDesc->nested_type_count() != 0 || - entryDesc->enum_type_count() != 0 || - entryDesc->extension_range_count() != 0 || - entryDesc->reserved_range_count() != 0) - { - fail( - std::string("Protobuf Descriptor '") + entryDesc->name() + - "' uses unsupported protobuf features", - __FILE__, - __LINE__); - return; - } - - // Dispatch to the correct validator - if (entryDesc->oneof_decl_count() > 0) - return validateOneOfDescriptor(entryDesc, sField); - - if (entryDesc->field_count() > 1) - return validateMultiFieldDescriptor(entryDesc, sField); - - return validateOneDescriptor(entryDesc, sField); - } - - // Compare a protobuf descriptor to a KnownFormat::Item - template - void - validateFields( - google::protobuf::Descriptor const* const pbufDescriptor, - google::protobuf::Descriptor const* const commonFields, - typename KnownFormats::Item const* const - knownFormatItem) - { - // Create namespace aliases for shorter names. - namespace pbuf = google::protobuf; - - // The names should usually be the same, but the bpufDescriptor - // name might have "Object" appended. - if (knownFormatItem->getName() != pbufDescriptor->name() && - knownFormatItem->getName() + "Object" != pbufDescriptor->name()) - { - fail( - std::string("Protobuf Descriptor '") + pbufDescriptor->name() + - "' and KnownFormat::Item '" + knownFormatItem->getName() + - "' don't have the same name", - __FILE__, - __LINE__); - return; - } - pass(); - - // Create a map we can use use to correlate each field in the - // gRPC Descriptor to its corresponding SField. - std::map sFields = soTemplateToSFields( - knownFormatItem->getSOTemplate(), knownFormatItem->getType()); - - // Compare the SFields to the FieldDescriptor->Descriptors. - validateDescriptorAgainstSFields( - pbufDescriptor, - commonFields, - knownFormatItem->getName(), - std::move(sFields)); - } - - template - void - testKnownFormats( - KnownFormats const& knownFormat, - std::string const& knownFormatName, - google::protobuf::Descriptor const* const commonFields, - google::protobuf::OneofDescriptor const* const oneOfDesc) - { - // Create namespace aliases for shorter names. - namespace grpc = org::xrpl::rpc::v1; - namespace pbuf = google::protobuf; - - if (!BEAST_EXPECT(oneOfDesc != nullptr)) - return; - - // Get corresponding names for all KnownFormat Items. - std::map< - std::string, - typename KnownFormats::Item const*> - formatTypes; - - for (auto const& item : knownFormat) - { - if constexpr (std::is_same_v) - { - // Skip LedgerEntryTypes that gRPC does not currently support. - static constexpr std::array notSupported{}; - - if (std::find( - notSupported.begin(), - notSupported.end(), - item.getType()) != notSupported.end()) - continue; - } - - if constexpr (std::is_same_v) - { - // Skip TxTypes that gRPC does not currently support. - static constexpr std::array notSupported{ - ttAMENDMENT, ttFEE, ttUNL_MODIFY}; - - if (std::find( - notSupported.begin(), - notSupported.end(), - item.getType()) != notSupported.end()) - continue; - } - - BEAST_EXPECT( - formatTypes - .insert({formatNameToEntryTypeName(item.getName()), &item}) - .second == true); - } - - // Verify that the OneOf objects match. Start by comparing - // KnownFormat vs gRPC OneOf counts. - { - BEAST_EXPECT(formatTypes.size() == oneOfDesc->field_count()); - } - - // This loop - // 1. Iterates through the gRPC OneOfs, - // 2. Finds each gRPC OneOf's matching KnownFormat::Item, - // 3. Sanity checks that the fields of the objects align well. - for (auto i = 0; i < oneOfDesc->field_count(); ++i) - { - pbuf::FieldDescriptor const* const fieldDesc = oneOfDesc->field(i); - - // The Field should be a TYPE_MESSAGE, which means we can get its - // descriptor. - if (fieldDesc->type() != fieldTYPE_MESSAGE) - { - fail( - std::string("gRPC OneOf '") + fieldDesc->name() + - "' is not TYPE_MESSAGE", - __FILE__, - __LINE__); - continue; - } - - auto const fmtIter = formatTypes.find(fieldDesc->name()); - - if (fmtIter == formatTypes.cend()) - { - fail( - std::string("gRPC OneOf '") + fieldDesc->name() + - "' not found in " + knownFormatName, - __FILE__, - __LINE__); - continue; - } - - // Validate that the gRPC and KnownFormat fields align. - validateFields( - fieldDesc->message_type(), commonFields, fmtIter->second); - - // Remove the checked KnownFormat from the map. This way we - // can check for leftovers when we're done processing. - formatTypes.erase(fieldDesc->name()); - } - - // Report any KnownFormats that don't have gRPC OneOfs. - for (auto const& spare : formatTypes) - { - fail( - knownFormatName + " '" + spare.second->getName() + - "' does not have a corresponding gRPC OneOf", - __FILE__, - __LINE__); - } - } - -public: - void - testLedgerObjectGRPCOneOfs() - { - testcase("Ledger object validation"); - - org::xrpl::rpc::v1::LedgerObject const ledgerObject; - - testKnownFormats( - LedgerFormats::getInstance(), - "LedgerFormats", - ledgerObject.GetDescriptor(), - ledgerObject.GetDescriptor()->FindOneofByName("object")); - - return; - } - - void - testTransactionGRPCOneOfs() - { - testcase("Transaction validation"); - - org::xrpl::rpc::v1::Transaction const txData; - - testKnownFormats( - TxFormats::getInstance(), - "TxFormats", - txData.GetDescriptor(), - txData.GetDescriptor()->FindOneofByName("transaction_data")); - - return; - } - - void - run() override - { - testLedgerObjectGRPCOneOfs(); - testTransactionGRPCOneOfs(); - } -}; - -BEAST_DEFINE_TESTSUITE(KnownFormatToGRPC, protocol, ripple); - -} // namespace ripple diff --git a/src/test/protocol/STTx_test.cpp b/src/test/protocol/STTx_test.cpp index 5d24efcfc..4ef30fb7a 100644 --- a/src/test/protocol/STTx_test.cpp +++ b/src/test/protocol/STTx_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -27,7 +28,6 @@ #include #include #include - #include #include @@ -1591,8 +1591,10 @@ public: }); j.sign(keypair.first, keypair.second); + Rules defaultRules{{}}; + unexpected( - !j.checkSign(STTx::RequireFullyCanonicalSig::yes), + !j.checkSign(STTx::RequireFullyCanonicalSig::yes, defaultRules), "Transaction fails signature test"); Serializer rawTxn; diff --git a/src/test/rpc/AccountInfo_test.cpp b/src/test/rpc/AccountInfo_test.cpp index 8ce8270ad..9772a0ffb 100644 --- a/src/test/rpc/AccountInfo_test.cpp +++ b/src/test/rpc/AccountInfo_test.cpp @@ -491,227 +491,6 @@ public: } } - // gRPC stuff - class GetAccountInfoClient : public GRPCTestClientBase - { - public: - org::xrpl::rpc::v1::GetAccountInfoRequest request; - org::xrpl::rpc::v1::GetAccountInfoResponse reply; - - explicit GetAccountInfoClient(std::string const& port) - : GRPCTestClientBase(port) - { - } - - void - GetAccountInfo() - { - status = stub_->GetAccountInfo(&context, request, &reply); - } - }; - - void - testSimpleGrpc() - { - testcase("gRPC simple"); - - using namespace jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - Account const alice{"alice"}; - env.fund(drops(1000 * 1000 * 1000), alice); - - { - // most simple case - GetAccountInfoClient client(grpcPort); - client.request.mutable_account()->set_address(alice.human()); - client.GetAccountInfo(); - if (!BEAST_EXPECT(client.status.ok())) - { - return; - } - BEAST_EXPECT( - client.reply.account_data().account().value().address() == - alice.human()); - } - { - GetAccountInfoClient client(grpcPort); - client.request.mutable_account()->set_address(alice.human()); - client.request.set_queue(true); - client.request.mutable_ledger()->set_sequence(3); - client.GetAccountInfo(); - if (!BEAST_EXPECT(client.status.ok())) - return; - BEAST_EXPECT( - client.reply.account_data() - .balance() - .value() - .xrp_amount() - .drops() == 1000 * 1000 * 1000); - BEAST_EXPECT( - client.reply.account_data().account().value().address() == - alice.human()); - BEAST_EXPECT( - client.reply.account_data().sequence().value() == - env.seq(alice)); - BEAST_EXPECT(client.reply.queue_data().txn_count() == 0); - } - } - - void - testErrorsGrpc() - { - testcase("gRPC errors"); - - using namespace jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - auto getClient = [&grpcPort]() { - return GetAccountInfoClient(grpcPort); - }; - Account const alice{"alice"}; - env.fund(drops(1000 * 1000 * 1000), alice); - - { - // bad address - auto client = getClient(); - client.request.mutable_account()->set_address("deadbeef"); - client.GetAccountInfo(); - BEAST_EXPECT(!client.status.ok()); - } - { - // no account - Account const bogie{"bogie"}; - auto client = getClient(); - client.request.mutable_account()->set_address(bogie.human()); - client.GetAccountInfo(); - BEAST_EXPECT(!client.status.ok()); - } - { - // bad ledger_index - auto client = getClient(); - client.request.mutable_account()->set_address(alice.human()); - client.request.mutable_ledger()->set_sequence(0); - client.GetAccountInfo(); - BEAST_EXPECT(!client.status.ok()); - } - } - - void - testSignerListsGrpc() - { - testcase("gRPC singer lists"); - - using namespace jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - auto getClient = [&grpcPort]() { - return GetAccountInfoClient(grpcPort); - }; - - Account const alice{"alice"}; - env.fund(drops(1000 * 1000 * 1000), alice); - - { - auto client = getClient(); - client.request.mutable_account()->set_address(alice.human()); - client.request.set_signer_lists(true); - client.GetAccountInfo(); - if (!BEAST_EXPECT(client.status.ok())) - return; - BEAST_EXPECT(client.reply.signer_list().signer_entries_size() == 0); - } - - // Give alice a SignerList. - Account const bogie{"bogie"}; - Json::Value const smallSigners = signers(alice, 2, {{bogie, 3}}); - env(smallSigners); - { - auto client = getClient(); - client.request.mutable_account()->set_address(alice.human()); - client.request.set_signer_lists(false); - client.GetAccountInfo(); - if (!BEAST_EXPECT(client.status.ok())) - return; - BEAST_EXPECT(client.reply.signer_list().signer_entries_size() == 0); - } - { - auto client = getClient(); - client.request.mutable_account()->set_address(alice.human()); - client.request.set_signer_lists(true); - client.GetAccountInfo(); - if (!BEAST_EXPECT(client.status.ok())) - { - return; - } - BEAST_EXPECT( - client.reply.account_data().owner_count().value() == 1); - BEAST_EXPECT(client.reply.signer_list().signer_entries_size() == 1); - } - - // Give alice a big signer list - Account const demon{"demon"}; - Account const ghost{"ghost"}; - Account const haunt{"haunt"}; - Account const jinni{"jinni"}; - Account const phase{"phase"}; - Account const shade{"shade"}; - Account const spook{"spook"}; - Json::Value const bigSigners = signers( - alice, - 4, - { - {bogie, 1}, - {demon, 1}, - {ghost, 1}, - {haunt, 1}, - {jinni, 1}, - {phase, 1}, - {shade, 1}, - {spook, 1}, - }); - env(bigSigners); - - std::set accounts; - accounts.insert(bogie.human()); - accounts.insert(demon.human()); - accounts.insert(ghost.human()); - accounts.insert(haunt.human()); - accounts.insert(jinni.human()); - accounts.insert(phase.human()); - accounts.insert(shade.human()); - accounts.insert(spook.human()); - { - auto client = getClient(); - client.request.mutable_account()->set_address(alice.human()); - client.request.set_signer_lists(true); - client.GetAccountInfo(); - if (!BEAST_EXPECT(client.status.ok())) - { - return; - } - BEAST_EXPECT( - client.reply.account_data().owner_count().value() == 1); - auto& signerList = client.reply.signer_list(); - BEAST_EXPECT(signerList.signer_quorum().value() == 4); - BEAST_EXPECT(signerList.signer_entries_size() == 8); - for (int i = 0; i < 8; ++i) - { - BEAST_EXPECT( - signerList.signer_entries(i).signer_weight().value() == 1); - BEAST_EXPECT( - accounts.erase(signerList.signer_entries(i) - .account() - .value() - .address()) == 1); - } - BEAST_EXPECT(accounts.size() == 0); - } - } - void run() override { @@ -719,9 +498,6 @@ public: testSignerLists(); testSignerListsApiVersion2(); testSignerListsV2(); - testSimpleGrpc(); - testErrorsGrpc(); - testSignerListsGrpc(); } }; diff --git a/src/test/rpc/AccountTx_test.cpp b/src/test/rpc/AccountTx_test.cpp index f5709c28e..1d537d477 100644 --- a/src/test/rpc/AccountTx_test.cpp +++ b/src/test/rpc/AccountTx_test.cpp @@ -547,7 +547,7 @@ class AccountTx_test : public beast::unit_test::suite // All it takes is a large enough XRP payment to resurrect // becky's account. Try too small a payment. - env(pay(alice, becky, XRP(19)), ter(tecNO_DST_INSUF_XRP)); + env(pay(alice, becky, XRP(9)), ter(tecNO_DST_INSUF_XRP)); env.close(); // Actually resurrect becky's account. diff --git a/src/test/rpc/AmendmentBlocked_test.cpp b/src/test/rpc/AmendmentBlocked_test.cpp index 54c872922..a90bcdcd0 100644 --- a/src/test/rpc/AmendmentBlocked_test.cpp +++ b/src/test/rpc/AmendmentBlocked_test.cpp @@ -43,6 +43,12 @@ class AmendmentBlocked_test : public beast::unit_test::suite Account const ali{"ali", KeyType::secp256k1}; env.fund(XRP(10000), alice, bob, gw); env.memoize(ali); + // This close() ensures that all the accounts get created and their + // default ripple flag gets set before the trust lines are created. + // Without it, the ordering manages to create alice's trust line with + // noRipple set on gw's end. The existing tests pass either way, but + // better to do it right. + env.close(); env.trust(USD(600), alice); env.trust(USD(700), bob); env(pay(gw, alice, USD(70))); diff --git a/src/test/rpc/Fee_test.cpp b/src/test/rpc/Fee_test.cpp deleted file mode 100644 index 17ada929e..000000000 --- a/src/test/rpc/Fee_test.cpp +++ /dev/null @@ -1,138 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ripple { -namespace test { - -class Fee_test : public beast::unit_test::suite -{ - class GrpcFeeClient : public GRPCTestClientBase - { - public: - org::xrpl::rpc::v1::GetFeeRequest request; - org::xrpl::rpc::v1::GetFeeResponse reply; - - explicit GrpcFeeClient(std::string const& grpcPort) - : GRPCTestClientBase(grpcPort) - { - } - - void - GetFee() - { - status = stub_->GetFee(&context, request, &reply); - } - }; - - std::pair - grpcGetFee(std::string const& grpcPort) - { - GrpcFeeClient client(grpcPort); - client.GetFee(); - return std::pair( - client.status.ok(), client.reply); - } - - void - testFeeGrpc() - { - testcase("Test Fee Grpc"); - - using namespace test::jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - Account A1{"A1"}; - Account A2{"A2"}; - env.fund(XRP(10000), A1); - env.fund(XRP(10000), A2); - env.close(); - env.trust(A2["USD"](1000), A1); - env.close(); - for (int i = 0; i < 7; ++i) - { - env(pay(A2, A1, A2["USD"](100))); - if (i == 4) - env.close(); - } - - auto view = env.current(); - - auto const metrics = env.app().getTxQ().getMetrics(*env.current()); - - auto const result = grpcGetFee(grpcPort); - - BEAST_EXPECT(result.first == true); - - auto reply = result.second; - - // current ledger data - BEAST_EXPECT(reply.current_ledger_size() == metrics.txInLedger); - BEAST_EXPECT(reply.current_queue_size() == metrics.txCount); - BEAST_EXPECT(reply.expected_ledger_size() == metrics.txPerLedger); - BEAST_EXPECT(reply.ledger_current_index() == view->info().seq); - BEAST_EXPECT(reply.max_queue_size() == *metrics.txQMaxSize); - - // fee levels data - org::xrpl::rpc::v1::FeeLevels& levels = *reply.mutable_levels(); - BEAST_EXPECT(levels.median_level() == metrics.medFeeLevel); - BEAST_EXPECT(levels.minimum_level() == metrics.minProcessingFeeLevel); - BEAST_EXPECT(levels.open_ledger_level() == metrics.openLedgerFeeLevel); - BEAST_EXPECT(levels.reference_level() == metrics.referenceFeeLevel); - - // fee data - org::xrpl::rpc::v1::Fee& fee = *reply.mutable_fee(); - auto const baseFee = view->fees().base; - BEAST_EXPECT( - fee.base_fee().drops() == - toDrops(metrics.referenceFeeLevel, baseFee)); - BEAST_EXPECT( - fee.minimum_fee().drops() == - toDrops(metrics.minProcessingFeeLevel, baseFee)); - BEAST_EXPECT( - fee.median_fee().drops() == toDrops(metrics.medFeeLevel, baseFee)); - auto openLedgerFee = - toDrops(metrics.openLedgerFeeLevel - FeeLevel64{1}, baseFee) + 1; - BEAST_EXPECT(fee.open_ledger_fee().drops() == openLedgerFee.drops()); - } - -public: - void - run() override - { - testFeeGrpc(); - } -}; - -BEAST_DEFINE_TESTSUITE(Fee, app, ripple); - -} // namespace test -} // namespace ripple diff --git a/src/test/rpc/LedgerRPC_test.cpp b/src/test/rpc/LedgerRPC_test.cpp index 1692b9806..2580c4bfe 100644 --- a/src/test/rpc/LedgerRPC_test.cpp +++ b/src/test/rpc/LedgerRPC_test.cpp @@ -1250,13 +1250,10 @@ class LedgerRPC_test : public beast::unit_test::suite // no amendments env.fund(XRP(10000), "alice"); env.close(); - log << env.closed()->info().hash; env.fund(XRP(10000), "bob"); env.close(); - log << env.closed()->info().hash; env.fund(XRP(10000), "jim"); env.close(); - log << env.closed()->info().hash; env.fund(XRP(10000), "jill"); { diff --git a/src/test/rpc/NodeToShardRPC_test.cpp b/src/test/rpc/NodeToShardRPC_test.cpp index edfaf6c20..867f2cfe3 100644 --- a/src/test/rpc/NodeToShardRPC_test.cpp +++ b/src/test/rpc/NodeToShardRPC_test.cpp @@ -194,64 +194,74 @@ public: result[jss::status] == "success" || importCompleted(shardStore, numberOfShards, result)); - std::chrono::seconds const maxWait{60}; - auto const start = std::chrono::system_clock::now(); + std::chrono::seconds const maxWait{180}; - while (true) { - // Verify that the status object accurately - // reflects import progress. - - auto const completeShards = - shardStore->getShardInfo()->finalized(); - - if (!completeShards.empty()) + auto const start = std::chrono::system_clock::now(); + while (true) { - auto const result = env.rpc( - "json", - "node_to_shard", - to_string(jvParams))[jss::result]; + // Verify that the status object accurately + // reflects import progress. - if (!importCompleted(shardStore, numberOfShards, result)) + auto const completeShards = + shardStore->getShardInfo()->finalized(); + + if (!completeShards.empty()) { - BEAST_EXPECT(result[jss::firstShardIndex] == 1); - BEAST_EXPECT(result[jss::lastShardIndex] == 10); + auto const result = env.rpc( + "json", + "node_to_shard", + to_string(jvParams))[jss::result]; + + if (!importCompleted( + shardStore, numberOfShards, result)) + { + BEAST_EXPECT(result[jss::firstShardIndex] == 1); + BEAST_EXPECT(result[jss::lastShardIndex] == 10); + } } - } - if (boost::icl::contains(completeShards, 1)) - { - auto const result = env.rpc( - "json", - "node_to_shard", - to_string(jvParams))[jss::result]; + if (boost::icl::contains(completeShards, 1)) + { + auto const result = env.rpc( + "json", + "node_to_shard", + to_string(jvParams))[jss::result]; - BEAST_EXPECT( - result[jss::currentShardIndex] >= 1 || - importCompleted(shardStore, numberOfShards, result)); + BEAST_EXPECT( + result[jss::currentShardIndex] >= 1 || + importCompleted( + shardStore, numberOfShards, result)); - break; - } + break; + } - if (std::this_thread::sleep_for(std::chrono::milliseconds{100}); - std::chrono::system_clock::now() - start > maxWait) - { - BEAST_EXPECTS( - false, "Import timeout: could just be a slow machine."); - break; + if (std::this_thread::sleep_for( + std::chrono::milliseconds{100}); + std::chrono::system_clock::now() - start > maxWait) + { + BEAST_EXPECTS( + false, + "Import timeout: could just be a slow machine."); + break; + } } } - // Wait for the import to complete - while (!boost::icl::contains( - shardStore->getShardInfo()->finalized(), 10)) { - if (std::this_thread::sleep_for(std::chrono::milliseconds{100}); - std::chrono::system_clock::now() - start > maxWait) + // Wait for the import to complete + auto const start = std::chrono::system_clock::now(); + while (!boost::icl::contains( + shardStore->getShardInfo()->finalized(), 10)) { - BEAST_EXPECT( - importCompleted(shardStore, numberOfShards, result)); - break; + if (std::this_thread::sleep_for( + std::chrono::milliseconds{100}); + std::chrono::system_clock::now() - start > maxWait) + { + BEAST_EXPECT(importCompleted( + shardStore, numberOfShards, result)); + break; + } } } } @@ -276,7 +286,8 @@ public: sectionNode.set("ledgers_per_shard", "256"); c->setupControl(true, true, true); - return jtx::Env(*this, std::move(c)); + return jtx::Env( + *this, std::move(c), nullptr, beast::severities::kDisabled); }(); std::uint8_t const numberOfShards = 10; @@ -322,7 +333,7 @@ public: result[jss::status] == "success" || importCompleted(shardStore, numberOfShards, result)); - std::chrono::seconds const maxWait{10}; + std::chrono::seconds const maxWait{30}; auto const start = std::chrono::system_clock::now(); while (shardStore->getShardInfo()->finalized().empty()) @@ -351,7 +362,7 @@ public: importCompleted(shardStore, numberOfShards, result)); } - std::chrono::seconds const maxWait{10}; + std::chrono::seconds const maxWait{30}; auto const start = std::chrono::system_clock::now(); while (true) diff --git a/src/test/rpc/ReportingETL_test.cpp b/src/test/rpc/ReportingETL_test.cpp index 36b2f9b0b..d8e6fc684 100644 --- a/src/test/rpc/ReportingETL_test.cpp +++ b/src/test/rpc/ReportingETL_test.cpp @@ -773,25 +773,6 @@ class ReportingETL_test : public beast::unit_test::suite testNeedCurrentOrClosed() { testcase("NeedCurrentOrClosed"); - { - org::xrpl::rpc::v1::GetAccountInfoRequest request; - request.mutable_ledger()->set_sequence(1); - BEAST_EXPECT(!needCurrentOrClosed(request)); - request.mutable_ledger()->set_hash(""); - BEAST_EXPECT(!needCurrentOrClosed(request)); - request.mutable_ledger()->set_shortcut( - org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_VALIDATED); - BEAST_EXPECT(!needCurrentOrClosed(request)); - request.mutable_ledger()->set_shortcut( - org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_UNSPECIFIED); - BEAST_EXPECT(!needCurrentOrClosed(request)); - request.mutable_ledger()->set_shortcut( - org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CURRENT); - BEAST_EXPECT(needCurrentOrClosed(request)); - request.mutable_ledger()->set_shortcut( - org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CLOSED); - BEAST_EXPECT(needCurrentOrClosed(request)); - } { org::xrpl::rpc::v1::GetLedgerRequest request; @@ -904,18 +885,6 @@ class ReportingETL_test : public beast::unit_test::suite org::xrpl::rpc::v1::LedgerSpecifier::SHORTCUT_CURRENT); BEAST_EXPECT(needCurrentOrClosed(request)); } - - { - org::xrpl::rpc::v1::GetFeeRequest feeRequest; - BEAST_EXPECT(!needCurrentOrClosed(feeRequest)); - - org::xrpl::rpc::v1::GetAccountTransactionHistoryRequest - accountTxRequest; - BEAST_EXPECT(!needCurrentOrClosed(accountTxRequest)); - - org::xrpl::rpc::v1::GetTransactionRequest txRequest; - BEAST_EXPECT(!needCurrentOrClosed(txRequest)); - } } void diff --git a/src/test/rpc/ShardArchiveHandler_test.cpp b/src/test/rpc/ShardArchiveHandler_test.cpp index 296699b3c..ee0bec1ea 100644 --- a/src/test/rpc/ShardArchiveHandler_test.cpp +++ b/src/test/rpc/ShardArchiveHandler_test.cpp @@ -17,7 +17,7 @@ */ //============================================================================== -#include +#include #include #include #include @@ -173,7 +173,8 @@ public: } c->setupControl(true, true, true); - jtx::Env env(*this, std::move(c)); + jtx::Env env( + *this, std::move(c), nullptr, beast::severities::kDisabled); std::uint8_t const numberOfDownloads = 10; @@ -276,7 +277,8 @@ public: } c->setupControl(true, true, true); - jtx::Env env(*this, std::move(c)); + jtx::Env env( + *this, std::move(c), nullptr, beast::severities::kDisabled); std::uint8_t const numberOfDownloads = 10; @@ -380,7 +382,8 @@ public: } c->setupControl(true, true, true); - jtx::Env env(*this, std::move(c)); + jtx::Env env( + *this, std::move(c), nullptr, beast::severities::kDisabled); std::uint8_t const numberOfDownloads = 10; // Create some ledgers so that the ShardArchiveHandler diff --git a/src/test/rpc/Submit_test.cpp b/src/test/rpc/Submit_test.cpp deleted file mode 100644 index 252e1c32a..000000000 --- a/src/test/rpc/Submit_test.cpp +++ /dev/null @@ -1,276 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace ripple { -namespace test { - -class Submit_test : public beast::unit_test::suite -{ -public: - class SubmitClient : public GRPCTestClientBase - { - public: - org::xrpl::rpc::v1::SubmitTransactionRequest request; - org::xrpl::rpc::v1::SubmitTransactionResponse reply; - - explicit SubmitClient(std::string const& port) - : GRPCTestClientBase(port) - { - } - - void - SubmitTransaction() - { - status = stub_->SubmitTransaction(&context, request, &reply); - } - }; - - struct TestData - { - std::string xrpTxBlob; - std::string xrpTxHash; - std::string usdTxBlob; - std::string usdTxHash; - const static int fund = 10000; - } testData; - - void - fillTestData() - { - testcase("fill test data"); - - using namespace jtx; - Env env(*this, envconfig(addGrpcConfig)); - auto const alice = Account("alice"); - auto const bob = Account("bob"); - env.fund(XRP(TestData::fund), "alice", "bob"); - env.trust(bob["USD"](TestData::fund), alice); - env.close(); - - auto toBinary = [this](std::string const& text) { - auto blob = strUnHex(text); - BEAST_EXPECT(blob); - return std::string{ - reinterpret_cast(blob->data()), blob->size()}; - }; - - // use a websocket client to fill transaction blobs - auto wsc = makeWSClient(env.app().config()); - { - Json::Value jrequestXrp; - jrequestXrp[jss::secret] = toBase58(generateSeed("alice")); - jrequestXrp[jss::tx_json] = - pay("alice", "bob", XRP(TestData::fund / 2)); - Json::Value jreply_xrp = wsc->invoke("sign", jrequestXrp); - - if (!BEAST_EXPECT(jreply_xrp.isMember(jss::result))) - return; - if (!BEAST_EXPECT(jreply_xrp[jss::result].isMember(jss::tx_blob))) - return; - testData.xrpTxBlob = - toBinary(jreply_xrp[jss::result][jss::tx_blob].asString()); - if (!BEAST_EXPECT(jreply_xrp[jss::result].isMember(jss::tx_json))) - return; - if (!BEAST_EXPECT( - jreply_xrp[jss::result][jss::tx_json].isMember(jss::hash))) - return; - testData.xrpTxHash = toBinary( - jreply_xrp[jss::result][jss::tx_json][jss::hash].asString()); - } - { - Json::Value jrequestUsd; - jrequestUsd[jss::secret] = toBase58(generateSeed("bob")); - jrequestUsd[jss::tx_json] = - pay("bob", "alice", bob["USD"](TestData::fund / 2)); - Json::Value jreply_usd = wsc->invoke("sign", jrequestUsd); - - if (!BEAST_EXPECT(jreply_usd.isMember(jss::result))) - return; - if (!BEAST_EXPECT(jreply_usd[jss::result].isMember(jss::tx_blob))) - return; - testData.usdTxBlob = - toBinary(jreply_usd[jss::result][jss::tx_blob].asString()); - if (!BEAST_EXPECT(jreply_usd[jss::result].isMember(jss::tx_json))) - return; - if (!BEAST_EXPECT( - jreply_usd[jss::result][jss::tx_json].isMember(jss::hash))) - return; - testData.usdTxHash = toBinary( - jreply_usd[jss::result][jss::tx_json][jss::hash].asString()); - } - } - - void - testSubmitGoodBlobGrpc() - { - testcase("Submit good blobs, XRP, USD, and same transaction twice"); - - using namespace jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - auto const alice = Account("alice"); - auto const bob = Account("bob"); - env.fund(XRP(TestData::fund), "alice", "bob"); - env.trust(bob["USD"](TestData::fund), alice); - env.close(); - - auto getClient = [&grpcPort]() { return SubmitClient(grpcPort); }; - - // XRP - { - auto client = getClient(); - client.request.set_signed_transaction(testData.xrpTxBlob); - client.SubmitTransaction(); - if (!BEAST_EXPECT(client.status.ok())) - { - return; - } - BEAST_EXPECT(client.reply.engine_result().result() == "tesSUCCESS"); - BEAST_EXPECT(client.reply.engine_result_code() == 0); - BEAST_EXPECT(client.reply.hash() == testData.xrpTxHash); - } - // USD - { - auto client = getClient(); - client.request.set_signed_transaction(testData.usdTxBlob); - client.SubmitTransaction(); - if (!BEAST_EXPECT(client.status.ok())) - { - return; - } - BEAST_EXPECT(client.reply.engine_result().result() == "tesSUCCESS"); - BEAST_EXPECT(client.reply.engine_result_code() == 0); - BEAST_EXPECT(client.reply.hash() == testData.usdTxHash); - } - // USD, error, same transaction again - { - auto client = getClient(); - client.request.set_signed_transaction(testData.usdTxBlob); - client.SubmitTransaction(); - if (!BEAST_EXPECT(client.status.ok())) - { - return; - } - BEAST_EXPECT( - client.reply.engine_result().result() == "tefPAST_SEQ"); - BEAST_EXPECT(client.reply.engine_result_code() == -190); - } - } - - void - testSubmitErrorBlobGrpc() - { - testcase("Submit error, bad blob, no account"); - - using namespace jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - - auto getClient = [&grpcPort]() { return SubmitClient(grpcPort); }; - - // short transaction blob, cannot parse - { - auto client = getClient(); - client.request.set_signed_transaction("deadbeef"); - client.SubmitTransaction(); - BEAST_EXPECT(!client.status.ok()); - } - // bad blob with correct length, cannot parse - { - auto client = getClient(); - auto xrpTxBlobCopy(testData.xrpTxBlob); - std::reverse(xrpTxBlobCopy.begin(), xrpTxBlobCopy.end()); - client.request.set_signed_transaction(xrpTxBlobCopy); - client.SubmitTransaction(); - BEAST_EXPECT(!client.status.ok()); - } - // good blob, can parse but no account - { - auto client = getClient(); - client.request.set_signed_transaction(testData.xrpTxBlob); - client.SubmitTransaction(); - if (!BEAST_EXPECT(client.status.ok())) - { - return; - } - BEAST_EXPECT( - client.reply.engine_result().result() == "terNO_ACCOUNT"); - BEAST_EXPECT(client.reply.engine_result_code() == -96); - } - } - - void - testSubmitInsufficientFundsGrpc() - { - testcase("Submit good blobs but insufficient funds"); - - using namespace jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - - auto const alice = Account("alice"); - auto const bob = Account("bob"); - // fund 1000 (TestData::fund/10) XRP, the transaction sends 5000 - // (TestData::fund/2) XRP, so insufficient funds - env.fund(XRP(TestData::fund / 10), "alice", "bob"); - env.trust(bob["USD"](TestData::fund), alice); - env.close(); - - { - SubmitClient client(grpcPort); - client.request.set_signed_transaction(testData.xrpTxBlob); - client.SubmitTransaction(); - if (!BEAST_EXPECT(client.status.ok())) - { - return; - } - BEAST_EXPECT( - client.reply.engine_result().result() == "tecUNFUNDED_PAYMENT"); - BEAST_EXPECT(client.reply.engine_result_code() == 104); - } - } - - void - run() override - { - fillTestData(); - testSubmitGoodBlobGrpc(); - testSubmitErrorBlobGrpc(); - testSubmitInsufficientFundsGrpc(); - } -}; - -BEAST_DEFINE_TESTSUITE(Submit, app, ripple); - -} // namespace test -} // namespace ripple diff --git a/src/test/rpc/Transaction_test.cpp b/src/test/rpc/Transaction_test.cpp index a20a20aa6..08e97c1c2 100644 --- a/src/test/rpc/Transaction_test.cpp +++ b/src/test/rpc/Transaction_test.cpp @@ -17,7 +17,7 @@ */ //============================================================================== -#include +#include #include #include #include @@ -119,8 +119,7 @@ class Transaction_test : public beast::unit_test::suite const auto deletedLedger = (startLegSeq + endLegSeq) / 2; { // Remove one of the ledgers from the database directly - dynamic_cast( - &env.app().getRelationalDBInterface()) + dynamic_cast(&env.app().getRelationalDatabase()) ->deleteTransactionByLedgerSeq(deletedLedger); } diff --git a/src/test/rpc/Tx_test.cpp b/src/test/rpc/Tx_test.cpp deleted file mode 100644 index e4e0507b9..000000000 --- a/src/test/rpc/Tx_test.cpp +++ /dev/null @@ -1,830 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2020 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -namespace ripple { -namespace test { - -class Tx_test : public beast::unit_test::suite -{ - template - std::string - toByteString(T const& data) - { - const char* bytes = reinterpret_cast(data.data()); - return {bytes, data.size()}; - } - - void - cmpAmount( - const org::xrpl::rpc::v1::CurrencyAmount& proto_amount, - STAmount amount) - { - if (amount.native()) - { - if (!BEAST_EXPECT(proto_amount.has_xrp_amount())) - return; - BEAST_EXPECT( - proto_amount.xrp_amount().drops() == amount.xrp().drops()); - } - else - { - if (!BEAST_EXPECT(proto_amount.has_issued_currency_amount())) - return; - - org::xrpl::rpc::v1::IssuedCurrencyAmount issuedCurrency = - proto_amount.issued_currency_amount(); - Issue const& issue = amount.issue(); - Currency currency = issue.currency; - BEAST_EXPECT( - issuedCurrency.currency().name() == to_string(currency)); - BEAST_EXPECT( - issuedCurrency.currency().code() == toByteString(currency)); - BEAST_EXPECT(issuedCurrency.value() == to_string(amount.iou())); - BEAST_EXPECT( - issuedCurrency.issuer().address() == toBase58(issue.account)); - } - } - - void - cmpPaymentTx( - const org::xrpl::rpc::v1::Transaction& proto, - std::shared_ptr txnSt) - { - if (!BEAST_EXPECT(proto.has_payment())) - return; - - if (!BEAST_EXPECT( - safe_cast(txnSt->getFieldU16(sfTransactionType)) == - TxType::ttPAYMENT)) - return; - - AccountID account = txnSt->getAccountID(sfAccount); - - if (!BEAST_EXPECT(proto.has_account())) - return; - BEAST_EXPECT(proto.account().value().address() == toBase58(account)); - - STAmount amount = txnSt->getFieldAmount(sfAmount); - if (!BEAST_EXPECT(proto.payment().has_amount())) - return; - cmpAmount(proto.payment().amount().value(), amount); - - AccountID accountDest = txnSt->getAccountID(sfDestination); - if (!BEAST_EXPECT(proto.payment().has_destination())) - return; - BEAST_EXPECT( - proto.payment().destination().value().address() == - toBase58(accountDest)); - - STAmount fee = txnSt->getFieldAmount(sfFee); - if (!BEAST_EXPECT(proto.has_fee())) - return; - BEAST_EXPECT(proto.fee().drops() == fee.xrp().drops()); - - if (!BEAST_EXPECT(proto.has_sequence())) - return; - BEAST_EXPECT( - proto.sequence().value() == txnSt->getFieldU32(sfSequence)); - - if (!BEAST_EXPECT(proto.has_signing_public_key())) - return; - - Blob signingPubKey = txnSt->getFieldVL(sfSigningPubKey); - BEAST_EXPECT( - proto.signing_public_key().value() == toByteString(signingPubKey)); - - if (txnSt->isFieldPresent(sfFlags)) - { - if (!BEAST_EXPECT(proto.has_flags())) - return; - BEAST_EXPECT(proto.flags().value() == txnSt->getFieldU32(sfFlags)); - } - else - { - BEAST_EXPECT(!proto.has_flags()); - } - - if (txnSt->isFieldPresent(sfLastLedgerSequence)) - { - if (!BEAST_EXPECT(proto.has_last_ledger_sequence())) - return; - - BEAST_EXPECT( - proto.last_ledger_sequence().value() == - txnSt->getFieldU32(sfLastLedgerSequence)); - } - else - { - BEAST_EXPECT(!proto.has_last_ledger_sequence()); - } - - if (txnSt->isFieldPresent(sfTxnSignature)) - { - if (!BEAST_EXPECT(proto.has_transaction_signature())) - return; - - Blob blob = txnSt->getFieldVL(sfTxnSignature); - BEAST_EXPECT( - proto.transaction_signature().value() == toByteString(blob)); - } - - if (txnSt->isFieldPresent(sfSendMax)) - { - if (!BEAST_EXPECT(proto.payment().has_send_max())) - return; - STAmount const& send_max = txnSt->getFieldAmount(sfSendMax); - cmpAmount(proto.payment().send_max().value(), send_max); - } - else - { - BEAST_EXPECT(!proto.payment().has_send_max()); - } - - if (txnSt->isFieldPresent(sfAccountTxnID)) - { - if (!BEAST_EXPECT(proto.has_account_transaction_id())) - return; - auto field = txnSt->getFieldH256(sfAccountTxnID); - BEAST_EXPECT( - proto.account_transaction_id().value() == toByteString(field)); - } - else - { - BEAST_EXPECT(!proto.has_account_transaction_id()); - } - - if (txnSt->isFieldPresent(sfSourceTag)) - { - if (!BEAST_EXPECT(proto.has_source_tag())) - return; - BEAST_EXPECT( - proto.source_tag().value() == txnSt->getFieldU32(sfSourceTag)); - } - else - { - BEAST_EXPECT(!proto.has_source_tag()); - } - - if (txnSt->isFieldPresent(sfDestinationTag)) - { - if (!BEAST_EXPECT(proto.payment().has_destination_tag())) - return; - - BEAST_EXPECT( - proto.payment().destination_tag().value() == - txnSt->getFieldU32(sfDestinationTag)); - } - else - { - BEAST_EXPECT(!proto.payment().has_destination_tag()); - } - - if (txnSt->isFieldPresent(sfInvoiceID)) - { - if (!BEAST_EXPECT(proto.payment().has_invoice_id())) - return; - - auto field = txnSt->getFieldH256(sfInvoiceID); - BEAST_EXPECT( - proto.payment().invoice_id().value() == toByteString(field)); - } - else - { - BEAST_EXPECT(!proto.payment().has_invoice_id()); - } - - if (txnSt->isFieldPresent(sfDeliverMin)) - { - if (!BEAST_EXPECT(proto.payment().has_deliver_min())) - return; - STAmount const& deliverMin = txnSt->getFieldAmount(sfDeliverMin); - cmpAmount(proto.payment().deliver_min().value(), deliverMin); - } - else - { - BEAST_EXPECT(!proto.payment().has_deliver_min()); - } - - STPathSet const& pathset = txnSt->getFieldPathSet(sfPaths); - if (!BEAST_EXPECT(pathset.size() == proto.payment().paths_size())) - return; - - int ind = 0; - for (auto it = pathset.begin(); it < pathset.end(); ++it) - { - STPath const& path = *it; - - const org::xrpl::rpc::v1::Payment_Path& protoPath = - proto.payment().paths(ind++); - if (!BEAST_EXPECT(protoPath.elements_size() == path.size())) - continue; - - int ind2 = 0; - for (auto it2 = path.begin(); it2 != path.end(); ++it2) - { - const org::xrpl::rpc::v1::Payment_PathElement& protoElement = - protoPath.elements(ind2++); - STPathElement const& elt = *it2; - - if (elt.isOffer()) - { - if (elt.hasCurrency()) - { - Currency const& currency = elt.getCurrency(); - if (BEAST_EXPECT(protoElement.has_currency())) - { - BEAST_EXPECT( - protoElement.currency().name() == - to_string(currency)); - } - } - else - { - BEAST_EXPECT(!protoElement.has_currency()); - } - if (elt.hasIssuer()) - { - AccountID const& issuer = elt.getIssuerID(); - if (BEAST_EXPECT(protoElement.has_issuer())) - { - BEAST_EXPECT( - protoElement.issuer().address() == - toBase58(issuer)); - } - } - else - { - BEAST_EXPECT(!protoElement.has_issuer()); - } - } - else - { - if (BEAST_EXPECT(protoElement.has_account())) - { - AccountID const& path_account = elt.getAccountID(); - BEAST_EXPECT( - protoElement.account().address() == - toBase58(path_account)); - } - else - { - BEAST_EXPECT(!protoElement.has_account()); - } - - BEAST_EXPECT(!protoElement.has_issuer()); - BEAST_EXPECT(!protoElement.has_currency()); - } - } - } - - if (txnSt->isFieldPresent(sfMemos)) - { - auto arr = txnSt->getFieldArray(sfMemos); - if (BEAST_EXPECT(proto.memos_size() == arr.size())) - { - for (size_t i = 0; i < arr.size(); ++i) - { - auto protoMemo = proto.memos(i); - auto stMemo = arr[i]; - - if (stMemo.isFieldPresent(sfMemoData)) - { - if (BEAST_EXPECT(protoMemo.has_memo_data())) - { - BEAST_EXPECT( - protoMemo.memo_data().value() == - toByteString(stMemo.getFieldVL(sfMemoData))); - } - } - else - { - BEAST_EXPECT(!protoMemo.has_memo_data()); - } - - if (stMemo.isFieldPresent(sfMemoType)) - { - if (BEAST_EXPECT(protoMemo.has_memo_type())) - { - BEAST_EXPECT( - protoMemo.memo_type().value() == - toByteString(stMemo.getFieldVL(sfMemoType))); - } - } - else - { - BEAST_EXPECT(!protoMemo.has_memo_type()); - } - - if (stMemo.isFieldPresent(sfMemoFormat)) - { - if (BEAST_EXPECT(protoMemo.has_memo_format())) - { - BEAST_EXPECT( - protoMemo.memo_format().value() == - toByteString(stMemo.getFieldVL(sfMemoFormat))); - } - } - else - { - BEAST_EXPECT(!protoMemo.has_memo_format()); - } - } - } - } - else - { - BEAST_EXPECT(proto.memos_size() == 0); - } - - if (txnSt->isFieldPresent(sfSigners)) - { - auto arr = txnSt->getFieldArray(sfSigners); - if (BEAST_EXPECT(proto.signers_size() == arr.size())) - { - for (size_t i = 0; i < arr.size(); ++i) - { - auto protoSigner = proto.signers(i); - auto stSigner = arr[i]; - - if (stSigner.isFieldPresent(sfAccount)) - { - if (BEAST_EXPECT(protoSigner.has_account())) - { - BEAST_EXPECT( - protoSigner.account().value().address() == - toBase58(stSigner.getAccountID(sfAccount))); - } - } - else - { - BEAST_EXPECT(!protoSigner.has_account()); - } - - if (stSigner.isFieldPresent(sfTxnSignature)) - { - if (BEAST_EXPECT( - protoSigner.has_transaction_signature())) - { - Blob blob = stSigner.getFieldVL(sfTxnSignature); - BEAST_EXPECT( - protoSigner.transaction_signature().value() == - toByteString(blob)); - } - } - else - { - BEAST_EXPECT(!protoSigner.has_transaction_signature()); - } - - if (stSigner.isFieldPresent(sfSigningPubKey)) - { - if (BEAST_EXPECT(protoSigner.has_signing_public_key())) - { - Blob signingPubKey = - stSigner.getFieldVL(sfSigningPubKey); - BEAST_EXPECT( - protoSigner.signing_public_key().value() == - toByteString(signingPubKey)); - } - } - else - { - BEAST_EXPECT(!protoSigner.has_signing_public_key()); - } - } - } - } - else - { - BEAST_EXPECT(proto.signers_size() == 0); - } - } - - void - cmpMeta( - const org::xrpl::rpc::v1::Meta& proto, - std::shared_ptr txMeta) - { - BEAST_EXPECT(proto.transaction_index() == txMeta->getIndex()); - BEAST_EXPECT( - proto.transaction_result().result() == - transToken(txMeta->getResultTER())); - - org::xrpl::rpc::v1::TransactionResult r; - - RPC::convert(r, txMeta->getResultTER()); - - BEAST_EXPECT( - proto.transaction_result().result_type() == r.result_type()); - } - - void - cmpDeliveredAmount( - const org::xrpl::rpc::v1::Meta& meta, - const org::xrpl::rpc::v1::Transaction& txn, - const std::shared_ptr expMeta, - const std::shared_ptr expTxn, - bool checkAmount = true) - { - if (expMeta->hasDeliveredAmount()) - { - if (!BEAST_EXPECT(meta.has_delivered_amount())) - return; - cmpAmount( - meta.delivered_amount().value(), expMeta->getDeliveredAmount()); - } - else - { - if (expTxn->isFieldPresent(sfAmount)) - { - using namespace std::chrono_literals; - if (checkAmount) - { - cmpAmount( - meta.delivered_amount().value(), - expTxn->getFieldAmount(sfAmount)); - } - } - else - { - BEAST_EXPECT(!meta.has_delivered_amount()); - } - } - } - - // gRPC stuff - class GrpcTxClient : public GRPCTestClientBase - { - public: - org::xrpl::rpc::v1::GetTransactionRequest request; - org::xrpl::rpc::v1::GetTransactionResponse reply; - - explicit GrpcTxClient(std::string const& port) - : GRPCTestClientBase(port) - { - } - - void - Tx() - { - status = stub_->GetTransaction(&context, request, &reply); - } - }; - - class GrpcAccountTxClient : public GRPCTestClientBase - { - public: - org::xrpl::rpc::v1::GetAccountTransactionHistoryRequest request; - org::xrpl::rpc::v1::GetAccountTransactionHistoryResponse reply; - - explicit GrpcAccountTxClient(std::string const& port) - : GRPCTestClientBase(port) - { - } - - void - AccountTx() - { - status = - stub_->GetAccountTransactionHistory(&context, request, &reply); - } - }; - - void - testTxGrpc() - { - testcase("Test Tx Grpc"); - - using namespace test::jtx; - std::unique_ptr config = envconfig(addGrpcConfig); - std::string grpcPort = *(*config)["port_grpc"].get("port"); - Env env(*this, std::move(config)); - - using namespace std::chrono_literals; - // Set time to this value (or greater) to get delivered_amount in meta - env.timeKeeper().set(NetClock::time_point{446000001s}); - - auto grpcTx = [&grpcPort](auto hash, auto binary) { - GrpcTxClient client(grpcPort); - client.request.set_hash(&hash, sizeof(hash)); - client.request.set_binary(binary); - client.Tx(); - return std::pair( - client.status.ok(), client.reply); - }; - - Account A1{"A1"}; - Account A2{"A2"}; - Account A3{"A3"}; - env.fund(XRP(10000), A1); - env.fund(XRP(10000), A2); - env.close(); - env.trust(A2["USD"](1000), A1); - env.close(); - env(fset(A2, 5)); // set asfAccountTxnID flag - - // SignerListSet - env(signers(A2, 1, {{"bogie", 1}, {"demon", 1}, {A1, 1}, {A3, 1}}), - sig(A2)); - env.close(); - std::vector> txns; - auto const startLegSeq = env.current()->info().seq; - - uint256 prevHash; - for (int i = 0; i < 14; ++i) - { - auto const baseFee = env.current()->fees().base; - auto txfee = fee(i + (2 * baseFee)); - auto lls = last_ledger_seq(i + startLegSeq + 20); - auto dsttag = dtag(i * 456); - auto srctag = stag(i * 321); - auto sm = sendmax(A2["USD"](1000)); - auto dm = delivermin(A2["USD"](50)); - auto txf = txflags(131072); // partial payment flag - auto txnid = account_txn_id(prevHash); - auto inv = invoice_id(prevHash); - auto mem1 = memo("foo", "bar", "baz"); - auto mem2 = memo("dragons", "elves", "goblins"); - - if (i & 1) - { - if (i & 2) - { - env(pay(A2, A1, A2["USD"](100)), - txfee, - srctag, - dsttag, - lls, - sm, - dm, - txf, - txnid, - inv, - mem1, - mem2, - sig(A2)); - } - else - { - env(pay(A2, A1, A2["USD"](100)), - txfee, - srctag, - dsttag, - lls, - sm, - dm, - txf, - txnid, - inv, - mem1, - mem2, - msig(A3)); - } - } - else - { - if (i & 2) - { - env(pay(A2, A1, A2["XRP"](200)), - txfee, - srctag, - dsttag, - lls, - txnid, - inv, - mem1, - mem2, - sig(A2)); - } - else - { - env(pay(A2, A1, A2["XRP"](200)), - txfee, - srctag, - dsttag, - lls, - txnid, - inv, - mem1, - mem2, - msig(A3)); - } - } - txns.emplace_back(env.tx()); - prevHash = txns.back()->getTransactionID(); - env.close(); - } - - // Payment with Paths - auto const gw = Account("gateway"); - auto const USD = gw["USD"]; - env.fund(XRP(10000), "alice", "bob", gw); - env.trust(USD(600), "alice"); - env.trust(USD(700), "bob"); - env(pay(gw, "alice", USD(70))); - txns.emplace_back(env.tx()); - env.close(); - env(pay(gw, "bob", USD(50))); - txns.emplace_back(env.tx()); - env.close(); - env(pay("alice", "bob", Account("bob")["USD"](5)), path(gw)); - txns.emplace_back(env.tx()); - env.close(); - - auto const endLegSeq = env.closed()->info().seq; - - // Find the existing transactions - auto& ledgerMaster = env.app().getLedgerMaster(); - int index = startLegSeq; - for (auto&& tx : txns) - { - auto id = tx->getTransactionID(); - auto ledger = ledgerMaster.getLedgerBySeq(index); - - for (bool b : {false, true}) - { - auto const result = grpcTx(id, b); - - BEAST_EXPECT(result.first == true); - BEAST_EXPECT(result.second.ledger_index() == index); - BEAST_EXPECT(result.second.validated() == true); - if (b) - { - Serializer s = tx->getSerializer(); - BEAST_EXPECT( - result.second.transaction_binary() == toByteString(s)); - } - else - { - cmpPaymentTx(result.second.transaction(), tx); - } - - if (!ledger || b) - continue; - - auto rawMeta = ledger->txRead(id).second; - if (!rawMeta) - continue; - - auto txMeta = - std::make_shared(id, ledger->seq(), *rawMeta); - - cmpMeta(result.second.meta(), txMeta); - cmpDeliveredAmount( - result.second.meta(), - result.second.transaction(), - txMeta, - tx); - - auto grpcAccountTx = [&grpcPort]( - uint256 const& id, - bool binary, - AccountID const& account) - -> std:: - pair { - GrpcAccountTxClient client(grpcPort); - client.request.set_binary(binary); - client.request.mutable_account()->set_address( - toBase58(account)); - client.AccountTx(); - org::xrpl::rpc::v1::GetTransactionResponse res; - - for (auto const& tx : client.reply.transactions()) - { - if (uint256::fromVoid(tx.hash().data()) == id) - { - return {client.status.ok(), tx}; - } - } - return {false, res}; - }; - - // Compare result to result from account_tx - auto mentioned = tx->getMentionedAccounts(); - - if (!BEAST_EXPECT(mentioned.size())) - continue; - - auto account = *mentioned.begin(); - auto const accountTxResult = grpcAccountTx(id, b, account); - - if (!BEAST_EXPECT(accountTxResult.first)) - continue; - - cmpPaymentTx(accountTxResult.second.transaction(), tx); - cmpMeta(accountTxResult.second.meta(), txMeta); - cmpDeliveredAmount( - accountTxResult.second.meta(), - accountTxResult.second.transaction(), - txMeta, - tx); - } - index++; - } - - // Find not existing transaction - auto const tx = env.jt(noop(A1), seq(env.seq(A1))).stx; - for (bool b : {false, true}) - { - auto const result = grpcTx(tx->getTransactionID(), b); - - BEAST_EXPECT(result.first == false); - } - - // Delete one transaction - const auto deletedLedger = (startLegSeq + endLegSeq) / 2; - { - // Remove one of the ledgers from the database directly - dynamic_cast( - &env.app().getRelationalDBInterface()) - ->deleteTransactionByLedgerSeq(deletedLedger); - } - - for (bool b : {false, true}) - { - auto const result = grpcTx(tx->getTransactionID(), b); - - BEAST_EXPECT(result.first == false); - } - - // non final transaction - env(pay(A2, A1, A2["XRP"](200))); - auto res = grpcTx(env.tx()->getTransactionID(), false); - BEAST_EXPECT(res.first); - BEAST_EXPECT(res.second.has_transaction()); - if (!BEAST_EXPECT(res.second.has_meta())) - return; - if (!BEAST_EXPECT(res.second.meta().has_transaction_result())) - return; - - BEAST_EXPECT( - res.second.meta().transaction_result().result() == "tesSUCCESS"); - BEAST_EXPECT( - res.second.meta().transaction_result().result_type() == - org::xrpl::rpc::v1::TransactionResult::RESULT_TYPE_TES); - BEAST_EXPECT(!res.second.validated()); - BEAST_EXPECT(!res.second.meta().has_delivered_amount()); - env.close(); - - res = grpcTx(env.tx()->getTransactionID(), false); - BEAST_EXPECT(res.first); - BEAST_EXPECT(res.second.has_transaction()); - if (!BEAST_EXPECT(res.second.has_meta())) - return; - if (!BEAST_EXPECT(res.second.meta().has_transaction_result())) - return; - - BEAST_EXPECT( - res.second.meta().transaction_result().result() == "tesSUCCESS"); - BEAST_EXPECT( - res.second.meta().transaction_result().result_type() == - org::xrpl::rpc::v1::TransactionResult::RESULT_TYPE_TES); - BEAST_EXPECT(res.second.validated()); - BEAST_EXPECT(res.second.meta().has_delivered_amount()); - } - -public: - void - run() override - { - testTxGrpc(); - } -}; - -BEAST_DEFINE_TESTSUITE(Tx, app, ripple); -} // namespace test -} // namespace ripple diff --git a/src/test/server/Server_test.cpp b/src/test/server/Server_test.cpp index 35f9149ca..b5eb71f36 100644 --- a/src/test/server/Server_test.cpp +++ b/src/test/server/Server_test.cpp @@ -299,7 +299,6 @@ public: serverPort.back().port = 0; serverPort.back().protocol.insert("http"); auto eps = s->ports(serverPort); - log << "server listening on port " << eps[0].port() << std::endl; test_request(eps[0]); test_keepalive(eps[0]); // s->close(); diff --git a/src/test/shamap/SHAMapSync_test.cpp b/src/test/shamap/SHAMapSync_test.cpp index ba32f6e80..6b2648a96 100644 --- a/src/test/shamap/SHAMapSync_test.cpp +++ b/src/test/shamap/SHAMapSync_test.cpp @@ -184,7 +184,6 @@ public: BEAST_EXPECT(source.deepCompare(destination)); - log << "Checking destination invariants..." << std::endl; destination.invariants(); } }; diff --git a/src/test/unit_test/FileDirGuard.h b/src/test/unit_test/FileDirGuard.h index 6337365f0..3c79fb11b 100644 --- a/src/test/unit_test/FileDirGuard.h +++ b/src/test/unit_test/FileDirGuard.h @@ -86,9 +86,6 @@ public: if (rmSubDir_) rmDir(subDir_); - else - test_.log << "Skipping rm dir: " << subDir_.string() - << std::endl; } catch (std::exception& e) { diff --git a/src/test/unit_test/multi_runner.cpp b/src/test/unit_test/multi_runner.cpp index 5cd69111b..c1111e4cf 100644 --- a/src/test/unit_test/multi_runner.cpp +++ b/src/test/unit_test/multi_runner.cpp @@ -389,11 +389,6 @@ multi_runner_base::add_failures(std::size_t failures) any_failed(failures != 0); } -template -constexpr const char* multi_runner_base::shared_mem_name_; -template -constexpr const char* multi_runner_base::message_queue_name_; - } // namespace detail //------------------------------------------------------------------------------