Compare commits

..

1 Commits

Author SHA1 Message Date
Luc des Trois Maisons
56ba9e3a52 fix: add missing value_type to JSON iterators
ValueConstIterator and ValueIterator lacked the value_type typedef
required for std::iterator_traits deduction in C++20. GCC 13's STL
correctly diagnoses this when the iterators are used with algorithms
like std::all_of.

Fixes build on GCC 13 without requiring GCC 15.
2026-07-21 15:30:41 -04:00
55 changed files with 1227 additions and 1822 deletions

View File

@@ -216,7 +216,6 @@ words:
- Nyffenegger
- onlatest
- ostr
- oxalica
- pargs
- partitioner
- paychan
@@ -263,8 +262,6 @@ words:
- Rohrs
- roundings
- rustc
- rustfmt
- rustup
- sahyadri
- Satoshi
- scons
@@ -308,7 +305,6 @@ words:
- takerpays
- ters
- TMEndpointv2
- toolchain
- tparam
- trixie
- tx
@@ -341,7 +337,6 @@ words:
- unsquelch
- unsquelched
- unsquelching
- unsuffixed
- unvalidated
- unveto
- unvetoed

View File

@@ -11,9 +11,6 @@ endfunction()
function(create_symbolic_link target link)
endfunction()
function(xrpl_add_benchmark name)
endfunction()
macro(exclude_from_default target_)
endmacro()

View File

@@ -1,10 +1,10 @@
<!--
This PR template helps you write a good pull request description.
This PR template helps you to write a good pull request description.
Please feel free to include additional useful information even beyond what is requested below.
If your branch is on a personal fork and has a name that allows it to
run CI build/test jobs (e.g. "ci/foo"), remember to rename it BEFORE
opening the PR. This avoids redundant test runs. Renaming
opening the PR. This avoids unnecessary redundant test runs. Renaming
the branch after opening the PR will close the PR.
https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-branches-in-your-repository/renaming-a-branch
-->
@@ -15,7 +15,7 @@ https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-
Please include a summary of the changes.
This may be a direct input to the release notes.
If too broad, please consider splitting into multiple PRs.
If there is a relevant task or issue, please link it here.
If a relevant task or issue, please link it here.
-->
### Context of Change
@@ -65,5 +65,5 @@ This section may not be needed if your change includes thoroughly commented unit
<!--
## Future Tasks
For future tasks related to this PR.
For future tasks related to PR.
-->

View File

@@ -1,6 +1,3 @@
benchmarks.libxrpl > xrpl.basics
benchmarks.libxrpl > xrpl.config
benchmarks.libxrpl > xrpl.nodestore
libxrpl.basics > xrpl.basics
libxrpl.conditions > xrpl.basics
libxrpl.conditions > xrpl.conditions

View File

@@ -1,5 +1,5 @@
{
"image_tag": "sha-2e25435",
"image_tag": "sha-e29b523",
"configs": {
"ubuntu": [
{

View File

@@ -1,4 +1,4 @@
name: Build `nix` Docker images
name: Build Nix Docker images
on:
push:
@@ -8,7 +8,6 @@ on:
- ".github/workflows/build-nix-images.yml"
- "flake.nix"
- "flake.lock"
- "rust-toolchain.toml"
- "nix/**"
- "!nix/docker/README.md"
- "!nix/devshell.nix"
@@ -19,7 +18,6 @@ on:
- ".github/workflows/build-nix-images.yml"
- "flake.nix"
- "flake.lock"
- "rust-toolchain.toml"
- "nix/**"
- "!nix/docker/README.md"
- "!nix/devshell.nix"
@@ -38,7 +36,7 @@ defaults:
jobs:
build-merge:
name: Build and push `nix-${{ matrix.distro.name }}` image
name: Build and push nix-${{ matrix.distro.name }}
permissions:
contents: read
packages: write
@@ -56,7 +54,7 @@ jobs:
base_image: debian:bookworm
- name: rhel
base_image: registry.access.redhat.com/ubi9/ubi:latest
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@ee03d31bcc4501d7599dc1b1ecd7a34af582ad1c
with:
image_name: xrpld/nix-${{ matrix.distro.name }}
dockerfile: nix/docker/Dockerfile

View File

@@ -1,4 +1,4 @@
name: Build `packaging` Docker images
name: Build packaging Docker images
on:
push:
@@ -26,7 +26,7 @@ defaults:
jobs:
build-merge:
name: Build and push `packaging-${{ matrix.distro.name }}` image
name: Build and push packaging-${{ matrix.distro.name }}
permissions:
contents: read
packages: write
@@ -38,7 +38,7 @@ jobs:
base_image: debian:bookworm
- name: rhel
base_image: registry.access.redhat.com/ubi9/ubi:latest
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@ee03d31bcc4501d7599dc1b1ecd7a34af582ad1c
with:
image_name: xrpld/packaging-${{ matrix.distro.name }}
dockerfile: package/Dockerfile

View File

@@ -1,38 +0,0 @@
name: Build `pre-commit` Docker image
on:
push:
branches:
- develop
paths:
- ".github/workflows/build-pre-commit-image.yml"
- "bin/pre-commit/Dockerfile"
- "rust-toolchain.toml"
pull_request:
paths:
- ".github/workflows/build-pre-commit-image.yml"
- "bin/pre-commit/Dockerfile"
- "rust-toolchain.toml"
workflow_dispatch:
concurrency:
# Read `on-trigger.yml` for the rationale behind this concurrency group name.
group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' && github.sha || github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash
jobs:
build-merge:
name: Build and push `pre-commit` image
permissions:
contents: read
packages: write
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32
with:
image_name: xrpld/pre-commit
dockerfile: bin/pre-commit/Dockerfile
base_image: ubuntu:26.04
push: ${{ github.event_name == 'push' }}

View File

@@ -20,4 +20,4 @@ on:
jobs:
check_title:
if: ${{ github.event.pull_request.draft != true }}
uses: XRPLF/actions/.github/workflows/check-pr-title.yml@ed23185d25693c9a724c3d05ae4a58371b0f72b8
uses: XRPLF/actions/.github/workflows/check-pr-title.yml@cba1f0891650baf1a9c88624dc2d72573be2eb81

View File

@@ -14,7 +14,7 @@ on:
jobs:
# Call the workflow in the XRPLF/actions repo that runs the pre-commit hooks.
run-hooks:
uses: XRPLF/actions/.github/workflows/pre-commit.yml@3ba08d6ddf114092891d48491fc2e26c3ba15552
uses: XRPLF/actions/.github/workflows/pre-commit.yml@1bde119a1ab71305ba5d3716e7a82cea1c7bdede
with:
runs_on: ubuntu-latest
container: '{ "image": "ghcr.io/xrplf/xrpld/pre-commit:sha-f56b79f" }'
container: '{ "image": "ghcr.io/xrplf/ci/tools-rippled-pre-commit:sha-41ec7c1" }'

View File

@@ -41,13 +41,13 @@ env:
jobs:
build:
runs-on: ubuntu-latest
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-2e25435
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-e29b523
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Prepare runner
uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606
uses: XRPLF/actions/prepare-runner@64ec3cf3b152b4444638f470bbd6df7a7a30c81c
with:
enable_ccache: false

View File

@@ -113,7 +113,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Prepare runner
uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606
uses: XRPLF/actions/prepare-runner@64ec3cf3b152b4444638f470bbd6df7a7a30c81c
with:
enable_ccache: ${{ inputs.ccache_enabled }}
@@ -223,13 +223,11 @@ jobs:
BUILD_TYPE: ${{ inputs.build_type }}
CMAKE_TARGET: ${{ inputs.cmake_target }}
run: |
set -o pipefail
cmake \
--build . \
--config "${BUILD_TYPE}" \
--parallel "${BUILD_NPROC}" \
--target "${CMAKE_TARGET}" \
2>&1 | tee "${GITHUB_WORKSPACE}/build.log"
--target "${CMAKE_TARGET}"
- name: Show ccache statistics
if: ${{ inputs.ccache_enabled }}
@@ -324,46 +322,27 @@ jobs:
PRELOAD=""
fi
LD_PRELOAD="$PRELOAD" ./xrpld --unittest --unittest-jobs "${BUILD_NPROC}" 2>&1 | tee "${GITHUB_WORKSPACE}/unittest.log"
LD_PRELOAD="$PRELOAD" ./xrpld --unittest --unittest-jobs "${BUILD_NPROC}" 2>&1 | tee unittest.log
# Smoke-run every benchmark module with a single repetition to confirm the
# benchmarks still build and execute. This is a correctness check, not a
# performance measurement, so it is skipped for instrumented builds
# (sanitizers/coverage/voidstar), where it would be slow and meaningless,
# and on Windows, where the `install` target does not build them.
- name: Run the benchmarks
if: ${{ !inputs.build_only && runner.os != 'Windows' && env.SANITIZERS_ENABLED == 'false' && env.COVERAGE_ENABLED != 'true' && env.VOIDSTAR_ENABLED != 'true' }}
working-directory: ${{ env.BUILD_DIR }}
- name: Show test failure summary
if: ${{ failure() && !inputs.build_only }}
env:
WORKING_DIR: ${{ runner.os == 'Windows' && format('{0}\{1}', env.BUILD_DIR, inputs.build_type) || env.BUILD_DIR }}
run: |
rc=0
while IFS= read -r bench; do
echo "::group::${bench}"
"./${bench}" --benchmark_repetitions=1 || rc=1
echo "::endgroup::"
done < <(find src/benchmarks -type f -perm -u+x -name 'xrpl.bench.*')
exit "${rc}"
if [ ! -d "${WORKING_DIR}" ]; then
echo "Working directory '${WORKING_DIR}' does not exist."
exit 0
fi
- name: Show build/test failure summary
if: ${{ failure() }}
run: |
cd "${GITHUB_WORKSPACE}"
cd "${WORKING_DIR}"
if [ -f unittest.log ]; then
if ! grep -E "failed" unittest.log | grep -vE "^I[0-9]|^[0-9]+> (ERR:|FTL:)"; then
echo "unittest.log present but no failure lines found."
fi
elif [ -f build.log ]; then
# GCC/Clang emit "error:" (covers "fatal error:"); MSVC emits
# "error C####:", "error LNK####:", and "fatal error LNK####:".
# -A6 prints the lines that follow each match (source line, caret,
# notes, and the "N errors generated" tally) to capture the whole
# diagnostic block.
if ! grep -E -A6 "error:|error C[0-9]{4}|error LNK[0-9]{4}|fatal error" build.log; then
echo "build.log present but no compile errors found."
fi
else
echo "unittest.log/build.log not found; something went wrong."
exit 1
if [ ! -f unittest.log ]; then
echo "unittest.log not found; embedded tests may not have run."
exit 0
fi
if ! grep -E "failed" unittest.log; then
echo "Log present but no failure lines found in unittest.log."
fi
- name: Debug failure (Linux)
if: ${{ failure() && runner.os == 'Linux' && !inputs.build_only }}

View File

@@ -34,7 +34,7 @@ jobs:
needs: [determine-files]
if: ${{ needs.determine-files.outputs.cpp_changed_files != '' || needs.determine-files.outputs.need_full_run == 'true' }}
runs-on: ["self-hosted", "Linux", "X64", "heavy"]
container: "ghcr.io/xrplf/xrpld/nix-debian:sha-2e25435"
container: "ghcr.io/xrplf/xrpld/nix-debian:sha-e29b523"
permissions:
contents: read
issues: write
@@ -43,7 +43,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Prepare runner
uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606
uses: XRPLF/actions/prepare-runner@64ec3cf3b152b4444638f470bbd6df7a7a30c81c
with:
enable_ccache: false

View File

@@ -30,7 +30,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.13"

View File

@@ -26,7 +26,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
with:
python-version: "3.13"

View File

@@ -40,7 +40,7 @@ defaults:
jobs:
upload:
runs-on: ubuntu-latest
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-2e25435
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-e29b523
env:
REMOTE_NAME: ${{ inputs.remote_name }}
CONAN_LOGIN_USERNAME_XRPLF: ${{ secrets.remote_username }}

View File

@@ -68,7 +68,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Prepare runner
uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606
uses: XRPLF/actions/prepare-runner@64ec3cf3b152b4444638f470bbd6df7a7a30c81c
with:
enable_ccache: false

3
.gitignore vendored
View File

@@ -81,9 +81,6 @@ DerivedData
# Python
__pycache__
# Rust build artifacts.
target/
# Direnv's directory
/.direnv

View File

@@ -55,7 +55,7 @@ repos:
types_or: [c++, c]
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: f4d7745e17a28aad7eed2f4874ca8d1568c11c4c # frozen: v22.1.8
rev: dd18dad857d6133e90bbe478f4f2f22ec0030269 # frozen: v22.1.5
hooks:
- id: clang-format
args: [--style=file]
@@ -68,7 +68,7 @@ repos:
- id: gersemi
- repo: https://github.com/rbubley/mirrors-prettier
rev: 9337a74165b178ae2c766f60bee7252a0f06f3e8 # frozen: v3.9.5
rev: 39e2973981e6d2f9b6c543b0086a2d2393abdc89 # frozen: v3.9.4
hooks:
- id: prettier
args: [--end-of-line=auto]

View File

@@ -131,10 +131,6 @@ else()
endif()
target_link_libraries(xrpl_libs INTERFACE ${nudb})
if(benchmark)
find_package(benchmark REQUIRED)
endif()
if(coverage)
include(XrplCov)
endif()
@@ -149,7 +145,3 @@ if(tests)
include(CTest)
add_subdirectory(src/tests/libxrpl)
endif()
if(benchmark)
add_subdirectory(src/benchmarks/libxrpl)
endif()

View File

@@ -1,4 +0,0 @@
# By default, anyone can review changes.
# The CI tooling team should review changes to the CI configuration.
/.github/ @XRPLF/ci-tooling

View File

@@ -83,7 +83,6 @@ If you create new source files, they must be organized as follows:
`src/libxrpl`.
- All other non-test files must go under `src/xrpld`.
- All test source files must go under `src/test`.
- All benchmark source files must go under `src/benchmarks`.
The source must be formatted according to the style guide below. The easiest
way to satisfy this is to install the [`pre-commit`](#pre-commit-hooks) hooks,

View File

@@ -1,55 +0,0 @@
ARG BASE_IMAGE=ubuntu:26.04
FROM ${BASE_IMAGE}
SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
ENTRYPOINT ["/bin/bash"]
ARG DEBIAN_FRONTEND=noninteractive
RUN <<EOF
pkgs=()
pkgs+=(curl) # Required to install nix.
pkgs+=(doxygen) # Needed for Clio's check-doxygen-docs.sh.
pkgs+=(git) # Required for prepare-runner.
pkgs+=(libatomic1) # Required to run pre-commit provided `node`.
pkgs+=(python3) # Python 3 interpreter.
pkgs+=(python3-pip) # Package manager for Python applications.
pkgs+=(xz-utils) # Required to install nix
apt-get update
apt-get install -y --no-install-recommends "${pkgs[@]}"
apt-get clean
rm -rf /var/lib/apt/lists/*
EOF
ARG PRE_COMMIT_VERSION=4.6.0
RUN pip install --no-cache --break-system-packages \
pre-commit==${PRE_COMMIT_VERSION}
RUN sh <(curl --proto '=https' --tlsv1.2 -L https://nixos.org/nix/install) --daemon --yes
# Add nix to PATH and set NIX environment variables,
# so nix is available in all shells including non-interactive shells (e.g., GitHub Actions).
ENV PATH="/nix/var/nix/profiles/default/bin:${PATH}"
ENV NIX_PROFILES="/nix/var/nix/profiles/default"
ENV NIX_SSL_CERT_FILE="/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
# Verify nix installation
RUN nix --version
ENV RUSTUP_HOME="/opt/rust/rustup"
ENV CARGO_HOME="/opt/rust/cargo"
ENV PATH="/opt/rust/cargo/bin:${PATH}"
WORKDIR /tmp
COPY rust-toolchain.toml /tmp/rust-toolchain.toml
RUN <<EOF
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
| sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
rustup toolchain install
rustup show
cargo fmt --version
EOF
WORKDIR /

View File

@@ -1,36 +0,0 @@
include(isolate_headers)
# Define a benchmark executable for the module `name`.
#
# This follows the same general pattern as other build helpers in this repo
# (e.g. `add_module`): create a target and isolate headers, but here the target
# is a benchmark executable and no `add_test(...)` is registered.
#
# `isolate_headers` exposes only `${CMAKE_CURRENT_SOURCE_DIR}/${name}` on the
# include path, rooted at `src`, so a benchmark's own headers are reached as
# `<benchmarks/.../${name}/...>` and nothing else in the tree leaks in.
function(xrpl_add_benchmark name)
set(target ${PROJECT_NAME}.bench.${name})
file(
GLOB_RECURSE sources
CONFIGURE_DEPENDS
"${CMAKE_CURRENT_SOURCE_DIR}/${name}/*.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/${name}.cpp"
)
add_executable(${target} ${ARGN} ${sources})
# Benchmark sources register cases through Google Benchmark's static
# registrars (anonymous-namespace lambdas). Merging several such files into
# one unity translation unit collides those internal-linkage entities, so
# keep benchmarks out of the unity build - mirroring xrpl.libpb in
# XrplCore.cmake. Each file compiles fine on its own.
set_target_properties(${target} PROPERTIES UNITY_BUILD OFF)
isolate_headers(
${target}
"${CMAKE_SOURCE_DIR}/src"
"${CMAKE_CURRENT_SOURCE_DIR}/${name}"
PRIVATE
)
endfunction()

View File

@@ -30,8 +30,6 @@ if(tests)
endif()
endif()
option(benchmark "Build benchmarks" ON)
# Enabled by default so every header is compiled on its own as the main file of
# its own compile_commands.json entry - this is what lets clang-tidy (and clangd
# and IDEs) analyse a header's own includes directly. The per-header objects are

View File

@@ -25,7 +25,6 @@
"c-ares/1.34.6#545240bb1c40e2cacd4362d6b8967650%1782392402.681654",
"bzip2/1.0.8#c470882369c2d95c5c77e970c0c7e321%1782392402.296732",
"boost/1.91.0#ea540ca2133d831b560036aa24dece3c%1782392419.475605",
"benchmark/1.9.5#b885dc73ad67b40a55d45684d1c88ad1%1782736613.864841",
"abseil/20250127.0#9ef01c1451a8340f9022e46238c0fbb6%1783945159.651047"
],
"build_requires": [

View File

@@ -20,22 +20,6 @@ compiler.libcxx={{ detect_api.detect_libcxx(compiler, version, compiler_exe) }}
{% endif %}
[conf]
{# The Boost recipe builds with b2, which doesn't use Conan's toolchain files. #}
{# Instead it hand-rolls the compiler for user-config.jam, #}
{# and its fallback probes a version-suffixed binary (e.g. `g++-15`) before plain `g++`. #}
{# Inside the Nix shell the wrapper only provides `g++`/`gcc` (no `-15` suffix), #}
{# so on a host that also has a system `g++-15` the probe escapes Nix #}
{# and picks the system compiler, which is mismatched with the Nix libraries #}
{# and breaks the build (e.g. Boost.Stacktrace link checks fail). #}
{# Pinning the executables here short-circuits that probe so Boost (and the rest of the toolchain) #}
{# resolve the same compiler. #}
{# Not part of the package ID, so binaries stay shareable. #}
{% if os != "Windows" %}
{% set cc_exe = {"gcc": "gcc", "clang": "clang", "apple-clang": "clang"}.get(compiler) %}
{% set cxx_exe = {"gcc": "g++", "clang": "clang++", "apple-clang": "clang++"}.get(compiler) %}
tools.build:compiler_executables={'c':'{{ cc_exe }}','cpp':'{{ cxx_exe }}'}
{% endif %}
{# By default, Conan tries to reuse binaries built with different cppstd versions. #}
{# We want to avoid that to improve reproduceability, so we add the cppstd version to the package ID. #}
{# More info: https://docs.conan.io/2/reference/extensions/binary_compatibility.html #}

View File

@@ -15,7 +15,6 @@ class Xrpl(ConanFile):
settings = "os", "compiler", "build_type", "arch"
options = {
"assertions": [True, False],
"benchmark": [True, False],
"coverage": [True, False],
"fPIC": [True, False],
"jemalloc": [True, False],
@@ -47,7 +46,6 @@ class Xrpl(ConanFile):
default_options = {
"assertions": False,
"benchmark": True,
"coverage": False,
"fPIC": True,
"jemalloc": False,
@@ -131,8 +129,6 @@ class Xrpl(ConanFile):
self.options["boost"].without_cobalt = True
def requirements(self):
if self.options.benchmark:
self.requires("benchmark/1.9.5")
self.requires("boost/1.91.0", force=True, transitive_headers=True)
self.requires("date/3.0.4", transitive_headers=True)
if self.options.jemalloc:
@@ -166,7 +162,6 @@ class Xrpl(ConanFile):
def generate(self):
tc = CMakeToolchain(self)
tc.variables["tests"] = self.options.tests
tc.variables["benchmark"] = self.options.benchmark
tc.variables["assert"] = self.options.assertions
tc.variables["coverage"] = self.options.coverage
tc.variables["jemalloc"] = self.options.jemalloc

23
flake.lock generated
View File

@@ -36,28 +36,7 @@
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
"nixpkgs-custom-glibc": "nixpkgs-custom-glibc",
"rust-overlay": "rust-overlay"
}
},
"rust-overlay": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1784611586,
"narHash": "sha256-OfqgY+0hp/zseZB7uyH0U8kIDPS4scZZCyAurEplvG0=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "14f58845249f3552a89b07772626b8d3c632fa86",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
"nixpkgs-custom-glibc": "nixpkgs-custom-glibc"
}
}
},

View File

@@ -10,25 +10,12 @@
url = "github:NixOS/nixpkgs/9cd98386a38891d1074fc18036b842dc4416f562";
flake = false;
};
# Pinned Rust toolchains, delivered from the Nix store. Lets the Nix CI
# image and dev shell honour the single `rust-toolchain.toml` pin (shared
# with the rustup-based non-Nix runners) while staying hermetic — the
# toolchain lands in the image's Nix closure and is locked by flake.lock.
rust-overlay = {
url = "github:oxalica/rust-overlay";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs =
{
nixpkgs,
nixpkgs-custom-glibc,
rust-overlay,
...
}:
{ nixpkgs, nixpkgs-custom-glibc, ... }:
let
forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc rust-overlay; };
forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc; };
in
{
devShells = forEachSystem (import ./nix/devshell.nix);

View File

@@ -623,6 +623,7 @@ class ValueConstIterator : public ValueIteratorBase
public:
using size_t = unsigned int;
using difference_type = int;
using value_type = Value const;
using reference = Value const&;
using pointer = Value const*;
using SelfType = ValueConstIterator;
@@ -687,6 +688,7 @@ class ValueIterator : public ValueIteratorBase
public:
using size_t = unsigned int;
using difference_type = int;
using value_type = Value;
using reference = Value&;
using pointer = Value*;
using SelfType = ValueIterator;

View File

@@ -263,11 +263,10 @@ constructLoanState(
Number const& principalOutstanding,
Number const& managementFeeOutstanding);
// Overload of constructLoanState() that reads the three tracked fields
// directly from a Loan ledger object, which always holds rounded values,
// rather than taking them as separate Number arguments.
// Constructs a valid LoanState object from a Loan object, which always has
// rounded values
LoanState
constructLoanState(SLE::const_ref loan);
constructRoundedLoanState(SLE::const_ref loan);
Number
computeManagementFee(

View File

@@ -7,10 +7,8 @@ let
inherit (import ./packages.nix { inherit pkgs; })
commonPackages
gccPackage
gccVersion
llvmPackages
llvmVersion
mkVersionedToolLinks
;
# Underlying compiler toolchains to wrap (versions pinned in packages.nix).
@@ -129,25 +127,6 @@ in
customGcov
customClangForCiEnv
customBinutils
(mkVersionedToolLinks {
name = "gcc";
package = customGcc;
version = gccVersion;
tools = [
"gcc"
"g++"
"cpp"
];
})
(mkVersionedToolLinks {
name = "clang";
package = customClang;
version = llvmVersion;
tools = [
"clang"
"clang++"
];
})
# CA certificate bundle so HTTPS clients (git, curl, conan) can verify
# TLS connections without ca-certificates being installed in the system.
pkgs.cacert

View File

@@ -3,9 +3,7 @@ let
inherit (import ./packages.nix { inherit pkgs; })
commonPackages
gccVersion
llvmVersion
llvmPackages
mkVersionedToolLinks
;
# Plain nixpkgs stdenvs — no custom glibc, unlike ci-env.nix.
@@ -17,8 +15,6 @@ let
{
stdenv,
compilerName,
version ? null,
versionedTools ? [ ],
}:
let
compilerVersion =
@@ -29,15 +25,9 @@ let
echo "Compiler: "
${compilerName} --version
'';
versionedLinks = pkgs.lib.optional (version != null) (mkVersionedToolLinks {
name = compilerName;
package = stdenv.cc;
inherit version;
tools = versionedTools;
});
in
(pkgs.mkShell.override { inherit stdenv; }) {
packages = commonPackages ++ versionedLinks;
packages = commonPackages;
shellHook = ''
echo "Welcome to xrpld development shell";
${compilerVersion}
@@ -51,22 +41,11 @@ rec {
gcc = makeShell {
stdenv = gccStdenv;
compilerName = "gcc";
version = gccVersion;
versionedTools = [
"gcc"
"g++"
"cpp"
];
};
clang = makeShell {
stdenv = clangStdenv;
compilerName = "clang";
version = llvmVersion;
versionedTools = [
"clang"
"clang++"
];
};
# Nix provides no compiler; use the one from your system (e.g. Apple Clang).

View File

@@ -12,7 +12,6 @@ COPY nix/packages.nix /tmp/build/nix/packages.nix
COPY nix/utils.nix /tmp/build/nix/utils.nix
COPY flake.nix /tmp/build/
COPY flake.lock /tmp/build/
COPY rust-toolchain.toml /tmp/build/
WORKDIR /tmp/build
FROM builder-source AS builder

View File

@@ -47,9 +47,7 @@ work without `ca-certificates` being installed in the base image.
[`test_files/cpp/sources/`](./test_files/cpp/sources) with both `g++` and
`clang++`, and sanitizers, and
- compiles the Rust test programs in
[`test_files/rust/sources/`](./test_files/rust/sources) with `rustc`, and
builds the [`test_files/rust/proc_macro/`](./test_files/rust/proc_macro)
workspace with `cargo` to exercise proc-macro dylib loading.
[`test_files/rust/sources/`](./test_files/rust/sources) with `rustc`.
3. **`tester`** — Start again from a clean `BASE_IMAGE` (no Nix toolchain),
install only the sanitizer runtime libraries
([`install-sanitizer-libs.sh`](./install-sanitizer-libs.sh)), and run the
@@ -78,23 +76,20 @@ toolchain being present at runtime. Two pieces make that work:
[`loader-path.sh`](./loader-path.sh) reports the expected loader path for the
current architecture, so we can patch the binaries to use the correct loader.
The build then verifies all of this end to end, and the C++ and Rust programs
go through the same pipeline: each is compiled in `final`, has its `PT_INTERP`
patched to the target loader, and is then run in the clean `tester` stage to
confirm it emits the expected diagnostic on a stock base image. The C++ programs
are in `test_files/cpp/sources/` (a regular binary plus ASan/TSan/UBSan
variants); the Rust programs are in `test_files/rust/sources/` (a hello binary
plus panic and overflow-check variants), plus the `test_files/rust/proc_macro/`
workspace — a crate whose compilation additionally loads a proc-macro dylib, and
whose resulting binary is patched and run like the others.
The build then verifies all of this end to end: the C++ test programs in
`test_files/cpp/sources/` (a regular binary plus ASan/TSan/UBSan variants) and
the Rust test programs in `test_files/rust/sources/` (a hello binary plus panic
and overflow-check variants) are compiled in `final`, their `PT_INTERP` is
patched to the target loader, and they are run in the clean `tester` stage to
confirm each emits the expected diagnostic on a stock base image.
## Files
| File | Purpose |
| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. |
| [`./loader-path.sh`](./loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. |
| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. |
| [`./test_files/rust/`](./test_files/rust) | Rust smoke test: rustc sources + a cargo proc-macro workspace + compile/run scripts. |
| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. |
| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. |
| File | Purpose |
| ----------------------------------------------------------------------- | ----------------------------------------------------------------------------- |
| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. |
| [`./loader-path.sh`](./loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. |
| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. |
| [`./test_files/rust/`](./test_files/rust) | Rust rustc smoke test: sources + compile/run scripts. |
| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. |
| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. |

View File

@@ -40,29 +40,6 @@ compile hello
compile panic
compile overflow "-C overflow-checks=on"
function compile_proc_macro() {
local proj="${src_dir}/../proc_macro"
echo "=== Building proc-macro workspace (cargo) ==="
cargo build --manifest-path "${proj}/Cargo.toml" --offline
local built="${proj}/target/debug/test_macro"
if [ ! -f "${built}" ]; then
echo "ERROR: built test_macro binary not found at ${built}" >&2
exit 1
fi
local binary="${dst_dir}/proc_macro"
cp "${built}" "${binary}"
echo "=== Patching ${binary} to use ${loader} as PT_INTERP ==="
patchelf --set-interpreter "${loader}" --remove-rpath "${binary}"
rm -rf "${proj}/target"
}
compile_proc_macro
echo "=== All binaries compiled ==="
ls -la "${dst_dir}"

View File

@@ -1,14 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "echo_macro"
version = "0.0.0"
[[package]]
name = "test_macro"
version = "0.0.0"
dependencies = [
"echo_macro",
]

View File

@@ -1,3 +0,0 @@
[workspace]
resolver = "2"
members = ["echo_macro", "test_macro"]

View File

@@ -1,8 +0,0 @@
[package]
name = "echo_macro"
version = "0.0.0"
edition = "2024"
publish = false
[lib]
proc-macro = true

View File

@@ -1,6 +0,0 @@
use proc_macro::TokenStream;
#[proc_macro]
pub fn define_echo(item: TokenStream) -> TokenStream {
format!("fn echo() -> u32 {{ {item} }}").parse().unwrap()
}

View File

@@ -1,8 +0,0 @@
[package]
name = "test_macro"
version = "0.0.0"
edition = "2024"
publish = false
[dependencies]
echo_macro = { path = "../echo_macro" }

View File

@@ -1,9 +0,0 @@
use echo_macro::define_echo;
define_echo!(42);
fn main() {
let a = echo();
println!("proc-macro answer = {a}");
assert_eq!(a, 42, "proc-macro expansion produced the wrong value");
}

View File

@@ -1,7 +1,7 @@
#!/bin/bash
# Run pre-compiled Rust binaries and confirm each emits its expected diagnostic.
# Binaries must already exist in <bins_dir> as <name> for name in
# {hello,panic,overflow,proc_macro}.
# {hello,panic,overflow}.
set -eo pipefail
@@ -54,13 +54,12 @@ declare -A expect=(
[hello]="Hello from main thread"
[panic]="explicit panic from test"
[overflow]="attempt to add with overflow"
[proc_macro]="proc-macro answer = 42"
)
for name in hello panic overflow proc_macro; do
for name in hello panic overflow; do
binary="${bins_dir}/${name}"
if [ "${name}" = "hello" ] || [ "${name}" = "proc_macro" ]; then
if [ "${name}" = "hello" ]; then
expected_rc=0
else
expected_rc=nonzero

View File

@@ -15,55 +15,6 @@ let
runClangTidy = pkgs.writeShellScriptBin "run-clang-tidy" ''
exec ${pkgs.python3}/bin/python3 ${llvmPackages.clang-unwrapped}/bin/run-clang-tidy "$@"
'';
rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml;
# Nix wraps its toolchain so that binaries are exposed only under unsuffixed
# names (gcc, g++, clang-tidy, ...). Several tools probe for a
# version-suffixed name first and fall back to a system binary on the PATH
# when Nix doesn't provide it:
# - Conan's Boost recipe looks up `g++-<major>` before plain `g++`.
# - bin/pre-commit/clang_tidy_check.py looks up `run-clang-tidy-<v>` and
# `clang-apply-replacements-<v>` before the unsuffixed names.
# On a host that also has the matching system binary (e.g. Ubuntu's
# `/usr/bin/g++-15` or `clang-tidy-22`) the probe escapes Nix and mixes a
# system tool into the Nix environment. Generate version-suffixed symlinks
# next to a package's tools so those probes resolve to the Nix ones.
#
# Compiler links must point at whichever compiler is active in a given
# environment (the plain stdenv compiler in the dev shell, the custom-glibc
# wrappers in ci-env.nix), so those callers pass their own `package`; the
# clang tooling is environment-independent and is linked in commonPackages.
mkVersionedToolLinks =
{
name,
package,
version,
tools,
}:
pkgs.linkFarm "${name}-${toString version}-versioned-links" (
map (tool: {
name = "bin/${tool}-${toString version}";
path = "${package}/bin/${tool}";
}) tools
);
clangToolLinks = mkVersionedToolLinks {
name = "clang-tools";
package = clangTools;
version = llvmVersion;
tools = [
"clang-tidy"
"clang-apply-replacements"
"clang-format"
];
};
runClangTidyLink = mkVersionedToolLinks {
name = "run-clang-tidy";
package = runClangTidy;
version = llvmVersion;
tools = [ "run-clang-tidy" ];
};
in
{
inherit
@@ -71,12 +22,9 @@ in
llvmVersion
gccPackage
llvmPackages
mkVersionedToolLinks
;
commonPackages = with pkgs; [
clangToolLinks
runClangTidyLink
ccache
clangbuildanalyzer
clangTools
@@ -115,10 +63,14 @@ in
vim
zip
# Rust packages
cargo
cargo-audit
cargo-llvm-cov
cargo-nextest
clippy
corrosion
rustToolchain
rust-analyzer
rustc
rustfmt
];
}

View File

@@ -1,8 +1,4 @@
{
nixpkgs,
nixpkgs-custom-glibc,
rust-overlay,
}:
{ nixpkgs, nixpkgs-custom-glibc }:
function:
nixpkgs.lib.genAttrs
[
@@ -14,12 +10,7 @@ nixpkgs.lib.genAttrs
(
system:
function {
# rust-overlay adds `pkgs.rust-bin`, from which we build the pinned Rust
# toolchain (see packages.nix). Consumed by both the CI image and dev shell.
pkgs = import nixpkgs {
inherit system;
overlays = [ (import rust-overlay) ];
};
pkgs = import nixpkgs { inherit system; };
# glibc 2.31 — matches the system libc on Ubuntu 20.04 LTS. Sourced
# from the nixpkgs snapshot pinned via the `nixpkgs-custom-glibc`
# flake input, so the build uses the compiler from that snapshot

View File

@@ -1,4 +0,0 @@
[toolchain]
channel = "1.95"
components = ["rustfmt", "clippy", "rust-analyzer"]
profile = "minimal"

View File

@@ -1,22 +0,0 @@
include(XrplAddBenchmark)
# Benchmark requirements.
find_package(benchmark REQUIRED)
# Custom target for all benchmarks defined in this file.
add_custom_target(xrpl.benchmarks)
# Common library dependencies for every benchmark module. `benchmark_main`
# supplies a `main()` that parses the standard Google Benchmark CLI flags
# (`--benchmark_filter`, `--benchmark_format`, ...), so no per-module main.cpp
# is needed.
add_library(xrpl.imports.bench INTERFACE)
target_link_libraries(
xrpl.imports.bench
INTERFACE benchmark::benchmark_main xrpl.libxrpl
)
# One benchmark executable for each module.
xrpl_add_benchmark(nodestore)
target_link_libraries(xrpl.bench.nodestore PRIVATE xrpl.imports.bench)
add_dependencies(xrpl.benchmarks xrpl.bench.nodestore)

View File

@@ -1,329 +0,0 @@
#include <xrpl/nodestore/Backend.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Types.h>
#include <benchmark/benchmark.h>
#include <benchmarks/libxrpl/nodestore/NodeStoreBench.h>
#include <array>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace xrpl::NodeStore {
namespace {
constexpr std::size_t kPoolSizes[] = {1000, 10000, 100000};
constexpr int kThreadCounts[] = {1, 4, 8};
constexpr std::size_t kBatchSize = 256;
constexpr std::string_view kNamePrefix = "BM_Backend_";
constexpr std::string_view kNameSeparator = "/";
struct RunState
{
std::unique_ptr<BackendHarness> harness;
Batch present; // prefix-1 objects, eligible to be stored
Batch recent; // prefix-1 objects in the "future" key space
std::vector<uint256> missing; // prefix-2 keys that are never stored
std::vector<std::size_t> shuffle; // [0, poolSize) permutation for random-like access
std::size_t avgPayload = 0; // mean getData().size() over `present`
void
release()
{
harness.reset();
Batch{}.swap(present);
Batch{}.swap(recent);
std::vector<uint256>{}.swap(missing);
std::vector<std::size_t>{}.swap(shuffle);
}
};
struct SetupContext
{
RunState& rs;
Backend& backend;
std::size_t poolSize;
};
struct IterateContext
{
RunState& rs;
Backend& backend;
std::size_t index;
std::size_t poolSize;
};
struct Workload
{
std::string_view name;
std::function<void(SetupContext const&)> setup;
std::function<void(IterateContext const&)> iterate;
bool reportBytes = false; // SetBytesProcessed from rs.avgPayload
bool clobber = true; // ClobberMemory after the loop (false for pure stores)
bool pinToPool = false; // pin iterations to one pool sweep instead of autotuning
};
// One store() per iteration. Iterations are pinned to one pool sweep (per
// thread) so the index never wraps past the pool - otherwise NuDB::doInsert
// swallows key_exists and the workload degenerates into duplicate-detection
// no-ops.
Workload const kInsert{
.name = "Insert",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.avgPayload = averagePayload(ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
backend.store(rs.present[index % poolSize]);
},
.reportBytes = true,
.clobber = false,
.pinToPool = true,
};
// One fetch() of a present key (a hit) per iteration.
Workload const kFetch{
.name = "Fetch",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.avgPayload = averagePayload(ctx.rs.present);
prepopulate(ctx.backend, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
std::shared_ptr<NodeObject> result;
backend.fetch(rs.present[index % poolSize]->getHash(), &result);
benchmark::DoNotOptimize(result);
},
.reportBytes = true,
};
// One fetch() of a never-stored key (a miss); the backend is left empty.
Workload const kMissing{
.name = "Missing",
.setup = [](SetupContext const& ctx) { ctx.rs.missing = makeMissingKeys(ctx.poolSize); },
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
std::shared_ptr<NodeObject> result;
backend.fetch(rs.missing[index % poolSize], &result);
benchmark::DoNotOptimize(result);
},
};
// 80% hits / 20% misses. The fetch index comes from a shuffle table so access
// is random-like without per-iteration RNG cost; sequential `index % poolSize`
// would be artificially cache-friendly to RocksDB's block cache.
Workload const kMixed{
.name = "Mixed",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.missing = makeMissingKeys(ctx.poolSize);
ctx.rs.shuffle = makeShuffle(ctx.poolSize, /*seed=*/1);
prepopulate(ctx.backend, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
std::shared_ptr<NodeObject> result;
auto const pick = rs.shuffle[index % poolSize];
if (index % 5 == 0)
{
backend.fetch(rs.missing[pick], &result);
}
else
{
backend.fetch(rs.present[pick]->getHash(), &result);
}
benchmark::DoNotOptimize(result);
},
};
// An xrpld-like cycle: a hit, a maybe-miss recent fetch, and a store. The
// recent fetch uses the shuffle table (not `slot`) so it doesn't fetch the item
// it's about to store this iteration - which would give an all-miss-then-hit
// step instead of a smooth ramp. The store walks sequentially so each recent
// object is stored once.
Workload const kWork{
.name = "Work",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.recent = makePool(1, ctx.poolSize, ctx.poolSize);
ctx.rs.shuffle = makeShuffle(ctx.poolSize, /*seed=*/2);
prepopulate(ctx.backend, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
auto const slot = index % poolSize;
auto const pick = rs.shuffle[slot];
std::shared_ptr<NodeObject> historical;
backend.fetch(rs.present[pick]->getHash(), &historical);
benchmark::DoNotOptimize(historical);
std::shared_ptr<NodeObject> recent;
backend.fetch(rs.recent[pick]->getHash(), &recent);
benchmark::DoNotOptimize(recent);
backend.store(rs.recent[slot]);
},
.clobber = true,
.pinToPool = true,
};
auto
makeRunner(Workload w, std::string cfg, std::shared_ptr<RunState> rs)
{
return [w = std::move(w), cfg = std::move(cfg), rs = std::move(rs)](benchmark::State& state) {
auto const poolSize = static_cast<std::size_t>(state.range(0));
if (state.thread_index() == 0)
{
rs->harness = std::make_unique<BackendHarness>(cfg);
w.setup(
SetupContext{.rs = *rs, .backend = *rs->harness->backend, .poolSize = poolSize});
}
std::size_t index = state.thread_index();
for (auto _ : state)
{
w.iterate(
IterateContext{
.rs = *rs,
.backend = *rs->harness->backend,
.index = index,
.poolSize = poolSize});
index += state.threads();
}
if (w.clobber)
benchmark::ClobberMemory();
state.SetItemsProcessed(state.iterations());
if (w.reportBytes)
state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations() * rs->avgPayload));
if (state.thread_index() == 0)
rs->release();
};
}
// Register workload `w` against backend `bc`, choosing the registration shape
// from `w.pinToPool`.
void
registerWorkload(BackendConfig const& bc, Workload const& w)
{
std::string const cfg = bc.config;
std::string name{kNamePrefix};
name += w.name;
name += kNameSeparator;
name += bc.name;
if (!w.pinToPool)
{
auto rs = std::make_shared<RunState>();
auto* b = benchmark::RegisterBenchmark(name, makeRunner(w, cfg, rs));
b->RangeMultiplier(10)->Range(kPoolSizes[0], kPoolSizes[std::size(kPoolSizes) - 1]);
b->Threads(1)->Threads(4)->Threads(8)->UseRealTime();
return;
}
for (auto const poolSize : kPoolSizes)
{
for (auto const threads : kThreadCounts)
{
if (poolSize % static_cast<std::size_t>(threads) != 0)
continue;
auto rs = std::make_shared<RunState>();
benchmark::RegisterBenchmark(name, makeRunner(w, cfg, rs))
->Arg(poolSize)
->Iterations(poolSize / static_cast<std::size_t>(threads))
->Threads(threads)
->UseRealTime();
}
}
}
// One storeBatch() of kBatchSize objects per iteration. Single-threaded:
// Backend::storeBatch must not run concurrently with itself or store().
// Iterations are pinned to the batch count so the index never wraps into
// key_exists no-ops. Kept separate from Workload: batch slicing and the
// per-batch item/byte accounting don't fit the thread-axis mold.
void
registerStoreBatch(BackendConfig const& bc)
{
std::string const cfg = bc.config;
std::string name{kNamePrefix};
name += "StoreBatch";
name += kNameSeparator;
name += bc.name;
for (auto const poolSize : kPoolSizes)
{
auto const numBatches = poolSize / kBatchSize;
if (numBatches == 0)
continue;
auto rs = std::make_shared<RunState>();
benchmark::RegisterBenchmark(
name,
[rs, cfg](benchmark::State& state) {
auto const poolSize = static_cast<std::size_t>(state.range(0));
rs->harness = std::make_unique<BackendHarness>(cfg);
rs->present = makePool(1, poolSize);
rs->avgPayload = averagePayload(rs->present);
std::vector<Batch> const batches = sliceBatches(rs->present, kBatchSize);
if (batches.empty())
{
state.SkipWithError("pool smaller than one batch");
return;
}
std::size_t index = 0;
for (auto _ : state)
{
rs->harness->backend->storeBatch(batches[index % batches.size()]);
++index;
}
state.SetItemsProcessed(static_cast<std::int64_t>(state.iterations() * kBatchSize));
state.SetBytesProcessed(
static_cast<std::int64_t>(state.iterations() * kBatchSize * rs->avgPayload));
rs->release();
})
->Arg(poolSize)
->Iterations(numBatches);
}
}
[[maybe_unused]] bool const kRegistered = [] {
auto const workloads = std::to_array({&kInsert, &kFetch, &kMissing, &kMixed, &kWork});
for (auto const& bc : backendConfigs())
{
for (auto const* w : workloads)
registerWorkload(bc, *w);
registerStoreBatch(bc);
}
return true;
}();
} // namespace
} // namespace xrpl::NodeStore

View File

@@ -1,243 +0,0 @@
#include <xrpl/nodestore/Database.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Types.h>
#include <benchmark/benchmark.h>
#include <benchmarks/libxrpl/nodestore/NodeStoreBench.h>
#include <array>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace xrpl::NodeStore {
namespace {
// Number of distinct objects pre-generated per run.
constexpr std::size_t kDefaultPoolSize = 100000;
// Async read threads the Database spawns. Unused by the synchronous fetch path
// these benchmarks take; kept fixed so runs are comparable.
constexpr int kReadThreads = 4;
constexpr std::string_view kNamePrefix = "BM_Database_";
constexpr std::string_view kNameSeparator = "/";
struct RunState
{
std::unique_ptr<DatabaseHarness> harness;
Batch present; // prefix-1 objects, eligible to be stored
Batch recent; // prefix-1 objects in the "future" key space
std::vector<uint256> missing; // prefix-2 keys that are never stored
std::vector<std::size_t> shuffle; // [0, poolSize) permutation for random-like access
std::size_t avgPayload = 0; // mean getData().size() over `present`
};
struct SetupContext
{
RunState& rs;
Database& db;
std::size_t poolSize;
};
struct IterateContext
{
RunState& rs;
Database& db;
std::uint32_t seq;
std::size_t index;
std::size_t poolSize;
};
struct Workload
{
std::string_view name;
std::function<void(SetupContext const&)> setup;
std::function<void(IterateContext const&)> iterate;
bool reportBytes = false;
bool pinIterations = false;
};
void
prepopulate(Database& db, Batch const& objects)
{
auto const seq = db.earliestLedgerSeq();
for (auto const& obj : objects)
{
Blob data(obj->getData());
db.store(obj->getType(), std::move(data), obj->getHash(), seq);
}
db.sync();
}
// One store() per iteration; a fresh Blob copy is handed over each time.
Workload const kStore{
.name = "Store",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.avgPayload = averagePayload(ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto const& obj = rs.present[index % poolSize];
Blob data(obj->getData());
db.store(obj->getType(), std::move(data), obj->getHash(), seq);
},
.reportBytes = true,
.pinIterations = true,
};
// One fetchNodeObject() of a stored key (a hit) per iteration.
Workload const kFetch{
.name = "Fetch",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.avgPayload = averagePayload(ctx.rs.present);
prepopulate(ctx.db, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto obj = db.fetchNodeObject(rs.present[index % poolSize]->getHash(), seq);
benchmark::DoNotOptimize(obj);
},
.reportBytes = true,
};
// One fetchNodeObject() of a never-stored key (a miss) per iteration.
Workload const kMissing{
.name = "Missing",
.setup = [](SetupContext const& ctx) { ctx.rs.missing = makeMissingKeys(ctx.poolSize); },
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto obj = db.fetchNodeObject(rs.missing[index % poolSize], seq);
benchmark::DoNotOptimize(obj);
},
};
// 80% hits / 20% misses. The fetch index comes from a shuffle table so access
// is random-like without per-iteration RNG cost; sequential `index % poolSize`
// would be artificially cache-friendly.
Workload const kMixed{
.name = "Mixed",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.missing = makeMissingKeys(ctx.poolSize);
ctx.rs.shuffle = makeShuffle(ctx.poolSize, /*seed=*/1);
prepopulate(ctx.db, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto const pick = rs.shuffle[index % poolSize];
std::shared_ptr<NodeObject> obj;
if (index % 5 == 0)
{
obj = db.fetchNodeObject(rs.missing[pick], seq);
}
else
{
obj = db.fetchNodeObject(rs.present[pick]->getHash(), seq);
}
benchmark::DoNotOptimize(obj);
},
};
// An xrpld-like cycle: a hit, a maybe-miss recent fetch, and a store. The
// recent fetch uses the shuffle table (not `slot`) so it doesn't fetch the item
// it's about to store this iteration - which would give an all-miss-then-hit
// step instead of a smooth ramp. The store walks sequentially so each recent
// object is stored once.
Workload const kWork{
.name = "Work",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.recent = makePool(1, ctx.poolSize, ctx.poolSize);
ctx.rs.shuffle = makeShuffle(ctx.poolSize, /*seed=*/2);
prepopulate(ctx.db, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto const slot = index % poolSize;
auto const pick = rs.shuffle[slot];
auto historical = db.fetchNodeObject(rs.present[pick]->getHash(), seq);
benchmark::DoNotOptimize(historical);
auto recent = db.fetchNodeObject(rs.recent[pick]->getHash(), seq);
benchmark::DoNotOptimize(recent);
auto const& obj = rs.recent[slot];
Blob data(obj->getData());
db.store(obj->getType(), std::move(data), obj->getHash(), seq);
},
.pinIterations = true,
};
void
registerWorkload(BackendConfig const& bc, Workload const& w)
{
auto rs = std::make_shared<RunState>();
std::string const cfg = bc.config;
std::string name{kNamePrefix};
name += w.name;
name += kNameSeparator;
name += bc.name;
auto* b = benchmark::RegisterBenchmark(name, [rs, cfg, w](benchmark::State& state) {
auto const poolSize = static_cast<std::size_t>(state.range(0));
rs->harness = std::make_unique<DatabaseHarness>(cfg, kReadThreads);
auto& db = *rs->harness->db;
w.setup(SetupContext{.rs = *rs, .db = db, .poolSize = poolSize});
auto const seq = db.earliestLedgerSeq();
std::size_t index = 0;
for (auto _ : state)
{
w.iterate(
IterateContext{
.rs = *rs, .db = db, .seq = seq, .index = index, .poolSize = poolSize});
++index;
}
benchmark::ClobberMemory();
state.SetItemsProcessed(state.iterations());
if (w.reportBytes)
{
state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations() * rs->avgPayload));
}
rs->harness.reset();
});
b->Arg(kDefaultPoolSize);
if (w.pinIterations)
b->Iterations(kDefaultPoolSize);
}
[[maybe_unused]] bool const kRegistered = [] {
auto const workloads = std::to_array({&kStore, &kFetch, &kMissing, &kMixed, &kWork});
for (auto const& bc : backendConfigs())
{
for (auto const* w : workloads)
registerWorkload(bc, *w);
}
return true;
}();
} // namespace
} // namespace xrpl::NodeStore

View File

@@ -1,318 +0,0 @@
#pragma once
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/ByteUtilities.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/safe_cast.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/temp_dir.h>
#include <xrpl/beast/xor_shift_engine.h>
#include <xrpl/config/BasicConfig.h>
#include <xrpl/nodestore/Backend.h>
#include <xrpl/nodestore/Database.h>
#include <xrpl/nodestore/DummyScheduler.h>
#include <xrpl/nodestore/Manager.h>
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Scheduler.h>
#include <xrpl/nodestore/Types.h>
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/split.hpp>
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <memory>
#include <numeric>
#include <random>
#include <string>
#include <utility>
#include <vector>
// Shared helpers for the NodeStore benchmarks.
//
namespace xrpl::NodeStore {
// Fill `bytes` of memory at `buffer` with random bits drawn from `g`.
template <class Generator>
inline void
rngcpy(void* buffer, std::size_t bytes, Generator& g)
{
using result_type = typename Generator::result_type;
while (bytes >= sizeof(result_type))
{
auto const v = g();
std::memcpy(buffer, &v, sizeof(v));
buffer = reinterpret_cast<std::uint8_t*>(buffer) + sizeof(v);
bytes -= sizeof(v);
}
if (bytes > 0)
{
auto const v = g();
std::memcpy(buffer, &v, bytes);
}
}
/**
* @brief Deterministic generator of a reproducible sequence of random NodeObjects.
*
* Indexing is stable: `obj(n)` and `key(n)` always return the same value for a
* given `n`, regardless of call order, because the engine is reseeded from `n`
* on every call.
*
* Using different prefixes guarantees the two key spaces are disjoint for the fetch-miss
* workloads.
*/
class Sequence
{
private:
static constexpr auto kMinSize = 250;
static constexpr auto kMaxSize = 1250;
beast::xor_shift_engine gen_;
std::uint8_t prefix_;
std::discrete_distribution<std::uint32_t> dType_;
std::uniform_int_distribution<std::uint32_t> dSize_;
public:
explicit Sequence(std::uint8_t prefix)
: prefix_(prefix)
// uniform distribution over hotLEDGER - hotTRANSACTION_NODE
// but exclude hotTRANSACTION = 2 (removed)
, dType_({1, 1, 0, 1, 1})
, dSize_(kMinSize, kMaxSize)
{
}
// Returns the n-th key. Used to generate keys that are never stored.
// The layout mirrors obj()'s: prefix at byte 0, RNG over the rest, so the
// two key spaces stay disjoint by construction (not by coincidence).
uint256
key(std::size_t n)
{
gen_.seed(n + 1);
uint256 result;
auto const data = static_cast<std::uint8_t*>(&*result.begin());
*data = prefix_;
rngcpy(data + 1, result.size() - 1, gen_);
return result;
}
// Returns the n-th complete NodeObject.
std::shared_ptr<NodeObject>
obj(std::size_t n)
{
gen_.seed(n + 1);
uint256 key;
auto const data = static_cast<std::uint8_t*>(&*key.begin());
*data = prefix_;
rngcpy(data + 1, key.size() - 1, gen_);
Blob value(dSize_(gen_));
rngcpy(&value[0], value.size(), gen_);
return NodeObject::createObject(
safeCast<NodeObjectType>(dType_(gen_)), std::move(value), key);
}
// Fills `b` with `size` consecutive NodeObjects starting at index `n`.
void
batch(std::size_t n, Batch& b, std::size_t size)
{
b.clear();
b.reserve(size);
while ((size--) != 0u)
b.push_back(obj(n++));
}
};
// Parse a comma-separated "key=value,key=value" string into a config Section.
inline Section
parseConfig(std::string const& s)
{
Section section;
std::vector<std::string> values;
boost::split(values, s, boost::algorithm::is_any_of(","));
section.append(values);
return section;
}
// Pre-generate `count` distinct objects from key space `prefix`, starting at
// sequence index `start`.
inline Batch
makePool(std::uint8_t prefix, std::size_t count, std::size_t start = 0)
{
Sequence seq(prefix);
Batch pool;
pool.reserve(count);
for (std::size_t i = 0; i < count; ++i)
pool.push_back(seq.obj(start + i));
return pool;
}
// Pre-generate `count` keys disjoint from every `makePool(...)` object, for
// measuring fetches that miss.
inline std::vector<uint256>
makeMissingKeys(std::size_t count)
{
Sequence seq(2);
std::vector<uint256> keys;
keys.reserve(count);
for (std::size_t i = 0; i < count; ++i)
keys.push_back(seq.key(i));
return keys;
}
// Mean payload size across a pool, used for SetBytesProcessed throughput.
inline std::size_t
averagePayload(Batch const& pool)
{
if (pool.empty())
return 0;
std::size_t total = 0;
for (auto const& obj : pool)
total += obj->getData().size();
return total / pool.size();
}
// Store every object and flush, so a following fetch exercises the real read
// path rather than an in-memory write buffer.
//
// We chunk the write at kBatchWriteLimitSize because Types.h documents that as
// the maximum allowed batch size. NuDB happens to tolerate larger batches
// today, but the benchmark should not rely on that.
//
// sync() is a no-op for both NuDB and RocksDB at the moment (NuDB has a small
// internal burst buffer that the timed loop will warm up). That is a contract
// hint, not a guarantee; if either backend ever grows a real flush we get it
// here for free.
inline void
prepopulate(Backend& backend, Batch const& objects)
{
for (std::size_t i = 0; i < objects.size(); i += kBatchWriteLimitSize)
{
auto const end = std::min(i + kBatchWriteLimitSize, objects.size());
backend.storeBatch(Batch(objects.begin() + i, objects.begin() + end));
}
backend.sync();
}
// A deterministic permutation of [0, size). Lets the timed loop visit the
// pre-generated pool in a random-like order with zero RNG cost per iteration -
// the Timing_test workloads it replaces used uniform_int_distribution per
// fetch, and a shuffle table reproduces that access pattern without paying for
// the distribution inside the timed region.
inline std::vector<std::size_t>
makeShuffle(std::size_t size, std::uint64_t seed)
{
std::vector<std::size_t> v(size);
std::iota(v.begin(), v.end(), std::size_t{0});
beast::xor_shift_engine gen(seed);
std::shuffle(v.begin(), v.end(), gen);
return v;
}
// Partition a pool into fixed-size batches. Any trailing remainder shorter than
// `batchSize` is dropped, so every returned batch has exactly `batchSize`.
inline std::vector<Batch>
sliceBatches(Batch const& pool, std::size_t batchSize)
{
std::vector<Batch> batches;
if (batchSize == 0)
return batches;
batches.reserve(pool.size() / batchSize);
for (std::size_t i = 0; i + batchSize <= pool.size(); i += batchSize)
batches.emplace_back(pool.begin() + i, pool.begin() + i + batchSize);
return batches;
}
/**
* @brief RAII owner of a NodeStore Backend opened on a private temporary directory.
*
* Member declaration order matters: `tempDir` is declared first so it is
* destroyed last, after the backend has closed and released its files.
*/
struct BackendHarness
{
beast::TempDir tempDir;
DummyScheduler scheduler;
beast::Journal journal{beast::Journal::getNullSink()};
std::unique_ptr<Backend> backend;
explicit BackendHarness(std::string const& configString)
{
Section config = parseConfig(configString);
// A private, unique path per harness, so concurrent or repeated runs
// never share on-disk state.
config.set("path", tempDir.path());
backend =
Manager::instance().makeBackend(config, megabytes(std::size_t{4}), scheduler, journal);
backend->setDeletePath();
backend->open();
}
~BackendHarness()
{
if (backend)
backend->close();
}
};
/**
* RAII owner of a NodeStore Database - the application-facing wrapper around a
* Backend, which adds fetch/store accounting and the async read-thread pool.
*/
struct DatabaseHarness
{
beast::TempDir tempDir;
DummyScheduler scheduler;
beast::Journal journal{beast::Journal::getNullSink()};
std::unique_ptr<Database> db;
DatabaseHarness(std::string const& configString, int readThreads)
{
Section config = parseConfig(configString);
config.set("path", tempDir.path());
db = Manager::instance().makeDatabase(
megabytes(std::size_t{4}), scheduler, readThreads, config, journal);
}
~DatabaseHarness()
{
if (db)
db->stop();
}
};
// A NodeStore backend to benchmark, named for the --benchmark_filter CLI flag.
struct BackendConfig
{
char const* name; // short label, e.g. "nudb"
char const* config; // parseConfig() string, e.g. "type=nudb"
};
// The backends every workload is registered against.
//
// The in-memory backend is intentionally excluded. It keeps its table in a
// process-global map keyed by path, with no removal API, so building a fresh
// backend per run - as a microbenchmark must - would leak the whole dataset on
// every run. Timing_test, the suite this benchmark replaces, excluded it for
// the same reason. NuDB and RocksDB are the production backends worth timing.
//
// RocksDB is included only when it was compiled in (xrpl.libxrpl carries
// XRPL_ROCKSDB_AVAILABLE transitively).
inline std::vector<BackendConfig> const&
backendConfigs()
{
static std::vector<BackendConfig> const kConfigs = {
{.name = "nudb", .config = "type=nudb"},
#if XRPL_ROCKSDB_AVAILABLE
{.name = "rocksdb",
.config = "type=rocksdb,open_files=2000,filter_bits=12,cache_mb=256,"
"file_size_mb=8,file_size_mult=2"},
#endif
};
return kConfigs;
}
} // namespace xrpl::NodeStore

View File

@@ -312,25 +312,6 @@ computeInterestAndFeeParts(
return std::make_pair(interest - fee, fee);
}
/* Rounds a raw (unrounded) interest amount to the loan's scale, then splits
* the rounded amount into net interest (to the vault) and management fee (to
* the broker).
*
* This is the common "round then split" step shared by late payment, full
* payment, and overpayment interest calculations.
*/
std::pair<Number, Number>
roundAndSplitInterest(
Asset const& asset,
Number const& rawInterest,
TenthBips16 managementFeeRate,
std::int32_t loanScale,
Number::RoundingMode mode = Number::getround())
{
auto const interest = roundToAsset(asset, rawInterest, loanScale, mode);
return computeInterestAndFeeParts(asset, interest, managementFeeRate, loanScale);
}
/* Calculates penalty interest accrued on overdue payments.
* Returns 0 if payment is not late.
*
@@ -406,18 +387,22 @@ loanAccruedInterest(
*
* This is the core function that updates the Loan ledger object fields based on
* a computed payment.
*/
LoanPaymentParts
doPayment(ExtendedPaymentComponents const& payment, SLE::ref loan)
{
auto totalValueOutstandingProxy = loan->at(sfTotalValueOutstanding);
auto principalOutstandingProxy = loan->at(sfPrincipalOutstanding);
auto managementFeeOutstandingProxy = loan->at(sfManagementFeeOutstanding);
auto paymentRemainingProxy = loan->at(sfPaymentRemaining);
auto prevPaymentDateProxy = loan->at(sfPreviousPaymentDueDate);
auto nextDueDateProxy = loan->at(sfNextPaymentDueDate);
std::uint32_t const paymentInterval = loan->at(sfPaymentInterval);
* The function is templated to work with both direct Number/uint32_t values
* (for testing/simulation) and ValueProxy types (for actual ledger updates).
*/
template <class NumberProxy, class UInt32Proxy, class UInt32OptionalProxy>
LoanPaymentParts
doPayment(
ExtendedPaymentComponents const& payment,
NumberProxy& totalValueOutstandingProxy,
NumberProxy& principalOutstandingProxy,
NumberProxy& managementFeeOutstandingProxy,
UInt32Proxy& paymentRemainingProxy,
UInt32Proxy& prevPaymentDateProxy,
UInt32OptionalProxy& nextDueDateProxy,
std::uint32_t paymentInterval)
{
XRPL_ASSERT_PARTS(nextDueDateProxy, "xrpl::detail::doPayment", "Next due date proxy set");
if (payment.specialCase == PaymentSpecialCase::Final)
@@ -485,12 +470,16 @@ doPayment(ExtendedPaymentComponents const& payment, SLE::ref loan)
// Principal can never exceed total value (principal is part of total value)
XRPL_ASSERT_PARTS(
// Use an explicit cast because the template parameter can be
// ValueProxy<Number> or Number
static_cast<Number>(principalOutstandingProxy) <=
static_cast<Number>(totalValueOutstandingProxy),
"xrpl::detail::doPayment",
"principal does not exceed total");
XRPL_ASSERT_PARTS(
// Use an explicit cast because the template parameter can be
// ValueProxy<Number> or Number
static_cast<Number>(managementFeeOutstandingProxy) >= beast::kZero,
"xrpl::detail::doPayment",
"fee outstanding stays valid");
@@ -728,23 +717,22 @@ tryOverpayment(
* overpayment would leave the loan in an invalid state, we can reject it
* gracefully without corrupting the ledger data.
*/
template <class NumberProxy>
std::expected<LoanPaymentParts, TER>
doOverpayment(
Rules const& rules,
Asset const& asset,
std::int32_t loanScale,
ExtendedPaymentComponents const& overpaymentComponents,
SLE::ref loan,
NumberProxy& totalValueOutstandingProxy,
NumberProxy& principalOutstandingProxy,
NumberProxy& managementFeeOutstandingProxy,
NumberProxy& periodicPaymentProxy,
Number const& periodicRate,
std::uint32_t const paymentRemaining,
TenthBips16 const managementFeeRate,
beast::Journal j)
{
auto totalValueOutstandingProxy = loan->at(sfTotalValueOutstanding);
auto principalOutstandingProxy = loan->at(sfPrincipalOutstanding);
auto managementFeeOutstandingProxy = loan->at(sfManagementFeeOutstanding);
auto periodicPaymentProxy = loan->at(sfPeriodicPayment);
auto const paymentsRemaining = loan->at(sfPaymentRemaining);
auto const loanState = constructLoanState(
totalValueOutstandingProxy, principalOutstandingProxy, managementFeeOutstandingProxy);
auto const periodicPayment = periodicPaymentProxy;
@@ -756,7 +744,7 @@ doOverpayment(
<< ", interestPart: " << overpaymentComponents.trackedInterestPart()
<< ", untrackedInterest: " << overpaymentComponents.untrackedInterest
<< ", totalDue: " << overpaymentComponents.totalDue
<< ", payments remaining :" << paymentsRemaining;
<< ", payments remaining :" << paymentRemaining;
// Attempt to re-amortize the loan with the overpayment applied.
// This modifies the temporary copies, leaving the proxies unchanged.
@@ -768,7 +756,7 @@ doOverpayment(
loanState,
periodicPayment,
periodicRate,
paymentsRemaining,
paymentRemaining,
managementFeeRate,
j);
if (!ret)
@@ -876,15 +864,16 @@ std::expected<ExtendedPaymentComponents, TER>
computeLatePayment(
Asset const& asset,
ApplyView const& view,
SLE::const_ref loan,
Number const& principalOutstanding,
std::int32_t nextDueDate,
ExtendedPaymentComponents const& periodic,
TenthBips32 lateInterestRate,
std::int32_t loanScale,
Number const& latePaymentFee,
STAmount const& amount,
TenthBips16 managementFeeRate,
beast::Journal j)
{
std::int32_t const nextDueDate = loan->at(sfNextPaymentDueDate);
std::int32_t const loanScale = loan->at(sfLoanScale);
// Check if the due date has passed. If not, reject the payment as
// being too soon
if (!hasExpired(view, nextDueDate))
@@ -892,15 +881,15 @@ computeLatePayment(
// Calculate the penalty interest based on how long the payment is overdue.
auto const latePaymentInterest = loanLatePaymentInterest(
loan->at(sfPrincipalOutstanding),
TenthBips32{loan->at(sfLateInterestRate)},
view.parentCloseTime(),
nextDueDate);
principalOutstanding, lateInterestRate, view.parentCloseTime(), nextDueDate);
// Round the late interest and split it between the vault (net interest)
// and the broker (management fee portion).
auto const [roundedLateInterest, roundedLateManagementFee] =
roundAndSplitInterest(asset, latePaymentInterest, managementFeeRate, loanScale);
// and the broker (management fee portion). This lambda ensures we
// round before splitting to maintain precision.
auto const [roundedLateInterest, roundedLateManagementFee] = [&]() {
auto const interest = roundToAsset(asset, latePaymentInterest, loanScale);
return computeInterestAndFeeParts(asset, interest, managementFeeRate, loanScale);
}();
XRPL_ASSERT(roundedLateInterest >= 0, "xrpl::detail::computeLatePayment : valid late interest");
XRPL_ASSERT_PARTS(
@@ -919,7 +908,7 @@ computeLatePayment(
// 1. Regular service fee (from periodic.untrackedManagementFee)
// 2. Late payment fee (fixed penalty)
// 3. Management fee portion of late interest
periodic.untrackedManagementFee + loan->at(sfLatePaymentFee) + roundedLateManagementFee,
periodic.untrackedManagementFee + latePaymentFee + roundedLateManagementFee,
// Untracked interest includes:
// 1. Any untracked interest from the regular payment (usually 0)
@@ -969,15 +958,22 @@ std::expected<ExtendedPaymentComponents, TER>
computeFullPayment(
Asset const& asset,
ApplyView& view,
SLE::const_ref loan,
Number const& principalOutstanding,
Number const& managementFeeOutstanding,
Number const& periodicPayment,
std::uint32_t paymentRemaining,
std::uint32_t prevPaymentDate,
std::uint32_t const startDate,
std::uint32_t const paymentInterval,
TenthBips32 const closeInterestRate,
std::int32_t loanScale,
Number const& totalInterestOutstanding,
Number const& periodicRate,
Number const& closePaymentFee,
STAmount const& amount,
TenthBips16 managementFeeRate,
beast::Journal j)
{
std::uint32_t const paymentRemaining = loan->at(sfPaymentRemaining);
std::int32_t const loanScale = loan->at(sfLoanScale);
// Full payment must be made before the final scheduled payment.
if (paymentRemaining <= 1)
{
@@ -990,7 +986,7 @@ computeFullPayment(
// This theoretical (unrounded) value is used to compute interest and
// penalties accurately.
Number const theoreticalPrincipalOutstanding = loanPrincipalFromPeriodicPayment(
view.rules(), loan->at(sfPeriodicPayment), periodicRate, paymentRemaining);
view.rules(), periodicPayment, periodicRate, paymentRemaining);
// Full payment interest includes both accrued interest (time since last
// payment) and prepayment penalty (for closing early).
@@ -998,21 +994,18 @@ computeFullPayment(
theoreticalPrincipalOutstanding,
periodicRate,
view.parentCloseTime(),
loan->at(sfPaymentInterval),
loan->at(sfPreviousPaymentDueDate),
loan->at(sfStartDate),
TenthBips32{loan->at(sfCloseInterestRate)});
paymentInterval,
prevPaymentDate,
startDate,
closeInterestRate);
// Split the full payment interest into net interest (to vault) and management fee (to broker),
// applying proper rounding.
auto const [roundedFullInterest, roundedFullManagementFee] = roundAndSplitInterest(
asset, fullPaymentInterest, managementFeeRate, loanScale, Number::RoundingMode::Downward);
LoanState const loanState = constructLoanState(loan);
Number const principalOutstanding = loanState.principalOutstanding;
Number const managementFeeOutstanding = loanState.managementFeeDue;
Number const totalInterestOutstanding = loanState.interestDue;
Number const closePaymentFee = roundToAsset(asset, loan->at(sfClosePaymentFee), loanScale);
// Split the full payment interest into net interest (to vault) and
// management fee (to broker), applying proper rounding.
auto const [roundedFullInterest, roundedFullManagementFee] = [&]() {
auto const interest =
roundToAsset(asset, fullPaymentInterest, loanScale, Number::RoundingMode::Downward);
return computeInterestAndFeeParts(asset, interest, managementFeeRate, loanScale);
}();
ExtendedPaymentComponents const full{
PaymentComponents{
@@ -1053,7 +1046,8 @@ computeFullPayment(
"xrpl::detail::computeFullPayment",
"total due is rounded");
JLOG(j.trace()) << "computeFullPayment result: periodicRate: " << periodicRate
JLOG(j.trace()) << "computeFullPayment result: periodicPayment: " << periodicPayment
<< ", periodicRate: " << periodicRate
<< ", paymentRemaining: " << paymentRemaining
<< ", theoreticalPrincipalOutstanding: " << theoreticalPrincipalOutstanding
<< ", fullPaymentInterest: " << fullPaymentInterest
@@ -1304,34 +1298,6 @@ computePaymentComponents(
};
}
/* Thin overload of computePaymentComponents() that unwraps the tracked
* fields directly from the Loan ledger object. `periodicRate` is derived
* rather than stored, and `managementFeeRate` comes from the LoanBroker, not
* the Loan, so both remain explicit parameters. Kept separate from the
* value-based overload above, which is exercised directly by unit tests
* against simulated (non-ledger) loan states.
*/
PaymentComponents
computePaymentComponents(
Rules const& rules,
Asset const& asset,
SLE::ref loan,
Number const& periodicRate,
TenthBips16 managementFeeRate)
{
return computePaymentComponents(
rules,
asset,
loan->at(sfLoanScale),
loan->at(sfTotalValueOutstanding),
loan->at(sfPrincipalOutstanding),
loan->at(sfManagementFeeOutstanding),
loan->at(sfPeriodicPayment),
periodicRate,
loan->at(sfPaymentRemaining),
managementFeeRate);
}
/* Computes payment components for an overpayment scenario.
*
* An overpayment occurs when a borrower pays more than the scheduled periodic
@@ -1376,12 +1342,11 @@ computeOverpaymentComponents(
// This interest doesn't follow the normal amortization schedule - it's
// a one-time charge for paying early.
// Equation (20) and (21) from XLS-66 spec, Section A-2 Equation Glossary
auto const [roundedOverpaymentInterest, roundedOverpaymentManagementFee] =
roundAndSplitInterest(
asset,
tenthBipsOfValue(overpayment, overpaymentInterestRate),
managementFeeRate,
loanScale);
auto const [roundedOverpaymentInterest, roundedOverpaymentManagementFee] = [&]() {
auto const interest =
roundToAsset(asset, tenthBipsOfValue(overpayment, overpaymentInterestRate), loanScale);
return detail::computeInterestAndFeeParts(asset, interest, managementFeeRate, loanScale);
}();
auto const result = detail::ExtendedPaymentComponents{
// Build the payment components, after fees and penalty
@@ -1408,265 +1373,6 @@ computeOverpaymentComponents(
return result;
}
/* Derives the two rate values every make*Payment() helper needs: the
* broker's management fee rate, and the loan's periodic (per-payment-period)
* interest rate.
*/
std::pair<TenthBips16, Number>
loanRatesFor(SLE::const_ref loan, SLE::const_ref brokerSle)
{
TenthBips16 const managementFeeRate{brokerSle->at(sfManagementFeeRate)};
TenthBips32 const interestRate{loan->at(sfInterestRate)};
Number const periodicRate = loanPeriodicRate(interestRate, loan->at(sfPaymentInterval));
XRPL_ASSERT(interestRate == 0 || periodicRate > 0, "xrpl::detail::loanRatesFor : valid rate");
return {managementFeeRate, periodicRate};
}
/* Handles a full (early payoff) payment. Implements the "full payment"
* branch of the make_payment function from the XLS-66 spec, Section
* 3.2.4.4.
*/
std::expected<LoanPaymentParts, TER>
makeFullPayment(
Asset const& asset,
ApplyView& view,
SLE::ref loan,
SLE::const_ref brokerSle,
STAmount const& amount,
beast::Journal j)
{
auto const [managementFeeRate, periodicRate] = loanRatesFor(loan, brokerSle);
auto const fullPaymentComponents =
computeFullPayment(asset, view, loan, periodicRate, amount, managementFeeRate, j);
// computeFullPayment only ever fails with a genuine error TER (never
// tesSUCCESS), so there is no separate "no-op" outcome to handle here.
if (fullPaymentComponents.has_value())
return doPayment(*fullPaymentComponents, loan);
return std::unexpected(fullPaymentComponents.error());
}
/* Handles a late payment (past due date, with the late-payment flag set).
* Implements the "late payment" branch of the make_payment function from
* the XLS-66 spec, Section 3.2.4.4.
*/
std::expected<LoanPaymentParts, TER>
makeLatePayment(
Asset const& asset,
ApplyView const& view,
SLE::ref loan,
SLE::const_ref brokerSle,
STAmount const& amount,
beast::Journal j)
{
auto const [managementFeeRate, periodicRate] = loanRatesFor(loan, brokerSle);
Number const serviceFee = loan->at(sfLoanServiceFee);
ExtendedPaymentComponents const periodic{
computePaymentComponents(view.rules(), asset, loan, periodicRate, managementFeeRate),
serviceFee};
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::detail::makeLatePayment",
"regular payment valid principal");
auto const latePaymentComponents =
computeLatePayment(asset, view, loan, periodic, amount, managementFeeRate, j);
// computeLatePayment only ever fails with a genuine error TER (never
// tesSUCCESS), so there is no separate "no-op" outcome to handle here.
if (latePaymentComponents.has_value())
return doPayment(*latePaymentComponents, loan);
return std::unexpected(latePaymentComponents.error());
}
/* Handles regular scheduled payments, including an optional overpayment tail.
* Implements the "regular" and "overpayment" branches of the make_payment
* function from the XLS-66 spec, Section 3.2.4.4.
*/
std::expected<LoanPaymentParts, TER>
makeRegularPayment(
Asset const& asset,
ApplyView const& view,
SLE::ref loan,
SLE::const_ref brokerSle,
STAmount const& amount,
LoanPaymentType const paymentType,
beast::Journal j)
{
using namespace Lending;
XRPL_ASSERT_PARTS(
paymentType == LoanPaymentType::Regular || paymentType == LoanPaymentType::Overpayment,
"xrpl::detail::makeRegularPayment",
"regular payment type");
auto const [managementFeeRate, periodicRate] = loanRatesFor(loan, brokerSle);
std::int32_t const loanScale = loan->at(sfLoanScale);
Number const serviceFee = loan->at(sfLoanServiceFee);
ExtendedPaymentComponents periodic{
computePaymentComponents(view.rules(), asset, loan, periodicRate, managementFeeRate),
serviceFee};
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::detail::makeRegularPayment",
"regular payment valid principal");
// Keep a running total of the actual parts paid
LoanPaymentParts totalParts;
Number totalPaid = kNumZero;
std::size_t numPayments = 0;
// Cached here (rather than re-looking up loan->at(sfPaymentRemaining) at each use) since it's
// read multiple times below. It's a write-through proxy, so it still reflects doPayment's
// mutations each iteration.
auto paymentRemainingProxy = loan->at(sfPaymentRemaining);
while ((amount >= (totalPaid + periodic.totalDue)) && paymentRemainingProxy > 0 &&
numPayments < kLoanMaximumPaymentsPerTransaction)
{
// Try to make more payments
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::detail::makeRegularPayment",
"payment pays non-negative principal");
totalPaid += periodic.totalDue;
totalParts += doPayment(periodic, loan);
++numPayments;
XRPL_ASSERT_PARTS(
(periodic.specialCase == PaymentSpecialCase::Final) == (paymentRemainingProxy == 0),
"xrpl::detail::makeRegularPayment",
"final payment is the final payment");
// Don't compute the next payment if this was the last payment
if (periodic.specialCase == PaymentSpecialCase::Final)
break;
periodic = ExtendedPaymentComponents{
computePaymentComponents(view.rules(), asset, loan, periodicRate, managementFeeRate),
serviceFee};
}
if (numPayments == 0)
{
JLOG(j.warn()) << "Regular loan payment amount is insufficient. Due: " << periodic.totalDue
<< ", paid: " << amount;
return std::unexpected(tecINSUFFICIENT_PAYMENT);
}
XRPL_ASSERT_PARTS(
totalParts.principalPaid + totalParts.interestPaid + totalParts.feePaid == totalPaid,
"xrpl::detail::makeRegularPayment",
"payment parts add up");
XRPL_ASSERT_PARTS(
totalParts.valueChange == 0, "xrpl::detail::makeRegularPayment", "no value change");
// -------------------------------------------------------------
// overpayment handling
//
// If the "fixCleanup3_1_3" amendment is enabled, truncate "amount",
// at the loan scale. If the raw value is used, the overpayment
// amount could be meaningless dust. Trying to process such a small
// amount will, at best, waste time when all the result values round
// to zero. At worst, it can cause logical errors with tiny amounts
// of interest that don't add up correctly.
auto const roundedAmount = view.rules().enabled(fixCleanup3_1_3)
? roundToAsset(asset, amount, loanScale, Number::RoundingMode::TowardsZero)
: amount;
bool const overpaymentSupported =
paymentType == LoanPaymentType::Overpayment && loan->isFlag(lsfLoanOverpayment);
bool const overpaymentAllowed = //
paymentRemainingProxy > 0 && //
totalPaid < roundedAmount && //
numPayments < kLoanMaximumPaymentsPerTransaction;
if (overpaymentSupported && overpaymentAllowed)
{
TenthBips32 const overpaymentInterestRate{loan->at(sfOverpaymentInterestRate)};
TenthBips32 const overpaymentFeeRate{loan->at(sfOverpaymentFee)};
// It shouldn't be possible for the overpayment to be greater than
// totalValueOutstanding, because that would have been processed as
// another normal payment. But cap it just in case.
Number const overpaymentRaw =
std::min(roundedAmount - totalPaid, *loan->at(sfTotalValueOutstanding));
bool const fixEnabled = view.rules().enabled(fixCleanup3_2_0);
Number const overpayment = fixEnabled
? roundToAsset(asset, overpaymentRaw, loanScale, Number::RoundingMode::Downward)
: overpaymentRaw;
// Post-amendment, the rounded overpayment can be zero; pre-amendment
// it's always positive given the surrounding guards.
if (!fixEnabled || overpayment > 0)
{
ExtendedPaymentComponents const overpaymentComponents = computeOverpaymentComponents(
view.rules(),
asset,
loanScale,
overpayment,
overpaymentInterestRate,
overpaymentFeeRate,
managementFeeRate);
// Don't process an overpayment if the whole amount (or more!)
// gets eaten by fees and interest.
if (overpaymentComponents.trackedPrincipalDelta > 0)
{
XRPL_ASSERT_PARTS(
overpaymentComponents.untrackedInterest >= beast::kZero,
"xrpl::detail::makeRegularPayment",
"overpayment penalty did not reduce value of loan");
if (auto const overResult = doOverpayment(
view.rules(),
asset,
loanScale,
overpaymentComponents,
loan,
periodicRate,
managementFeeRate,
j))
{
totalParts += *overResult;
}
else if (overResult.error())
{
// error() will be the TER returned if a payment is not
// made. It will only evaluate to true if it's unsuccessful.
// Otherwise, tesSUCCESS means nothing was done, so
// continue.
return std::unexpected(overResult.error());
}
}
}
}
// Check the final results are rounded, to double-check that the
// intermediate steps were rounded.
XRPL_ASSERT(
isRounded(asset, totalParts.principalPaid, loanScale) &&
totalParts.principalPaid >= beast::kZero,
"xrpl::detail::makeRegularPayment : total principal paid is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.interestPaid, loanScale) &&
totalParts.interestPaid >= beast::kZero,
"xrpl::detail::makeRegularPayment : total interest paid is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.valueChange, loanScale),
"xrpl::detail::makeRegularPayment : loan value change is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.feePaid, loanScale) && totalParts.feePaid >= beast::kZero,
"xrpl::detail::makeRegularPayment : fee paid is valid");
return totalParts;
}
} // namespace detail
detail::LoanStateDeltas
@@ -1926,10 +1632,8 @@ constructLoanState(
}
LoanState
constructLoanState(SLE::const_ref loan)
constructRoundedLoanState(SLE::const_ref loan)
{
XRPL_ASSERT(loan && loan->getType() == ltLOAN, "xrpl::constructLoanState : valid loan SLE");
return constructLoanState(
loan->at(sfTotalValueOutstanding),
loan->at(sfPrincipalOutstanding),
@@ -2086,7 +1790,12 @@ loanMakePayment(
LoanPaymentType const paymentType,
beast::Journal j)
{
if (loan->at(sfPaymentRemaining) == 0 || loan->at(sfPrincipalOutstanding) == 0)
using namespace Lending;
auto principalOutstandingProxy = loan->at(sfPrincipalOutstanding);
auto paymentRemainingProxy = loan->at(sfPaymentRemaining);
if (paymentRemainingProxy == 0 || principalOutstandingProxy == 0)
{
// Loan complete this is already checked in LoanPay::preclaim()
// LCOV_EXCL_START
@@ -2095,6 +1804,9 @@ loanMakePayment(
// LCOV_EXCL_STOP
}
auto totalValueOutstandingProxy = loan->at(sfTotalValueOutstanding);
auto managementFeeOutstandingProxy = loan->at(sfManagementFeeOutstanding);
// Next payment due date must be set unless the loan is complete
auto nextDueDateProxy = loan->at(sfNextPaymentDueDate);
if (*nextDueDateProxy == 0)
@@ -2103,8 +1815,26 @@ loanMakePayment(
return std::unexpected(tecINTERNAL);
}
XRPL_ASSERT(
*loan->at(sfTotalValueOutstanding) > 0, "xrpl::loanMakePayment : valid total value");
std::int32_t const loanScale = loan->at(sfLoanScale);
TenthBips32 const interestRate{loan->at(sfInterestRate)};
Number const serviceFee = loan->at(sfLoanServiceFee);
TenthBips16 const managementFeeRate{brokerSle->at(sfManagementFeeRate)};
Number const periodicPayment = loan->at(sfPeriodicPayment);
auto prevPaymentDateProxy = loan->at(sfPreviousPaymentDueDate);
std::uint32_t const startDate = loan->at(sfStartDate);
std::uint32_t const paymentInterval = loan->at(sfPaymentInterval);
// Compute the periodic rate that will be used for calculations
// throughout
Number const periodicRate = loanPeriodicRate(interestRate, paymentInterval);
XRPL_ASSERT(interestRate == 0 || periodicRate > 0, "xrpl::loanMakePayment : valid rate");
XRPL_ASSERT(*totalValueOutstandingProxy > 0, "xrpl::loanMakePayment : valid total value");
view.update(loan);
@@ -2114,29 +1844,311 @@ loanMakePayment(
{
// If the payment is late, and the late flag was not set, it's not
// valid
JLOG(j.warn()) << "Loan payment is overdue. Use the tfLoanLatePayment transaction flag to "
"make a late payment. Loan was created on "
<< loan->at(sfStartDate) << ", prev payment due date is "
<< loan->at(sfPreviousPaymentDueDate) << ", next payment due date is "
<< nextDueDateProxy << ", ledger time is "
JLOG(j.warn()) << "Loan payment is overdue. Use the tfLoanLatePayment "
"transaction "
"flag to make a late payment. Loan was created on "
<< startDate << ", prev payment due date is " << prevPaymentDateProxy
<< ", next payment due date is " << nextDueDateProxy << ", ledger time is "
<< view.parentCloseTime().time_since_epoch().count();
return std::unexpected(tecEXPIRED);
}
switch (paymentType)
// -------------------------------------------------------------
// full payment handling
if (paymentType == LoanPaymentType::Full)
{
case LoanPaymentType::Full:
return detail::makeFullPayment(asset, view, loan, brokerSle, amount, j);
case LoanPaymentType::Late:
return detail::makeLatePayment(asset, view, loan, brokerSle, amount, j);
case LoanPaymentType::Regular:
case LoanPaymentType::Overpayment:
return detail::makeRegularPayment(asset, view, loan, brokerSle, amount, paymentType, j);
TenthBips32 const closeInterestRate{loan->at(sfCloseInterestRate)};
Number const closePaymentFee = roundToAsset(asset, loan->at(sfClosePaymentFee), loanScale);
LoanState const roundedLoanState = constructLoanState(
totalValueOutstandingProxy, principalOutstandingProxy, managementFeeOutstandingProxy);
auto const fullPaymentComponents = detail::computeFullPayment(
asset,
view,
principalOutstandingProxy,
managementFeeOutstandingProxy,
periodicPayment,
paymentRemainingProxy,
prevPaymentDateProxy,
startDate,
paymentInterval,
closeInterestRate,
loanScale,
roundedLoanState.interestDue,
periodicRate,
closePaymentFee,
amount,
managementFeeRate,
j);
if (fullPaymentComponents.has_value())
{
return doPayment(
*fullPaymentComponents,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
paymentRemainingProxy,
prevPaymentDateProxy,
nextDueDateProxy,
paymentInterval);
}
if (fullPaymentComponents.error())
{
// error() will be the TER returned if a payment is not made. It
// will only evaluate to true if it's unsuccessful. Otherwise,
// tesSUCCESS means nothing was done, so continue.
return std::unexpected(fullPaymentComponents.error());
}
// LCOV_EXCL_START
UNREACHABLE("xrpl::loanMakePayment : invalid full payment result");
JLOG(j.error()) << "Full payment computation failed unexpectedly.";
return std::unexpected(tecINTERNAL);
// LCOV_EXCL_STOP
}
// LCOV_EXCL_START
UNREACHABLE("xrpl::loanMakePayment : invalid payment type");
return std::unexpected(tecINTERNAL);
// LCOV_EXCL_STOP
// -------------------------------------------------------------
// compute the periodic payment info that will be needed whether the
// payment is late or regular
detail::ExtendedPaymentComponents periodic{
detail::computePaymentComponents(
view.rules(),
asset,
loanScale,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
periodicPayment,
periodicRate,
paymentRemainingProxy,
managementFeeRate),
serviceFee};
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::loanMakePayment",
"regular payment valid principal");
// -------------------------------------------------------------
// late payment handling
if (paymentType == LoanPaymentType::Late)
{
TenthBips32 const lateInterestRate{loan->at(sfLateInterestRate)};
Number const latePaymentFee = loan->at(sfLatePaymentFee);
auto const latePaymentComponents = detail::computeLatePayment(
asset,
view,
principalOutstandingProxy,
nextDueDateProxy,
periodic,
lateInterestRate,
loanScale,
latePaymentFee,
amount,
managementFeeRate,
j);
if (latePaymentComponents.has_value())
{
return doPayment(
*latePaymentComponents,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
paymentRemainingProxy,
prevPaymentDateProxy,
nextDueDateProxy,
paymentInterval);
}
if (latePaymentComponents.error())
{
// error() will be the TER returned if a payment is not made. It
// will only evaluate to true if it's unsuccessful.
return std::unexpected(latePaymentComponents.error());
}
// LCOV_EXCL_START
UNREACHABLE("xrpl::loanMakePayment : invalid late payment result");
JLOG(j.error()) << "Late payment computation failed unexpectedly.";
return std::unexpected(tecINTERNAL);
// LCOV_EXCL_STOP
}
// -------------------------------------------------------------
// regular periodic payment handling
XRPL_ASSERT_PARTS(
paymentType == LoanPaymentType::Regular || paymentType == LoanPaymentType::Overpayment,
"xrpl::loanMakePayment",
"regular payment type");
// Keep a running total of the actual parts paid
LoanPaymentParts totalParts;
Number totalPaid;
std::size_t numPayments = 0;
while ((amount >= (totalPaid + periodic.totalDue)) && paymentRemainingProxy > 0 &&
numPayments < kLoanMaximumPaymentsPerTransaction)
{
// Try to make more payments
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::loanMakePayment",
"payment pays non-negative principal");
totalPaid += periodic.totalDue;
totalParts += detail::doPayment(
periodic,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
paymentRemainingProxy,
prevPaymentDateProxy,
nextDueDateProxy,
paymentInterval);
++numPayments;
XRPL_ASSERT_PARTS(
(periodic.specialCase == detail::PaymentSpecialCase::Final) ==
(paymentRemainingProxy == 0),
"xrpl::loanMakePayment",
"final payment is the final payment");
// Don't compute the next payment if this was the last payment
if (periodic.specialCase == detail::PaymentSpecialCase::Final)
break;
periodic = detail::ExtendedPaymentComponents{
detail::computePaymentComponents(
view.rules(),
asset,
loanScale,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
periodicPayment,
periodicRate,
paymentRemainingProxy,
managementFeeRate),
serviceFee};
}
if (numPayments == 0)
{
JLOG(j.warn()) << "Regular loan payment amount is insufficient. Due: " << periodic.totalDue
<< ", paid: " << amount;
return std::unexpected(tecINSUFFICIENT_PAYMENT);
}
XRPL_ASSERT_PARTS(
totalParts.principalPaid + totalParts.interestPaid + totalParts.feePaid == totalPaid,
"xrpl::loanMakePayment",
"payment parts add up");
XRPL_ASSERT_PARTS(totalParts.valueChange == 0, "xrpl::loanMakePayment", "no value change");
// -------------------------------------------------------------
// overpayment handling
//
// If the "fixCleanup3_1_3" amendment is enabled, truncate "amount",
// at the loan scale. If the raw value is used, the overpayment
// amount could be meaningless dust. Trying to process such a small
// amount will, at best, waste time when all the result values round
// to zero. At worst, it can cause logical errors with tiny amounts
// of interest that don't add up correctly.
auto const roundedAmount = view.rules().enabled(fixCleanup3_1_3)
? roundToAsset(asset, amount, loanScale, Number::RoundingMode::TowardsZero)
: amount;
if (paymentType == LoanPaymentType::Overpayment && loan->isFlag(lsfLoanOverpayment) &&
paymentRemainingProxy > 0 && totalPaid < roundedAmount &&
numPayments < kLoanMaximumPaymentsPerTransaction)
{
TenthBips32 const overpaymentInterestRate{loan->at(sfOverpaymentInterestRate)};
TenthBips32 const overpaymentFeeRate{loan->at(sfOverpaymentFee)};
// It shouldn't be possible for the overpayment to be greater than
// totalValueOutstanding, because that would have been processed as
// another normal payment. But cap it just in case.
Number const overpaymentRaw =
std::min(roundedAmount - totalPaid, *totalValueOutstandingProxy);
bool const fixEnabled = view.rules().enabled(fixCleanup3_2_0);
Number const overpayment = fixEnabled
? roundToAsset(asset, overpaymentRaw, loanScale, Number::RoundingMode::Downward)
: overpaymentRaw;
// Post-amendment, the rounded overpayment can be zero; pre-amendment
// it's always positive given the surrounding guards.
if (!fixEnabled || overpayment > 0)
{
detail::ExtendedPaymentComponents const overpaymentComponents =
detail::computeOverpaymentComponents(
view.rules(),
asset,
loanScale,
overpayment,
overpaymentInterestRate,
overpaymentFeeRate,
managementFeeRate);
// Don't process an overpayment if the whole amount (or more!)
// gets eaten by fees and interest.
if (overpaymentComponents.trackedPrincipalDelta > 0)
{
XRPL_ASSERT_PARTS(
overpaymentComponents.untrackedInterest >= beast::kZero,
"xrpl::loanMakePayment",
"overpayment penalty did not reduce value of loan");
// Can't just use `periodicPayment` here, because it might
// change
auto periodicPaymentProxy = loan->at(sfPeriodicPayment);
if (auto const overResult = detail::doOverpayment(
view.rules(),
asset,
loanScale,
overpaymentComponents,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
periodicPaymentProxy,
periodicRate,
paymentRemainingProxy,
managementFeeRate,
j))
{
totalParts += *overResult;
}
else if (overResult.error())
{
// error() will be the TER returned if a payment is not
// made. It will only evaluate to true if it's unsuccessful.
// Otherwise, tesSUCCESS means nothing was done, so
// continue.
return std::unexpected(overResult.error());
}
}
}
}
// Check the final results are rounded, to double-check that the
// intermediate steps were rounded.
XRPL_ASSERT(
isRounded(asset, totalParts.principalPaid, loanScale) &&
totalParts.principalPaid >= beast::kZero,
"xrpl::loanMakePayment : total principal paid is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.interestPaid, loanScale) &&
totalParts.interestPaid >= beast::kZero,
"xrpl::loanMakePayment : total interest paid is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.valueChange, loanScale),
"xrpl::loanMakePayment : loan value change is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.feePaid, loanScale) && totalParts.feePaid >= beast::kZero,
"xrpl::loanMakePayment : fee paid is valid");
return totalParts;
}
} // namespace xrpl

View File

@@ -3,7 +3,6 @@
#include <test/jtx/Env.h>
#include <test/jtx/TestHelpers.h>
#include <test/jtx/amount.h>
#include <test/jtx/envconfig.h>
#include <test/jtx/fee.h>
#include <test/jtx/mpt.h>
#include <test/jtx/pay.h>
@@ -102,12 +101,6 @@ class Invariants_test : public beast::unit_test::Suite
return xrpl::test::jtx::testableAmendments() | fixCleanup3_1_3 | fixCleanup3_2_0;
}
test::jtx::Env
makeEnv(FeatureBitset features)
{
return {*this, test::jtx::envconfig(), features, nullptr, beast::Severity::Disabled};
}
/**
* Run a specific test case to put the ledger into a state that will be
* detected by an invariant. Simulates the actions of a transaction that
@@ -137,7 +130,7 @@ class Invariants_test : public beast::unit_test::Suite
TxAccount setTxAccount = TxAccount::None)
{
doInvariantCheck(
makeEnv(defaultAmendments()),
test::jtx::Env(*this, defaultAmendments()),
expectLogs,
precheck,
fee,
@@ -1501,7 +1494,7 @@ class Invariants_test : public beast::unit_test::Suite
testcase << "PermissionedDomain" + std::string(fixEnabled ? " fix" : "");
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"permissioned domain with no rules."}},
[](Account const& a1, Account const& a2, ApplyContext& ac) {
return createPermissionedDomain(ac, a1, a2, 0).get();
@@ -1514,7 +1507,7 @@ class Invariants_test : public beast::unit_test::Suite
static constexpr auto kTooBig = kMaxPermissionedDomainCredentialsArraySize + 1;
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"permissioned domain bad credentials size " + std::to_string(kTooBig)}},
[](Account const& a1, Account const& a2, ApplyContext& ac) {
return !!createPermissionedDomain(ac, a1, a2, kTooBig);
@@ -1525,7 +1518,7 @@ class Invariants_test : public beast::unit_test::Suite
testcase << "PermissionedDomain 3";
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"permissioned domain credentials aren't sorted"}},
[](Account const& a1, Account const& a2, ApplyContext& ac) {
auto slePd = createPermissionedDomain(ac, a1, a2, 0);
@@ -1549,7 +1542,7 @@ class Invariants_test : public beast::unit_test::Suite
testcase << "PermissionedDomain 4";
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"permissioned domain credentials aren't unique"}},
[](Account const& a1, Account const& a2, ApplyContext& ac) {
auto slePd = createPermissionedDomain(ac, a1, a2, 0);
@@ -1572,7 +1565,7 @@ class Invariants_test : public beast::unit_test::Suite
testcase << "PermissionedDomain Set 1";
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"permissioned domain with no rules."}},
[&](Account const& a1, Account const& a2, ApplyContext& ac) {
// create PD
@@ -1593,7 +1586,7 @@ class Invariants_test : public beast::unit_test::Suite
testcase << "PermissionedDomain Set 2";
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"permissioned domain bad credentials size " + std::to_string(kTooBig)}},
[&](Account const& a1, Account const& a2, ApplyContext& ac) {
// create PD
@@ -1624,7 +1617,7 @@ class Invariants_test : public beast::unit_test::Suite
testcase << "PermissionedDomain Set 3";
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"permissioned domain credentials aren't sorted"}},
[&](Account const& a1, Account const& a2, ApplyContext& ac) {
// create PD
@@ -1654,7 +1647,7 @@ class Invariants_test : public beast::unit_test::Suite
testcase << "PermissionedDomain Set 4";
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"permissioned domain credentials aren't unique"}},
[&](Account const& a1, Account const& a2, ApplyContext& ac) {
// create PD
@@ -1695,7 +1688,7 @@ class Invariants_test : public beast::unit_test::Suite
{
testcase << "PermissionedDomain set 2 domains ";
doInvariantCheck(
makeEnv(features),
Env(*this, features),
fixEnabled ? badMoreThan1 : emptyV,
[](Account const& a1, Account const& a2, ApplyContext& ac) {
createPermissionedDomain(ac, a1, a2);
@@ -1741,7 +1734,7 @@ class Invariants_test : public beast::unit_test::Suite
{
testcase << "PermissionedDomain set 0 domains ";
doInvariantCheck(
makeEnv(features),
Env(*this, features),
fixEnabled ? badNoDomains : emptyV,
[](Account const&, Account const&, ApplyContext&) { return true; },
XRPAmount{},
@@ -1764,7 +1757,7 @@ class Invariants_test : public beast::unit_test::Suite
env1.close();
doInvariantCheck(
makeEnv(features),
Env(*this, features),
a1,
a2,
fixEnabled ? badNoDomains : emptyV,
@@ -1805,7 +1798,7 @@ class Invariants_test : public beast::unit_test::Suite
{
testcase << "PermissionedDomain del, create domain ";
doInvariantCheck(
makeEnv(features),
Env(*this, features),
fixEnabled ? badNotDeleted : emptyV,
[](Account const& a1, Account const& a2, ApplyContext& ac) {
createPermissionedDomain(ac, a1, a2);
@@ -2002,7 +1995,7 @@ class Invariants_test : public beast::unit_test::Suite
testcase << "PermissionedDEX" + std::string(fixEnabled ? " fix" : "");
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"domain doesn't exist"}},
[](Account const& a1, Account const&, ApplyContext& ac) {
Keylet const offerKey = keylet::offer(a1.id(), 10);
@@ -2029,7 +2022,7 @@ class Invariants_test : public beast::unit_test::Suite
// missing domain ID in offer object
doInvariantCheck(
makeEnv(features),
Env(*this, features),
{{"hybrid offer is malformed"}},
[&](Account const& a1, Account const& a2, ApplyContext& ac) {
Keylet const offerKey = keylet::offer(a2.id(), 10);
@@ -4349,7 +4342,7 @@ class Invariants_test : public beast::unit_test::Suite
};
doInvariantCheck(
makeEnv(defaultAmendments() - fixCleanup3_2_0),
Env{*this, defaultAmendments() - fixCleanup3_2_0},
{},
[](Account const&, Account const&, ApplyContext&) { return true; },
XRPAmount{},
@@ -5475,7 +5468,7 @@ class Invariants_test : public beast::unit_test::Suite
// sfHighLimit issue, not the keylet currency).
testcase << "overwrite: NoXRPTrustLines" + std::string(fixEnabled ? " fix" : "");
doInvariantCheck(
makeEnv(features),
Env(*this, features),
fixEnabled ? std::vector<std::string>{{"an XRP trust line was created"}}
: std::vector<std::string>{},
[&insertOrderedTrustLinePair](Account const& a1, Account const& a2, ApplyContext& ac) {
@@ -5503,7 +5496,7 @@ class Invariants_test : public beast::unit_test::Suite
// Regression: bad deep-freeze trust line followed by a valid one.
testcase << "overwrite: NoDeepFreeze" + std::string(fixEnabled ? " fix" : "");
doInvariantCheck(
makeEnv(features),
Env(*this, features),
fixEnabled ? std::vector<std::string>{{"a trust line with deep freeze flag without "
"normal freeze was created"}}
: std::vector<std::string>{},
@@ -5537,7 +5530,7 @@ class Invariants_test : public beast::unit_test::Suite
// still fires ("a MPT issuance was created").
testcase << "overwrite: NoZeroEscrow MPT" + std::string(fixEnabled ? " fix" : "");
doInvariantCheck(
makeEnv(features),
Env(*this, features),
fixEnabled ? std::vector<std::string>{{"escrow specifies invalid amount"}}
: std::vector<std::string>{{"a MPT issuance was created"}},
[](Account const& a1, Account const&, ApplyContext& ac) {

View File

@@ -446,7 +446,7 @@ protected:
env.test.BEAST_EXPECT(loan->at(sfPeriodicPayment) == periodicPayment);
env.test.BEAST_EXPECT(loan->at(sfFlags) == flags);
auto const ls = constructLoanState(loan);
auto const ls = constructRoundedLoanState(loan);
auto const interestRate = TenthBips32{loan->at(sfInterestRate)};
auto const paymentInterval = loan->at(sfPaymentInterval);
@@ -1119,7 +1119,7 @@ protected:
// No reason for this not to exist
return;
}
auto const current = constructLoanState(loanSle);
auto const current = constructRoundedLoanState(loanSle);
auto const errors = nextTrueState - current;
log << currencyLabel << " Loan balances: "
<< "\n\tAmount taken: " << paymentComponents.trackedValueDelta
@@ -6056,7 +6056,7 @@ protected:
auto const loanSle = env.le(loanKeylet);
if (!BEAST_EXPECT(loanSle))
return;
auto const state = constructLoanState(loanSle);
auto const state = constructRoundedLoanState(loanSle);
log << "Loan state:" << std::endl;
log << " ValueOutstanding: " << state.valueOutstanding << std::endl;

View File

@@ -0,0 +1,729 @@
#include <test/nodestore/TestBase.h>
#include <test/unit_test/SuiteJournal.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/ByteUtilities.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/contract.h>
#include <xrpl/basics/safe_cast.h>
#include <xrpl/beast/unit_test/suite.h>
#include <xrpl/beast/unit_test/thread.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/temp_dir.h>
#include <xrpl/beast/xor_shift_engine.h>
#include <xrpl/config/BasicConfig.h>
#include <xrpl/config/Constants.h>
#include <xrpl/nodestore/Backend.h>
#include <xrpl/nodestore/DummyScheduler.h>
#include <xrpl/nodestore/Manager.h>
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Scheduler.h>
#include <xrpl/nodestore/Types.h>
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/split.hpp>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <functional>
#include <iomanip>
#include <ios>
#include <memory>
#include <ostream>
#include <random>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
#ifndef NODESTORE_TIMING_DO_VERIFY
#define NODESTORE_TIMING_DO_VERIFY 0
#endif
namespace xrpl::NodeStore {
std::unique_ptr<Backend>
makeBackend(Section const& config, Scheduler& scheduler, beast::Journal journal)
{
return Manager::instance().makeBackend(config, megabytes(4), scheduler, journal);
}
// Fill memory with random bits
template <class Generator>
static void
rngcpy(void* buffer, std::size_t bytes, Generator& g)
{
using result_type = Generator::result_type;
while (bytes >= sizeof(result_type))
{
auto const v = g();
memcpy(buffer, &v, sizeof(v));
buffer = reinterpret_cast<std::uint8_t*>(buffer) + sizeof(v);
bytes -= sizeof(v);
}
if (bytes > 0)
{
auto const v = g();
memcpy(buffer, &v, bytes);
}
}
// Instance of node factory produces a deterministic sequence
// of random NodeObjects within the given
class Sequence
{
private:
static constexpr auto kMinLedger = 1;
static constexpr auto kMaxLedger = 1000000;
static constexpr auto kMinSize = 250;
static constexpr auto kMaxSize = 1250;
beast::xor_shift_engine gen_;
std::uint8_t prefix_;
std::discrete_distribution<std::uint32_t> dType_;
std::uniform_int_distribution<std::uint32_t> dSize_;
public:
explicit Sequence(std::uint8_t prefix)
: prefix_(prefix)
// uniform distribution over hotLEDGER - hotTRANSACTION_NODE
// but exclude hotTRANSACTION = 2 (removed)
, dType_({1, 1, 0, 1, 1})
, dSize_(kMinSize, kMaxSize)
{
}
// Returns the n-th key
uint256
key(std::size_t n)
{
gen_.seed(n + 1);
uint256 result;
rngcpy(&*result.begin(), result.size(), gen_);
return result;
}
// Returns the n-th complete NodeObject
std::shared_ptr<NodeObject>
obj(std::size_t n)
{
gen_.seed(n + 1);
uint256 key;
auto const data = static_cast<std::uint8_t*>(&*key.begin());
*data = prefix_;
rngcpy(data + 1, key.size() - 1, gen_);
Blob value(dSize_(gen_));
rngcpy(&value[0], value.size(), gen_);
return NodeObject::createObject(
safeCast<NodeObjectType>(dType_(gen_)), std::move(value), key);
}
// returns a batch of NodeObjects starting at n
void
batch(std::size_t n, Batch& b, std::size_t size)
{
b.clear();
b.reserve(size);
while ((size--) != 0u)
b.emplace_back(obj(n++));
}
};
//----------------------------------------------------------------------------------
class Timing_test : public beast::unit_test::Suite
{
public:
static constexpr auto kMissingNodePercent = 20; // percent of fetches for missing nodes
std::size_t const defaultRepeat = 3;
#ifndef NDEBUG
std::size_t const defaultItems = 10000;
#else
std::size_t const defaultItems = 100000; // release
#endif
using clock_type = std::chrono::steady_clock;
using duration_type = std::chrono::milliseconds;
struct Params
{
std::size_t items;
std::size_t threads;
};
static std::string
toString(Section const& config)
{
std::string s;
for (auto iter = config.begin(); iter != config.end(); ++iter)
s += (iter != config.begin() ? "," : "") + iter->first + "=" + iter->second;
return s;
}
static std::string
toString(duration_type const& d)
{
std::stringstream ss;
ss << std::fixed << std::setprecision(3) << (d.count() / 1000.) << "s";
return ss.str();
}
static Section
parse(std::string s)
{
Section section;
std::vector<std::string> v;
boost::split(v, s, boost::algorithm::is_any_of(","));
section.append(v);
return section;
}
//--------------------------------------------------------------------------
// Workaround for GCC's parameter pack expansion in lambdas
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=47226
template <class Body>
class ParallelForLambda
{
private:
std::size_t const n_;
std::atomic<std::size_t>& c_;
public:
ParallelForLambda(std::size_t n, std::atomic<std::size_t>& c) : n_(n), c_(c)
{
}
template <class... Args>
void
operator()(Args&&... args)
{
Body body(args...);
for (;;)
{
auto const i = c_++;
if (i >= n_)
break;
body(i);
}
}
};
/* Execute parallel-for loop.
Constructs `number_of_threads` instances of `Body`
with `args...` parameters and runs them on individual threads
with unique loop indexes in the range [0, n).
*/
template <class Body, class... Args>
void
parallelFor(std::size_t const n, std::size_t numberOfThreads, Args const&... args)
{
std::atomic<std::size_t> c(0);
std::vector<beast::unit_test::Thread> t;
t.reserve(numberOfThreads);
for (std::size_t id = 0; id < numberOfThreads; ++id)
t.emplace_back(*this, ParallelForLambda<Body>(n, c), args...);
for (auto& _ : t)
_.join();
}
template <class Body, class... Args>
void
parallelForId(std::size_t const n, std::size_t numberOfThreads, Args const&... args)
{
std::atomic<std::size_t> c(0);
std::vector<beast::unit_test::Thread> t;
t.reserve(numberOfThreads);
for (std::size_t id = 0; id < numberOfThreads; ++id)
t.emplace_back(*this, ParallelForLambda<Body>(n, c), id, args...);
for (auto& _ : t)
_.join();
}
//--------------------------------------------------------------------------
// Insert only
void
doInsert(Section const& config, Params const& params, beast::Journal journal)
{
DummyScheduler scheduler;
auto backend = makeBackend(config, scheduler, journal);
BEAST_EXPECT(backend != nullptr);
backend->open();
class Body
{
private:
Suite& suite_;
Backend& backend_;
Sequence seq_;
public:
explicit Body(Suite& s, Backend& backend) : suite_(s), backend_(backend), seq_(1)
{
}
void
operator()(std::size_t i)
{
try
{
backend_.store(seq_.obj(i));
}
catch (std::exception const& e)
{
suite_.fail(e.what());
}
}
};
try
{
parallelFor<Body>(params.items, params.threads, std::ref(*this), std::ref(*backend));
}
catch (std::exception const&)
{
#if NODESTORE_TIMING_DO_VERIFY
backend->verify();
#endif
rethrow();
}
backend->close();
}
// Fetch existing keys
void
doFetch(Section const& config, Params const& params, beast::Journal journal)
{
DummyScheduler scheduler;
auto backend = makeBackend(config, scheduler, journal);
BEAST_EXPECT(backend != nullptr);
backend->open();
class Body
{
private:
Suite& suite_;
Backend& backend_;
Sequence seq1_;
beast::xor_shift_engine gen_;
std::uniform_int_distribution<std::size_t> dist_;
public:
Body(std::size_t id, Suite& s, Params const& params, Backend& backend)
: suite_(s), backend_(backend), seq1_(1), gen_(id + 1), dist_(0, params.items - 1)
{
}
void
operator()(std::size_t i)
{
try
{
std::shared_ptr<NodeObject> obj;
std::shared_ptr<NodeObject> result;
obj = seq1_.obj(dist_(gen_));
backend_.fetch(obj->getHash(), &result);
suite_.expect(result && isSame(result, obj));
}
catch (std::exception const& e)
{
suite_.fail(e.what());
}
}
};
try
{
parallelForId<Body>(
params.items,
params.threads,
std::ref(*this),
std::ref(params),
std::ref(*backend));
}
catch (std::exception const&)
{
#if NODESTORE_TIMING_DO_VERIFY
backend->verify();
#endif
rethrow();
}
backend->close();
}
// Perform lookups of non-existent keys
void
doMissing(Section const& config, Params const& params, beast::Journal journal)
{
DummyScheduler scheduler;
auto backend = makeBackend(config, scheduler, journal);
BEAST_EXPECT(backend != nullptr);
backend->open();
class Body
{
private:
Suite& suite_;
// Params const& params_;
Backend& backend_;
Sequence seq2_;
beast::xor_shift_engine gen_;
std::uniform_int_distribution<std::size_t> dist_;
public:
Body(std::size_t id, Suite& s, Params const& params, Backend& backend)
: suite_(s)
//, params_ (params)
, backend_(backend)
, seq2_(2)
, gen_(id + 1)
, dist_(0, params.items - 1)
{
}
void
operator()(std::size_t i)
{
try
{
auto const hash = seq2_.key(i);
std::shared_ptr<NodeObject> result;
backend_.fetch(hash, &result);
suite_.expect(!result);
}
catch (std::exception const& e)
{
suite_.fail(e.what());
}
}
};
try
{
parallelForId<Body>(
params.items,
params.threads,
std::ref(*this),
std::ref(params),
std::ref(*backend));
}
catch (std::exception const&)
{
#if NODESTORE_TIMING_DO_VERIFY
backend->verify();
#endif
rethrow();
}
backend->close();
}
// Fetch with present and missing keys
void
doMixed(Section const& config, Params const& params, beast::Journal journal)
{
DummyScheduler scheduler;
auto backend = makeBackend(config, scheduler, journal);
BEAST_EXPECT(backend != nullptr);
backend->open();
class Body
{
private:
Suite& suite_;
// Params const& params_;
Backend& backend_;
Sequence seq1_;
Sequence seq2_;
beast::xor_shift_engine gen_;
std::uniform_int_distribution<std::uint32_t> rand_;
std::uniform_int_distribution<std::size_t> dist_;
public:
Body(std::size_t id, Suite& s, Params const& params, Backend& backend)
: suite_(s)
//, params_ (params)
, backend_(backend)
, seq1_(1)
, seq2_(2)
, gen_(id + 1)
, rand_(0, 99)
, dist_(0, params.items - 1)
{
}
void
operator()(std::size_t i)
{
try
{
if (rand_(gen_) < kMissingNodePercent)
{
auto const hash = seq2_.key(dist_(gen_));
std::shared_ptr<NodeObject> result;
backend_.fetch(hash, &result);
suite_.expect(!result);
}
else
{
std::shared_ptr<NodeObject> obj;
std::shared_ptr<NodeObject> result;
obj = seq1_.obj(dist_(gen_));
backend_.fetch(obj->getHash(), &result);
suite_.expect(result && isSame(result, obj));
}
}
catch (std::exception const& e)
{
suite_.fail(e.what());
}
}
};
try
{
parallelForId<Body>(
params.items,
params.threads,
std::ref(*this),
std::ref(params),
std::ref(*backend));
}
catch (std::exception const&)
{
#if NODESTORE_TIMING_DO_VERIFY
backend->verify();
#endif
rethrow();
}
backend->close();
}
// Simulate an xrpld workload:
// Each thread randomly:
// inserts a new key
// fetches an old key
// fetches recent, possibly non existent data
void
doWork(Section const& config, Params const& params, beast::Journal journal)
{
DummyScheduler scheduler;
auto backend = makeBackend(config, scheduler, journal);
BEAST_EXPECT(backend != nullptr);
backend->setDeletePath();
backend->open();
class Body
{
private:
Suite& suite_;
Params const& params_;
Backend& backend_;
Sequence seq1_;
beast::xor_shift_engine gen_;
std::uniform_int_distribution<std::uint32_t> rand_;
std::uniform_int_distribution<std::size_t> recent_;
std::uniform_int_distribution<std::size_t> older_;
public:
Body(std::size_t id, Suite& s, Params const& params, Backend& backend)
: suite_(s)
, params_(params)
, backend_(backend)
, seq1_(1)
, gen_(id + 1)
, rand_(0, 99)
, recent_(params.items, (params.items * 2) - 1)
, older_(0, params.items - 1)
{
}
void
operator()(std::size_t i)
{
try
{
if (rand_(gen_) < 200)
{
// historical lookup
std::shared_ptr<NodeObject> obj;
std::shared_ptr<NodeObject> result;
auto const j = older_(gen_);
obj = seq1_.obj(j);
backend_.fetch(obj->getHash(), &result);
suite_.expect(result != nullptr);
suite_.expect(isSame(result, obj));
}
char p[2];
p[0] = rand_(gen_) < 50 ? 0 : 1;
p[1] = 1 - p[0];
for (char const op : p)
{
// NOLINTNEXTLINE(bugprone-switch-missing-default-case)
switch (op)
{
case 0: {
// fetch recent
std::shared_ptr<NodeObject> obj;
std::shared_ptr<NodeObject> result;
auto const j = recent_(gen_);
obj = seq1_.obj(j);
backend_.fetch(obj->getHash(), &result);
suite_.expect(!result || isSame(result, obj));
break;
}
case 1: {
// insert new
auto const j = i + params_.items;
backend_.store(seq1_.obj(j));
break;
}
}
}
}
catch (std::exception const& e)
{
suite_.fail(e.what());
}
}
};
try
{
parallelForId<Body>(
params.items,
params.threads,
std::ref(*this),
std::ref(params),
std::ref(*backend));
}
catch (std::exception const&)
{
#if NODESTORE_TIMING_DO_VERIFY
backend->verify();
#endif
rethrow();
}
backend->close();
}
//--------------------------------------------------------------------------
using test_func = void (Timing_test::*)(Section const&, Params const&, beast::Journal);
using test_list = std::vector<std::pair<std::string, test_func>>;
duration_type
doTest(test_func f, Section const& config, Params const& params, beast::Journal journal)
{
auto const start = clock_type::now();
(this->*f)(config, params, journal);
return std::chrono::duration_cast<duration_type>(clock_type::now() - start);
}
void
doTests(
std::size_t threads,
test_list const& tests,
std::vector<std::string> const& configStrings)
{
using std::setw;
int w = 8;
for (auto const& test : tests)
{
w = std::max<std::basic_string<char>::size_type>(w, test.first.size());
}
log << threads << " Thread" << (threads > 1 ? "s" : "") << ", " << defaultItems
<< " Objects" << std::endl;
{
std::stringstream ss;
ss << std::left << setw(10) << "Backend" << std::right;
for (auto const& test : tests)
ss << " " << setw(w) << test.first;
log << ss.str() << std::endl;
}
using beast::Severity;
test::SuiteJournal journal("Timing_test", *this);
for (auto const& configString : configStrings)
{
Params params{};
params.items = defaultItems;
params.threads = threads;
for (auto i = defaultRepeat; (i--) != 0u;)
{
beast::TempDir const tempDir;
Section config = parse(configString);
config.set(Keys::kPath, tempDir.path());
std::stringstream ss;
ss << std::left << setw(10) << get(config, Keys::kType, std::string())
<< std::right;
for (auto const& test : tests)
{
ss << " " << setw(w) << toString(doTest(test.second, config, params, journal));
}
ss << " " << toString(config);
log << ss.str() << std::endl;
}
}
}
void
run() override
{
testcase("Timing", beast::unit_test::AbortT::AbortOnFail);
/* Parameters:
repeat Number of times to repeat each test
items Number of objects to create in the database
*/
std::string const defaultArgs =
"type=nudb"
#if XRPL_ROCKSDB_AVAILABLE
";type=rocksdb,open_files=2000,filter_bits=12,cache_mb=256,"
"file_size_mb=8,file_size_mult=2"
#endif
;
test_list const tests = {
{"Insert", &Timing_test::doInsert},
{"Fetch", &Timing_test::doFetch},
{"Missing", &Timing_test::doMissing},
{"Mixed", &Timing_test::doMixed},
{"Work", &Timing_test::doWork}};
auto args = arg().empty() ? defaultArgs : arg();
std::vector<std::string> configStrings;
boost::split(configStrings, args, boost::algorithm::is_any_of(";"));
for (auto iter = configStrings.begin(); iter != configStrings.end();)
{
if (iter->empty())
{
iter = configStrings.erase(iter);
}
else
{
++iter;
}
}
doTests(1, tests, configStrings);
doTests(4, tests, configStrings);
doTests(8, tests, configStrings);
// do_tests (16, tests, config_strings);
}
};
BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(Timing, nodestore, xrpl, 1);
} // namespace xrpl::NodeStore