Compare commits

..
Author SHA1 Message Date
Denis Angell 6d1016ecdd fix: merge memory-fix into memory-pressure 2026-10-09 20:28:49 -04:00
Denis Angell c8c4bd3eef fix: Evict from a CLOCK ring of strong keys instead of walking buckets 2026-10-09 19:50:48 -04:00
Denis Angell d3768854fd fix: merge memory-fix into memory-pressure 2026-10-09 12:21:27 -04:00
Denis Angell f41dfae183 fix: Address third round of review findings on evictForHardCap and cgroup detection 2026-10-07 19:04:14 +00:00
Denis Angell 2c9e237547 fix: report byte budget in hard-cap eviction logs 2026-10-03 22:46:43 -04:00
Denis Angell a1d470197e fix: merge memory-fix into memory-pressure 2026-10-03 22:43:25 -04:00
Denis Angell 9a0c1dd4cc fix: clear clang-tidy and cspell findings 2026-10-03 22:35:00 -04:00
Denis Angell 4db010d37d fix: Pass the new partitions slot before byteBudget in the byte-budget cache test 2026-10-04 00:16:43 +00:00
Denis Angell 0d0aef63a9 Merge dangell7/memory-fix into dangell7/memory-pressure 2026-10-04 00:16:35 +00:00
Denis Angell 70dd0d9520 fix: Address second round of review findings on memory_limit and evictForHardCap 2026-10-03 01:05:59 +00:00
Denis Angell 20f1a7eaa4 fix: Address review findings on memory_limit and evictForHardCap 2026-10-02 22:33:21 +00:00
Denis Angell 4d2aaed51d fix: Use the renamed CacheType/MapType/ClockType aliases in evictForHardCap 2026-10-01 19:41:51 +00:00
Denis Angell 7d90ade1f4 Local merge develop into dangell7/memory-fix 2026-10-01 19:41:29 +00:00
Denis Angell 503927cfa4 Local merge develop into dangell7/memory-fix 2026-09-17 11:53:35 +00:00
Denis Angell a6130a9fae docs: show the memory_limit section form and name the cgroup limit in the budget warning 2026-09-02 15:57:43 -04:00
Denis Angell 60b7f0f20e Merge branch 'dangell7/memory-fix' into dangell7/memory-pressure 2026-08-27 21:21:57 -04:00
Denis Angell ead10f4732 fix: pin the unit test memory budget and gate the ledger cache cap 2026-08-27 21:21:54 -04:00
Denis Angell d6c119614d fix: preserve the medium-tier gate for the RocksDB bloom filter 2026-08-27 21:16:55 -04:00
Denis Angell 0c4b6c07cd fix: include json_forwards for json::UInt 2026-08-27 14:54:10 -04:00
Denis Angell a968ece8ec fix: address clang-tidy findings 2026-08-27 14:46:05 -04:00
Denis Angell fbb84fa421 Merge branch 'dangell7/memory-fix' into dangell7/memory-pressure 2026-08-27 14:33:28 -04:00
Denis Angell e554e03921 fix: address clang-tidy include-cleaner findings 2026-08-27 14:33:11 -04:00
Denis Angell a6d0f70aaa Merge branch 'dangell7/memory-fix' into dangell7/memory-pressure 2026-08-27 14:02:38 -04:00
Denis Angell d3c04fe06d Merge remote-tracking branch 'origin/develop' into dangell7/memory-fix
# Conflicts:
#	src/xrpld/core/detail/Config.cpp
2026-08-27 14:01:31 -04:00
Denis Angell 36a659905f feat: report node cache size and bytes in get_counts 2026-08-06 15:17:43 -04:00
Denis Angell 7786baf5b5 feat: bound the remaining advisory caches within the memory budget 2026-08-06 15:10:58 -04:00
Denis Angell f80e72c6f7 fix: address clang-tidy findings and review feedback 2026-08-06 15:08:42 -04:00
Denis Angell c16f18f79b fix: harden cgroup detection and rename ledger_fetch_size 2026-08-06 12:37:30 -04:00
Denis Angell a40b88436d feat: add overrides for fixed cache policy values 2026-08-06 12:20:44 -04:00
Denis Angell c65e4539f5 feat: resolve the process cgroup path for nested memory limits 2026-08-06 12:18:39 -04:00
Denis Angell 45ed9e4de7 feat: replace node_size with memory_limit 2026-08-06 11:06:51 -04:00
51 changed files with 1680 additions and 603 deletions

No files matched your search

-1
View File
@@ -166,7 +166,6 @@ words:
- legleux
- levelization
- levelized
- lgrdb
- libpb
- libxrpl
- llection
+10 -5
View File
@@ -15,21 +15,26 @@ outputs:
runs:
using: composite
steps:
# A tag names its own version. Anything else is a development build named by
# its commit hash, matching what cmake/XrplVersion.cmake derives: the head of
# a pull request rather than the merge commit GitHub creates for it.
# A tag names its own version. Anything else takes it from BuildInfo.cpp and
# appends the commit hash as build metadata, joined with a plus sign because a
# Conan version cannot contain two hyphens.
- name: Determine version
id: version
shell: bash
env:
IS_TAG: ${{ startsWith(github.ref, 'refs/tags/') }}
REF_NAME: ${{ github.ref_name }}
SHA: ${{ github.event.pull_request.head.sha || github.sha }}
SHA: ${{ github.sha }}
run: |
if [[ "${IS_TAG}" == "true" ]]; then
version="${REF_NAME}"
else
version="0.0.0-dev+${SHA:0:7}"
version="$(awk -F'"' '/versionString =/ { print $2 }' src/libxrpl/protocol/BuildInfo.cpp)"
if [[ -z "${version}" ]]; then
echo "Unable to read versionString from BuildInfo.cpp." >&2
exit 1
fi
version="${version}+${SHA:0:7}"
fi
echo "version=${version}" | tee -a "${GITHUB_OUTPUT}"
+3 -3
View File
@@ -74,7 +74,7 @@
"extra_cmake_args": "-Dvalidator_keys=ON",
"package": {
"type": "deb",
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-3a2d19f"
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-49cdc10"
}
},
{
@@ -86,7 +86,7 @@
"extra_cmake_args": "-Dvalidator_keys=ON -Dassert=ON",
"package": {
"type": "deb",
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-3a2d19f",
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-49cdc10",
"variant": "assert"
}
}
@@ -101,7 +101,7 @@
"extra_cmake_args": "-Dvalidator_keys=ON",
"package": {
"type": "rpm",
"image": "ghcr.io/xrplf/xrpld/packaging-rhel:sha-3a2d19f"
"image": "ghcr.io/xrplf/xrpld/packaging-rhel:sha-49cdc10"
}
}
]
+3 -3
View File
@@ -7,12 +7,12 @@ on:
paths:
- ".github/workflows/build-packaging-images.yml"
- "bin/install-packaging-tools.sh"
- "package/images/packaging/**"
- "package/docker/**"
pull_request:
paths:
- ".github/workflows/build-packaging-images.yml"
- "bin/install-packaging-tools.sh"
- "package/images/packaging/**"
- "package/docker/**"
workflow_dispatch:
concurrency:
@@ -44,6 +44,6 @@ jobs:
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@696384b292577293292daed06af0306d1b83bd7d
with:
image_name: xrpld/packaging-${{ matrix.distro.name }}
dockerfile: package/images/packaging/Dockerfile
dockerfile: package/docker/Dockerfile
base_image: ${{ matrix.distro.base_image }}
push: ${{ github.event_name == 'push' }}
+1 -8
View File
@@ -1,12 +1,9 @@
# When a versioned tag is pushed, this workflow:
#
# - builds and tests the release binaries
# - uploads the libxrpl recipe to the Conan remote
# - builds and tests the release binaries
# - builds the DEB and RPM packages
# - publishes those packages to the XRPLF package repositories
#
# Nothing is published unless the build passes, which is also where CMake
# rejects a tag that is not a valid version, e.g. 3.2.01.
name: Tag
on:
@@ -25,7 +22,6 @@ defaults:
jobs:
upload-recipe:
if: ${{ github.repository == 'XRPLF/rippled' }}
needs: build-test
uses: ./.github/workflows/reusable-upload-recipe.yml
secrets:
remote_username: ${{ secrets.NEXUS_REMOTE_USERNAME }}
@@ -55,6 +51,3 @@ jobs:
remote_password: ${{ secrets.NEXUS_REMOTE_PASSWORD }}
signing_key: ${{ secrets.NEXUS_PACKAGES_PRIVATE_KEY }}
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
antithesis_docker_host: ${{ secrets.ANTITHESIS_DOCKER_HOST }}
antithesis_docker_path: ${{ secrets.ANTITHESIS_DOCKER_PATH }}
antithesis_docker_credentials: ${{ secrets.ANTITHESIS_DOCKER_CREDENTIALS }}
-3
View File
@@ -130,6 +130,3 @@ jobs:
remote_password: ${{ secrets.NEXUS_REMOTE_PASSWORD }}
signing_key: ${{ secrets.NEXUS_PACKAGES_PRIVATE_KEY }}
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
antithesis_docker_host: ${{ secrets.ANTITHESIS_DOCKER_HOST }}
antithesis_docker_path: ${{ secrets.ANTITHESIS_DOCKER_PATH }}
antithesis_docker_credentials: ${{ secrets.ANTITHESIS_DOCKER_CREDENTIALS }}
@@ -196,19 +196,6 @@ jobs:
env:
BUILD_TYPE: ${{ inputs.build_type }}
CMAKE_ARGS: ${{ inputs.cmake_args }}
# GitHub creates a merge commit for a PR
# https://www.kenmuse.com/blog/the-many-shas-of-a-github-pull-request/
#
# We:
# - explicitly provide branch name
# - use `github.event.pull_request.head.sha` to get the SHA of last commit in the PR branch
#
# This way it works both for PRs and pushes to branches.
GITHUB_BRANCH_NAME: "${{ github.head_ref || github.ref_name }}"
GITHUB_HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
#
# If tag is being pushed, we use that version.
FORCE_XRPLD_VERSION: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || '' }}
run: |
cmake \
-G '${{ runner.os == 'Windows' && 'Visual Studio 18 2026' || 'Ninja' }}' \
+13 -49
View File
@@ -9,9 +9,8 @@
# never reaches Nexus
# - 'publish' uploads with the image's publish_pkg.py, doing a --dry-run
# unless 'publish: true'
# - 'docker' builds an Ubuntu image from the tested DEB, and one with the
# voidstar binary for Antithesis, pushing them to Docker Hub and to the
# Antithesis registry only with 'publish: true'
# - 'docker' builds an Ubuntu image from the tested DEB, pushing it to Docker
# Hub only with 'publish: true'
#
# Only linux/amd64 is supported; the runner is hardcoded in the jobs below.
name: Package
@@ -38,19 +37,10 @@ on:
description: "The password or token for that Nexus account."
required: false
signing_key:
description: "Armoured PGP private key used to sign the RPMs."
description: "Armoured PGP private key used to sign the RPMs. Required when publishing."
required: false
dockerhub_token:
description: "A Docker Hub organization access token for xrplf, with push access to xrplf/xrpld."
required: false
antithesis_docker_host:
description: "The host of the Antithesis container registry, e.g. us-central1-docker.pkg.dev."
required: false
antithesis_docker_path:
description: "The repository path in that registry, the image name excluded."
required: false
antithesis_docker_credentials:
description: "The JSON key of a service account with push access to that repository."
description: "A Docker Hub organization access token for xrplf, with push access to xrplf/xrpld. Required when publishing."
required: false
defaults:
@@ -257,21 +247,13 @@ jobs:
docker:
needs: [test-install-deb, test-install-rpm]
strategy:
fail-fast: false
matrix:
target: [xrpld, voidstar]
name: "docker ${{ matrix.target }}${{ !inputs.publish && ' (dry run)' || '' }}"
name: "docker${{ !inputs.publish && ' (dry run)' || '' }}"
permissions:
contents: read
runs-on: ubuntu-latest
timeout-minutes: 15
timeout-minutes: 5
env:
CONTEXT: image-context
IMAGE: ${{ matrix.target == 'voidstar' && 'xrpld-voidstar' || 'xrplf/xrpld' }}:${{ github.ref_type == 'tag' && github.ref_name || 'develop' }}
# Docker Hub is public, so a private build never reaches it;
# the Antithesis registry is not.
PUSH: ${{ inputs.publish && (matrix.target == 'voidstar' || github.event.repository.visibility == 'public') }}
IMAGE: xrplf/xrpld:${{ github.ref_type == 'tag' && github.ref_name || 'develop' }}
steps:
- name: Checkout repository
@@ -284,20 +266,13 @@ jobs:
merge-multiple: true
path: ${{ env.PACKAGE_DIR }}
- name: Download voidstar binary
if: ${{ matrix.target == 'voidstar' }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: xrpld-ubuntu-clang-debug-amd64-voidstar
path: ${{ env.CONTEXT }}
- name: Build image
env:
TARGET: ${{ matrix.target }}
CONTEXT: image-context
run: |
mkdir -p "${CONTEXT}"
find "${PACKAGE_DIR}" -type f -name 'xrpld_[0-9]*.deb' -exec cp {} "${CONTEXT}/" \;
docker build --pull --file package/images/xrpld/Dockerfile --target "${TARGET}" --tag "${IMAGE}" "${CONTEXT}"
docker build --pull --file package/image/Dockerfile --tag "${IMAGE}" "${CONTEXT}"
- name: Start the server
run: |
@@ -315,25 +290,14 @@ jobs:
docker logs "${container}"
exit 1
# Docker Hub is public, so a private build never reaches it.
- name: Log in to Docker Hub
if: ${{ env.PUSH == 'true' && matrix.target == 'xrpld' }}
if: ${{ inputs.publish && github.event.repository.visibility == 'public' }}
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: xrplf
password: ${{ secrets.dockerhub_token }}
- name: Log in to the Antithesis registry
if: ${{ env.PUSH == 'true' && matrix.target == 'voidstar' }}
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ${{ secrets.antithesis_docker_host }}
username: _json_key
password: ${{ secrets.antithesis_docker_credentials }}
- name: Push image
if: ${{ env.PUSH == 'true' }}
env:
REGISTRY: ${{ matrix.target == 'voidstar' && format('{0}/{1}/', secrets.antithesis_docker_host, secrets.antithesis_docker_path) || '' }}
run: |
docker tag "${IMAGE}" "${REGISTRY}${IMAGE}"
docker push "${REGISTRY}${IMAGE}"
if: ${{ inputs.publish && github.event.repository.visibility == 'public' }}
run: docker push "${IMAGE}"
+1
View File
@@ -28,6 +28,7 @@ Version 3.4.0 is not yet released. These changes are available in the 3.4.0 beta
### Additions in 3.4.0
- `server_info` (admin): The `node_size` field has been removed; it reported the deprecated `[node_size]` config setting, which is still accepted as an alias for `[memory_limit]` and still emits a startup warning. Admin responses now include `memory_limit`, the cache memory budget in gigabytes rounded up (0 when enforcement is disabled).
- `ledger`: `nftoken_id`, `nftoken_ids`, and `offer_id` are now included in transaction metadata when transactions are expanded (`expand`, or admin-only `full`), matching the `tx`, `account_tx`, and `subscribe` (`transactions` stream) responses. ([#5706](https://github.com/XRPLF/rippled/pull/5706))
### Bugfixes in 3.4.0
-6
View File
@@ -51,12 +51,6 @@ git checkout release/3.2.x
For a stable release, choose one of the [tagged
releases](https://github.com/XRPLF/rippled/releases).
A build reports `0.0.0-dev` with its short commit hash as build metadata, e.g.
`0.0.0-dev+0123abc`. Only builds of a release, from a tag in CI or a versioned
Conan reference, report a release version. To use another version, set the
`FORCE_XRPLD_VERSION` environment variable when running CMake, e.g.
`FORCE_XRPLD_VERSION=3.4.0`.
### Set Up Conan
Once your [development environment](./docs/build/environment.md) is ready, set
+1
View File
@@ -77,6 +77,7 @@ endif()
include(PatchNixBinary)
include(XrplSanity)
include(XrplVersion)
include(XrplSettings)
# this check has to remain in the top-level cmake because of the early return statement
if(packages_only)
+56
View File
@@ -0,0 +1,56 @@
#!/bin/bash
if [[ $# -ne 3 || "$1" == "--help" || "$1" = "-h" ]]; then
name=$(basename $0)
cat <<-USAGE
Usage: $name workbranch base/branch version
* workbranch will be created locally from base/branch. If it exists,
it will be reused, so make sure you don't overwrite any work.
* base/branch may be specified as user:branch to allow easy copying
from Github PRs.
USAGE
exit 0
fi
work="$1"
shift
base=$(echo "$1" | sed "s/:/\//")
shift
version=$1
shift
set -e
git fetch upstreams
git checkout -B "${work}" --no-track "${base}"
push=$(git rev-parse --abbrev-ref --symbolic-full-name '@{push}' \
2>/dev/null) || true
if [[ "${push}" != "" ]]; then
echo "Warning: ${push} may already exist."
fi
build=$(find -name BuildInfo.cpp)
sed 's/\(^.*versionString =\).*$/\1 "'${version}'"/' ${build} >version.cpp &&
diff "${build}" version.cpp && exit 1 ||
mv -vi version.cpp ${build}
git diff
git add ${build}
git commit -S -m "Set version to ${version}"
git log --oneline --first-parent ${base}^..
cat <<PUSH
-------------------------------------------------------------------
This script will not push. Verify everything is correct, then push
to your repo, and create a PR as described in CONTRIBUTING.md.
-------------------------------------------------------------------
PUSH
+33 -15
View File
@@ -1368,23 +1368,41 @@
#
# [node_size]
#
# Tunes the servers based on the expected load and available memory. Legal
# sizes are "tiny", "small", "medium", "large", and "huge". We recommend
# you start at the default and raise the setting if you have extra memory.
# DEPRECATED. Each size is now an alias for a [memory_limit] value:
# tiny = 4, small = 8, medium = 32, large = 64, huge = 128. Set
# [memory_limit] instead; setting this logs a warning at startup.
#
# The code attempts to automatically determine the appropriate size for
# this parameter based on the amount of RAM and the number of execution
# cores available to the server. The current decision matrix is:
# [memory_limit]
#
# | | Cores |
# |---------|------------------------|
# | RAM | 1 | 2 or 3 | ≥ 4 |
# |---------|------|--------|--------|
# | < ~8GB | tiny | tiny | tiny |
# | < ~12GB | tiny | small | small |
# | < ~16GB | tiny | small | medium |
# | < ~24GB | tiny | small | large |
# | < ~32GB | tiny | small | huge |
# The memory budget, in gigabytes, that the server sizes its caches
# within. Cache sizes scale with the budget; the SHAMap tree node cache
# is capped to fit within half of it, enforced as it grows. Defaults to
# detected physical RAM (capped by the container limit when one is set);
# 0 selects minimal sizes with no enforcement. Values above 1024 are
# rejected, and a value above detected RAM logs a warning.
# Set this when xrpld shares the machine with other services or runs in
# a container with a memory limit below the host's RAM. Thread counts
# are unrelated: they come from the core count and the [workers] /
# [io_workers] overrides.
#
# Example:
# [memory_limit]
# 16
#
# [tree_cache_age]
#
# Seconds a SHAMap tree node stays cached after its last use. The default
# is 300. Accepted values are 10 to 3600.
#
# [ledger_cache_age]
#
# Seconds a full ledger stays in the ledger cache after its last use. The
# default is 180. Accepted values are 10 to 3600.
#
# [ledger_fetch_size]
#
# How many historical ledgers to acquire per fetch pass while backfilling.
# The default is 4. Accepted values are 1 to 16.
#
# [signing_support]
#
+28
View File
@@ -0,0 +1,28 @@
include_guard()
set(GIT_BUILD_BRANCH "")
set(GIT_COMMIT_HASH "")
find_package(Git)
if(NOT Git_FOUND)
message(WARNING "Git not found. Git branch and commit hash will be empty.")
return()
endif()
set(GIT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.git)
execute_process(
COMMAND
${GIT_EXECUTABLE} --git-dir=${GIT_DIRECTORY} rev-parse --abbrev-ref HEAD
OUTPUT_STRIP_TRAILING_WHITESPACE
OUTPUT_VARIABLE GIT_BUILD_BRANCH
)
execute_process(
COMMAND ${GIT_EXECUTABLE} --git-dir=${GIT_DIRECTORY} rev-parse HEAD
OUTPUT_STRIP_TRAILING_WHITESPACE
OUTPUT_VARIABLE GIT_COMMIT_HASH
)
message(STATUS "Git branch: ${GIT_BUILD_BRANCH}")
message(STATUS "Git commit hash: ${GIT_COMMIT_HASH}")
+1 -6
View File
@@ -81,7 +81,7 @@ include(target_link_modules)
add_module(xrpl beast)
target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main)
include(XrplVersion)
include(GitInfo)
add_module(xrpl git)
target_compile_definitions(
xrpl.libxrpl.git
@@ -111,11 +111,6 @@ target_link_libraries(
xrpl.libxrpl.protocol
PUBLIC xrpl.libxrpl.crypto xrpl.libxrpl.git xrpl.libxrpl.json
)
# Only on BuildInfo.cpp, so a new version does not rebuild the whole module.
set_source_files_properties(
${CMAKE_CURRENT_SOURCE_DIR}/src/libxrpl/protocol/BuildInfo.cpp
PROPERTIES COMPILE_DEFINITIONS XRPLD_VERSION="${XRPLD_VERSION}"
)
# Level 05
add_module(xrpl protocol_autogen)
+11 -71
View File
@@ -1,75 +1,15 @@
find_package(Git)
#[===================================================================[
read version from source
#]===================================================================]
set(GIT_BUILD_BRANCH "")
set(GIT_COMMIT_HASH "")
if(DEFINED ENV{GITHUB_BRANCH_NAME})
set(GIT_BUILD_BRANCH $ENV{GITHUB_BRANCH_NAME})
set(GIT_COMMIT_HASH $ENV{GITHUB_HEAD_SHA})
elseif(Git_FOUND AND EXISTS "${CMAKE_CURRENT_LIST_DIR}/../.git")
execute_process(
COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/..
OUTPUT_VARIABLE GIT_BUILD_BRANCH
OUTPUT_STRIP_TRAILING_WHITESPACE
COMMAND_ERROR_IS_FATAL ANY
)
execute_process(
COMMAND ${GIT_EXECUTABLE} rev-parse HEAD
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/..
OUTPUT_VARIABLE GIT_COMMIT_HASH
OUTPUT_STRIP_TRAILING_WHITESPACE
COMMAND_ERROR_IS_FATAL ANY
)
endif()
message(STATUS "Git branch: ${GIT_BUILD_BRANCH}")
message(STATUS "Git commit hash: ${GIT_COMMIT_HASH}")
if(
DEFINED ENV{FORCE_XRPLD_VERSION}
AND NOT "$ENV{FORCE_XRPLD_VERSION}" STREQUAL ""
)
message(
STATUS
"Using explicitly provided '$ENV{FORCE_XRPLD_VERSION}' as xrpld version"
)
set(XRPLD_VERSION "$ENV{FORCE_XRPLD_VERSION}")
# The rules beast::SemanticVersion::parse applies, so that an invalid version
# fails here rather than when xrpld starts. It reads each number as an int.
set(SEMVER_NUMBER "(0|[1-9][0-9]*)")
set(SEMVER_PRE_RELEASE "[A-Za-z1-9-][A-Za-z0-9-]*")
set(SEMVER_METADATA "[A-Za-z0-9-]+")
set(SEMVER_NUMBER_MAX 2147483647)
if(
NOT XRPLD_VERSION
MATCHES
"^${SEMVER_NUMBER}\\.${SEMVER_NUMBER}\\.${SEMVER_NUMBER}(-${SEMVER_PRE_RELEASE}(\\.${SEMVER_PRE_RELEASE})*)?(\\+${SEMVER_METADATA}(\\.${SEMVER_METADATA})*)?$"
OR CMAKE_MATCH_1 GREATER SEMVER_NUMBER_MAX
OR CMAKE_MATCH_2 GREATER SEMVER_NUMBER_MAX
OR CMAKE_MATCH_3 GREATER SEMVER_NUMBER_MAX
)
message(
FATAL_ERROR
"FORCE_XRPLD_VERSION '${XRPLD_VERSION}' is not a semantic version xrpld accepts, see https://semver.org"
)
file(STRINGS src/libxrpl/protocol/BuildInfo.cpp BUILD_INFO)
foreach(line_ ${BUILD_INFO})
if(line_ MATCHES "versionString[ ]*=[ ]*\"(.+)\"")
set(xrpld_version ${CMAKE_MATCH_1})
endif()
endforeach()
if(xrpld_version)
message(STATUS "xrpld version: ${xrpld_version}")
else()
message(STATUS "Using '0.0.0-dev+<git short rev>' as xrpld version")
if(GIT_COMMIT_HASH STREQUAL "")
message(
FATAL_ERROR
"Unable to determine xrpld version without git, set FORCE_XRPLD_VERSION"
)
endif()
string(SUBSTRING ${GIT_COMMIT_HASH} 0 7 GIT_COMMIT_HASH_SHORT)
set(XRPLD_VERSION "0.0.0-dev+${GIT_COMMIT_HASH_SHORT}")
message(FATAL_ERROR "unable to determine xrpld version")
endif()
message(STATUS "Build version: ${XRPLD_VERSION}")
+7 -16
View File
@@ -1,13 +1,9 @@
import os
import re
from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
from conan.tools.env import Environment
from conan import ConanFile
DEV_VERSION = "0.0.0-dev"
class Xrpl(ConanFile):
name = "xrpl"
@@ -124,7 +120,13 @@ class Xrpl(ConanFile):
}
def set_version(self):
self.version = self.version or DEV_VERSION
if self.version is None:
path = f"{self.recipe_folder}/src/libxrpl/protocol/BuildInfo.cpp"
regex = r"versionString\s?=\s?\"(.*)\""
with open(path, encoding="utf-8") as file:
matches = (re.search(regex, line) for line in file)
match = next(m for m in matches if m)
self.version = match.group(1)
def configure(self):
if self.settings.compiler == "apple-clang":
@@ -167,17 +169,6 @@ class Xrpl(ConanFile):
generators = "CMakeDeps"
def generate(self):
# The sources in the Conan cache have no git history, so the version
# comes from the reference, unless it is not one, like 'develop'.
if not os.path.exists(os.path.join(self.source_folder, ".git")):
version = str(self.version)
env = Environment()
env.define(
"FORCE_XRPLD_VERSION",
version if re.match(r"\d+\.\d+\.\d+", version) else DEV_VERSION,
)
env.vars(self).save_script("xrpld_version")
tc = CMakeToolchain(self)
tc.variables["tests"] = self.options.tests
tc.variables["benchmark"] = self.options.benchmark
+1 -1
View File
@@ -4,7 +4,7 @@
the `xrpld` DEB package installed on Ubuntu 26.04, running as the `xrpld` user.
Each release is tagged with its version, `xrplf/xrpld:<version>`, and
`xrplf/xrpld:develop` follows the `develop` branch.
See [`package/README.md`](../package/README.md#docker-images) for how it is built
See [`package/README.md`](../package/README.md#docker-image) for how it is built
and tagged.
```bash
+122 -1
View File
@@ -15,9 +15,11 @@
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <deque>
#include <functional>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <thread>
#include <type_traits>
@@ -74,13 +76,39 @@ public:
using SharedPointerType = SharedPointer;
public:
/**
* A byte budget for the strongly-cached entries. Each entry is charged
* `cost(ptr)` bytes when it becomes strong; growth past `bytes` evicts
* like the entry cap. The cost should include the value's heap payload
* plus per-entry overhead.
*/
struct ByteBudget
{
std::size_t bytes = 0;
std::function<std::size_t(SharedPointerType const&)> cost;
};
/**
* @param cacheHardCap When positive, a hard upper bound on the number of
* strongly-cached entries, enforced by demoting the approximately
* oldest entry whenever growth would exceed it. 0 disables the cap
* (the periodic sweep alone bounds the cache).
* @param partitions Number of partitions the underlying map is split
* into; defaults to the hardware concurrency. Exposed so tests can
* pin a small, known partition count.
* @param byteBudget When set, a hard upper bound on the charged bytes of
* strongly-cached entries, enforced the same way.
*/
TaggedCache(
std::string const& name,
int size,
ClockType::duration expiration,
ClockType& clock,
beast::Journal journal,
beast::insight::Collector::Ptr const& collector = beast::insight::NullCollector::make());
beast::insight::Collector::Ptr const& collector = beast::insight::NullCollector::make(),
int cacheHardCap = 0,
std::optional<std::size_t> partitions = std::nullopt,
std::optional<ByteBudget> byteBudget = std::nullopt);
public:
/**
@@ -101,6 +129,20 @@ public:
int
getTrackSize() const;
/**
* Returns the charged bytes of strongly-cached entries; 0 unless a
* byte budget with a cost function is configured.
*/
std::size_t
getCacheBytes() const;
/**
* Returns the number of strong-key slots evictForHardCap has examined
* since construction.
*/
std::uint64_t
getEvictVisits() const;
float
getHitRate();
@@ -314,6 +356,13 @@ private:
public:
SharedWeakComboPointerType ptr;
ClockType::time_point lastAccess;
// Stamped by queueStrong each time the entry becomes strong; the
// strong-key slot carrying the same seq is the entry's live slot.
std::uint64_t strongSeq{0};
// Bytes charged against the byte budget while strong; 0 when weak
// or when no budget is configured.
std::uint32_t costBytes{0};
ValueEntry(ClockType::time_point const& lastAccess, SharedPointerType const& ptr)
: ptr(ptr), lastAccess(lastAccess)
@@ -357,6 +406,27 @@ private:
using CacheType = HardenedPartitionedHashMap<key_type, Entry, Hash, KeyEqual>;
// Counts an entry that just became strong and, under an entry cap or a
// byte budget, queues its key for eviction and evicts back within both.
// Caller holds mutex_.
void
addStrong(CacheType::Iterator const& it);
// Stamps the entry with a new strong seq and appends its key to
// strongRing_, dropping stale slots once they outnumber the live ones.
// Caller holds mutex_.
void
queueStrong(key_type const& key, Entry& entry);
// CLOCK eviction over strongRing_: pops slots from the front, discards
// stale ones, gives `keep` and entries used since they became strong a
// second chance at the back, and demotes the first other strong entry,
// repeating until the count is within cacheHardCap_ and the charged
// bytes within the byte budget, or the demotion budget is spent.
// Amortized O(1) per insert. No-op for key caches; caller holds mutex_.
void
evictForHardCap(CacheType::Iterator const& keep);
[[nodiscard]] std::thread
sweepHelper(
ClockType::time_point const& whenExpire,
@@ -364,6 +434,7 @@ private:
KeyValueCacheType::MapType& partition,
SweptPointersVector& stuffToSweep,
std::atomic<int>& allRemovals,
std::atomic<std::uint64_t>& allBytesRemoved,
std::scoped_lock<std::recursive_mutex> const&);
[[nodiscard]] std::thread
@@ -373,6 +444,7 @@ private:
KeyOnlyCacheType::MapType& partition,
SweptPointersVector&,
std::atomic<int>& allRemovals,
std::atomic<std::uint64_t>& allBytesRemoved,
std::scoped_lock<std::recursive_mutex> const&);
beast::Journal journal_;
@@ -390,8 +462,57 @@ private:
// Desired maximum cache age
ClockType::duration const targetAge_;
// Hard upper bound on strongly-cached entries, enforced by
// evictForHardCap whenever the strong count grows (fresh inserts and
// weak-to-strong revivals). 0 disables it (sweep-only sizing).
int const cacheHardCap_;
// Byte-denominated bound on strongly-cached entries, enforced the same
// way; each entry is charged by the budget's cost function while strong.
std::optional<ByteBudget> const byteBudget_;
// Charged bytes of strongly-cached entries (under mutex_).
std::size_t cacheBytes_{0};
// Total hard-cap evictions (under mutex_); the first marks saturation
// onset for logging.
std::uint64_t hardCapEvictions_{0};
// Number of items cached
int cacheCount_{0};
// Charge or release an entry's bytes against the byte budget. No-ops
// without a configured budget; callers hold mutex_.
void
chargeEntry(ValueEntry& entry, SharedPointerType const& ptr);
void
dischargeEntry(ValueEntry& entry);
// True when either the entry cap or the byte budget is exceeded.
[[nodiscard]] bool
overHardCap() const;
// A key, the strong seq its entry had when the slot was queued, and the
// time it was queued.
struct StrongSlot
{
key_type key;
std::uint64_t seq;
ClockType::time_point since;
};
// Keys in the order their entries became strong, consumed front-first
// by evictForHardCap. Filled only under an entry cap or a byte budget.
// A slot is stale once its entry is gone, weak, or strong again under a
// newer seq.
std::deque<StrongSlot> strongRing_;
// Source of strong seqs; advanced under mutex_.
std::uint64_t strongSeqNext_{0};
// Slots examined by evictForHardCap; read by getEvictVisits.
std::uint64_t evictVisits_{0};
CacheType cache_; // Hold strong reference to recent objects
std::uint64_t hits_{0};
std::uint64_t misses_{0};
+281 -10
View File
@@ -6,6 +6,7 @@
#include <xrpl/basics/scope.h>
#include <algorithm>
#include <limits>
namespace xrpl {
@@ -57,7 +58,10 @@ inline TaggedCache<
ClockType::duration expiration,
ClockType& clock,
beast::Journal journal,
beast::insight::Collector::Ptr const& collector)
beast::insight::Collector::Ptr const& collector,
int cacheHardCap,
std::optional<std::size_t> partitions,
std::optional<ByteBudget> byteBudget)
: journal_(journal)
, clock_(clock)
, stats_(
@@ -67,9 +71,67 @@ inline TaggedCache<
, name_(name)
, targetSize_(size)
, targetAge_(expiration)
, cacheHardCap_(cacheHardCap)
, byteBudget_(std::move(byteBudget))
, cache_(partitions)
{
}
template <
class Key,
class T,
bool IsKeyCache,
class SharedWeakUnionPointer,
class SharedPointerType,
class Hash,
class KeyEqual,
class Mutex>
inline void
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
chargeEntry(ValueEntry& entry, SharedPointerType const& ptr)
{
if (byteBudget_ && byteBudget_->cost)
{
entry.costBytes = static_cast<std::uint32_t>(std::min<std::size_t>(
byteBudget_->cost(ptr), std::numeric_limits<std::uint32_t>::max()));
cacheBytes_ += entry.costBytes;
}
}
template <
class Key,
class T,
bool IsKeyCache,
class SharedWeakUnionPointer,
class SharedPointerType,
class Hash,
class KeyEqual,
class Mutex>
inline void
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
dischargeEntry(ValueEntry& entry)
{
cacheBytes_ -= entry.costBytes;
entry.costBytes = 0;
}
template <
class Key,
class T,
bool IsKeyCache,
class SharedWeakUnionPointer,
class SharedPointerType,
class Hash,
class KeyEqual,
class Mutex>
inline bool
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
overHardCap() const
{
return (cacheHardCap_ > 0 && cacheCount_ > cacheHardCap_) ||
(byteBudget_ && cacheBytes_ > byteBudget_->bytes);
}
template <
class Key,
class T,
@@ -120,6 +182,40 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
return cacheCount_;
}
template <
class Key,
class T,
bool IsKeyCache,
class SharedWeakUnionPointer,
class SharedPointerType,
class Hash,
class KeyEqual,
class Mutex>
inline std::size_t
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
getCacheBytes() const
{
std::scoped_lock const lock(mutex_);
return cacheBytes_;
}
template <
class Key,
class T,
bool IsKeyCache,
class SharedWeakUnionPointer,
class SharedPointerType,
class Hash,
class KeyEqual,
class Mutex>
inline std::uint64_t
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
getEvictVisits() const
{
std::scoped_lock const lock(mutex_);
return evictVisits_;
}
template <
class Key,
class T,
@@ -170,7 +266,9 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
{
std::scoped_lock const lock(mutex_);
cache_.clear();
strongRing_.clear();
cacheCount_ = 0;
cacheBytes_ = 0;
}
template <
@@ -188,7 +286,9 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
{
std::scoped_lock const lock(mutex_);
cache_.clear();
strongRing_.clear();
cacheCount_ = 0;
cacheBytes_ = 0;
hits_ = 0;
misses_ = 0;
}
@@ -219,6 +319,146 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
return true;
}
template <
class Key,
class T,
bool IsKeyCache,
class SharedWeakUnionPointer,
class SharedPointerType,
class Hash,
class KeyEqual,
class Mutex>
inline void
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
addStrong(CacheType::Iterator const& it)
{
++cacheCount_;
if constexpr (!IsKeyCache)
{
if (cacheHardCap_ > 0 || byteBudget_)
{
queueStrong(it->first, it->second);
if (overHardCap())
evictForHardCap(it);
}
}
}
template <
class Key,
class T,
bool IsKeyCache,
class SharedWeakUnionPointer,
class SharedPointerType,
class Hash,
class KeyEqual,
class Mutex>
inline void
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
queueStrong(key_type const& key, Entry& entry)
{
entry.strongSeq = ++strongSeqNext_;
strongRing_.push_back({key, entry.strongSeq, clock_.now()});
// Stale slots accumulate from sweep and del; drop them once
// they outnumber the live ones so the ring stays O(strong entries).
constexpr std::size_t kRingSlack = 1024;
if (strongRing_.size() > 2 * static_cast<std::size_t>(cacheCount_) + kRingSlack)
{
std::erase_if(strongRing_, [this](StrongSlot const& s) {
auto const it = cache_.find(s.key);
return it == cache_.end() || it->second.isWeak() || it->second.strongSeq != s.seq;
});
}
}
template <
class Key,
class T,
bool IsKeyCache,
class SharedWeakUnionPointer,
class SharedPointerType,
class Hash,
class KeyEqual,
class Mutex>
inline void
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
evictForHardCap(CacheType::Iterator const& keep)
{
if constexpr (!IsKeyCache)
{
key_type const keepKey = keep->first;
// Growth paths raise the count by one at a time, so a few demotions
// per call let eviction catch up without stalling them.
constexpr int kMaxDemotionsPerCall = 8;
for (int demotions = 0; overHardCap() && demotions < kMaxDemotionsPerCall; ++demotions)
{
// Every strong entry owns one live slot, and only `keep` can be
// re-queued twice in one call, so two passes over the ring reach
// a victim whenever one exists.
bool demoted = false;
for (std::size_t budget = 2 * strongRing_.size();
!demoted && budget > 0 && !strongRing_.empty();
--budget)
{
StrongSlot const slot = strongRing_.front();
strongRing_.pop_front();
++evictVisits_;
auto it = cache_.find(slot.key);
if (it == cache_.end() || it->second.isWeak() || it->second.strongSeq != slot.seq)
continue; // stale: the entry is gone, weak, or re-queued since
if (slot.key == keepKey || it->second.lastAccess > slot.since)
{
// The newest entry stays; one used since it became
// strong gets a second chance at the back.
queueStrong(slot.key, it->second);
continue;
}
dischargeEntry(it->second);
if (it->second.ptr.useCount() == 1)
{
// Sole owner: release entirely.
cache_.erase(it);
}
else
{
// Others hold it: keep it weakly tracked.
it->second.ptr.convertToWeak();
}
--cacheCount_;
demoted = true;
// First eviction marks saturation onset; then a heartbeat
// every 100k to avoid flooding.
++hardCapEvictions_;
if (hardCapEvictions_ == 1 || hardCapEvictions_ % 100000 == 0)
{
JLOG(journal_.warn())
<< name_ << ": hard-cap eviction #" << hardCapEvictions_ << " (cap "
<< cacheHardCap_ << ", strong " << cacheCount_ << ", bytes " << cacheBytes_
<< " of budget " << (byteBudget_ ? byteBudget_->bytes : 0)
<< ") - cache saturated, growth now evicts";
}
}
if (!demoted)
{
JLOG(journal_.debug())
<< name_ << ": over hard cap or byte budget (cap " << cacheHardCap_
<< ", strong " << cacheCount_ << ", bytes " << cacheBytes_ << " of budget "
<< (byteBudget_ ? byteBudget_->bytes : 0)
<< ") but no strong entry other than the newest to demote";
return;
}
}
}
}
template <
class Key,
class T,
@@ -264,16 +504,24 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
std::vector<std::thread> workers;
workers.reserve(cache_.partitions());
std::atomic<int> allRemovals = 0;
std::atomic<std::uint64_t> allBytesRemoved = 0;
for (std::size_t p = 0; p < cache_.partitions(); ++p)
{
workers.push_back(sweepHelper(
whenExpire, now, cache_.map()[p], allStuffToSweep[p], allRemovals, lock));
whenExpire,
now,
cache_.map()[p],
allStuffToSweep[p],
allRemovals,
allBytesRemoved,
lock));
}
for (std::thread& worker : workers)
worker.join();
cacheCount_ -= allRemovals;
cacheBytes_ -= std::min<std::uint64_t>(allBytesRemoved, cacheBytes_);
}
// At this point allStuffToSweep will go out of scope outside the lock
// and decrement the reference count on each strong pointer.
@@ -312,6 +560,7 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
if (entry.isCached())
{
--cacheCount_;
dischargeEntry(entry);
entry.ptr.convertToWeak();
ret = true;
}
@@ -360,11 +609,15 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
if (cit == cache_.end())
{
cache_.emplace(
std::piecewise_construct,
std::forward_as_tuple(key),
std::forward_as_tuple(clock_.now(), data));
++cacheCount_;
auto const emplacedIt = cache_
.emplace(
std::piecewise_construct,
std::forward_as_tuple(key),
std::forward_as_tuple(clock_.now(), data))
.first;
chargeEntry(emplacedIt->second, data);
// The just-inserted entry is the newest; evictForHardCap keeps it.
addStrong(emplacedIt);
return false;
}
@@ -390,7 +643,9 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
{
if (shouldReplaceCached())
{
dischargeEntry(entry);
entry.ptr = data;
chargeEntry(entry, data);
}
else if constexpr (!replaceCached)
{
@@ -414,12 +669,14 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
data = cachedData;
}
++cacheCount_;
chargeEntry(entry, entry.ptr.getStrong());
addStrong(cit);
return true;
}
entry.ptr = data;
++cacheCount_;
chargeEntry(entry, data);
addStrong(cit);
return false;
}
@@ -695,7 +952,14 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
++misses_;
auto const [it, inserted] = cache_.emplace(digest, Entry(clock_.now(), std::move(sle)));
if (!inserted)
{
it->second.touch(clock_.now());
}
else
{
chargeEntry(it->second, it->second.ptr.getStrong());
addStrong(it);
}
return it->second.ptr.getStrong();
}
// End CachedSLEs functions.
@@ -728,7 +992,8 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
if (entry.isCached())
{
// independent of cache size, so not counted as a hit
++cacheCount_;
chargeEntry(entry, entry.ptr.getStrong());
addStrong(cit);
entry.touch(clock_.now());
return entry.ptr.getStrong();
}
@@ -781,11 +1046,13 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
KeyValueCacheType::MapType& partition,
SweptPointersVector& stuffToSweep,
std::atomic<int>& allRemovals,
std::atomic<std::uint64_t>& allBytesRemoved,
std::scoped_lock<std::recursive_mutex> const&)
{
return std::thread([&, this]() {
int cacheRemovals = 0;
int mapRemovals = 0;
std::uint64_t bytesRemoved = 0;
// Keep references to all the stuff we sweep
// so that we can destroy them outside the lock.
@@ -812,6 +1079,8 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
{
// strong, expired
++cacheRemovals;
bytesRemoved += cit->second.costBytes;
cit->second.costBytes = 0;
if (cit->second.ptr.useCount() == 1)
{
stuffToSweep.emplace_back(std::move(cit->second.ptr));
@@ -841,6 +1110,7 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
}
allRemovals += cacheRemovals;
allBytesRemoved += bytesRemoved;
});
}
@@ -861,6 +1131,7 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
KeyOnlyCacheType::MapType& partition,
SweptPointersVector&,
std::atomic<int>& allRemovals,
std::atomic<std::uint64_t>&,
std::scoped_lock<std::recursive_mutex> const&)
{
return std::thread([&, this]() {
+5
View File
@@ -22,11 +22,14 @@ struct Sections
static constexpr auto kIoWorkers = "io_workers";
static constexpr auto kIps = "ips";
static constexpr auto kIpsFixed = "ips_fixed";
static constexpr auto kLedgerCacheAge = "ledger_cache_age";
static constexpr auto kLedgerFetchSize = "ledger_fetch_size";
static constexpr auto kLedgerHistory = "ledger_history";
static constexpr auto kLedgerReplay = "ledger_replay";
static constexpr auto kLedgerTxTables = "ledger_tx_tables";
static constexpr auto kMaxSubscriptionsPerConnection = "max_subscriptions_per_connection";
static constexpr auto kMaxTransactions = "max_transactions";
static constexpr auto kMemoryLimit = "memory_limit";
static constexpr auto kNetworkId = "network_id";
static constexpr auto kNetworkQuorum = "network_quorum";
static constexpr auto kNodeDatabase = "node_db";
@@ -64,6 +67,7 @@ struct Sections
static constexpr auto kSslVerifyFile = "ssl_verify_file";
static constexpr auto kSweepInterval = "sweep_interval";
static constexpr auto kTransactionQueue = "transaction_queue";
static constexpr auto kTreeCacheAge = "tree_cache_age";
static constexpr auto kValidationSeed = "validation_seed";
static constexpr auto kValidatorKeys = "validator_keys";
static constexpr auto kValidatorKeyRevocation = "validator_key_revocation";
@@ -95,6 +99,7 @@ struct Keys
static constexpr auto kBgThreads = "bg_threads";
static constexpr auto kBlockSize = "block_size";
static constexpr auto kCacheAge = "cache_age";
static constexpr auto kCacheBytes = "cache_bytes";
static constexpr auto kCacheMb = "cache_mb";
static constexpr auto kCacheSize = "cache_size";
static constexpr auto kClientMaxWindowBits = "client_max_window_bits";
-4
View File
@@ -95,10 +95,6 @@ public:
private:
beast::Journal mutable journal_;
std::mutex mutable mutex_;
// Initialized to nullptr for safety. Set by load() during the second
// phase of ApplicationImp initialization. Methods that dereference
// this pointer must validate it first, since two-phase init means
// load() may not have been called yet.
DatabaseCon* connection_{};
std::unordered_set<PeerReservation, beast::Uhash<>, KeyEqual> table_;
};
+1 -1
View File
@@ -197,7 +197,7 @@ public:
return fetchSz_;
}
void
virtual void
getCountsJson(json::Value& obj);
/**
@@ -5,14 +5,18 @@
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/chrono.h>
#include <xrpl/basics/contract.h>
#include <xrpl/beast/insight/NullCollector.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/config/BasicConfig.h>
#include <xrpl/config/Constants.h>
#include <xrpl/json/json_forwards.h>
#include <xrpl/json/json_value.h>
#include <xrpl/nodestore/Backend.h>
#include <xrpl/nodestore/Database.h>
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Scheduler.h>
#include <xrpl/protocol/jss.h>
#include <chrono>
#include <cstdint>
@@ -20,6 +24,7 @@
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <utility>
namespace xrpl::node_store {
@@ -58,12 +63,33 @@ public:
if (cacheSize.has_value() || cacheAge.has_value())
{
cache_ = std::make_shared<TaggedCache<UInt256, NodeObject>>(
using Cache = TaggedCache<UInt256, NodeObject>;
// Serialized sizes are exact, so bound this cache by bytes when
// cache_bytes is set; the count target remains advisory.
std::optional<Cache::ByteBudget> byteBudget;
if (auto const bytes = config.exists(Keys::kCacheBytes)
? get<std::uint64_t>(config, Keys::kCacheBytes)
: 0)
{
// Charge the blob plus per-entry overhead (map node, weak
// tracking, control block).
byteBudget = Cache::ByteBudget{
.bytes = bytes, .cost = [](std::shared_ptr<NodeObject> const& obj) {
return (obj ? obj->getData().size() : 0) + 160;
}};
}
cache_ = std::make_shared<Cache>(
"DatabaseNodeImp",
cacheSize.value_or(0),
std::chrono::minutes(cacheAge.value_or(0)),
stopwatch(),
j);
j,
beast::insight::NullCollector::make(),
0,
std::nullopt,
std::move(byteBudget));
}
XRPL_ASSERT(
@@ -120,6 +146,17 @@ public:
void
sweep() override;
void
getCountsJson(json::Value& obj) override
{
Database::getCountsJson(obj);
if (cache_)
{
obj[jss::node_cache_size] = static_cast<json::UInt>(cache_->getCacheSize());
obj[jss::node_cache_bytes] = std::to_string(cache_->getCacheBytes());
}
}
private:
// Cache for database objects. This cache is not always initialized. Check
// for null before using.
+3 -1
View File
@@ -396,6 +396,7 @@ JSS(mean); // out: get_aggregate_price
JSS(median); // out: get_aggregate_price
JSS(median_fee); // out: TxQ
JSS(median_level); // out: TxQ
JSS(memory_limit); // out: server_info
JSS(message); // error.
JSS(meta); // out: NetworkOPs, AccountTx*, Tx
JSS(meta_blob); // out: NetworkOPs, AccountTx*, Tx
@@ -430,13 +431,14 @@ JSS(no_ripple); // out: AccountLines
JSS(no_ripple_peer); // out: AccountLines
JSS(node); // out: LedgerEntry
JSS(node_binary); // out: LedgerEntry
JSS(node_cache_bytes); // out: GetCounts
JSS(node_cache_size); // out: GetCounts
JSS(node_read_bytes); // out: GetCounts
JSS(node_read_errors); // out: GetCounts
JSS(node_read_retries); // out: GetCounts
JSS(node_reads_hit); // out: GetCounts
JSS(node_reads_total); // out: GetCounts
JSS(node_reads_duration_us); // out: GetCounts
JSS(node_size); // out: server_info
JSS(nodes); // out: VaultInfo
JSS(nodestore); // out: GetCounts
JSS(node_writes); // out: GetCounts
+49 -66
View File
@@ -10,12 +10,11 @@ a build configured with `-Dvalidator_keys=ON`.
package/
build_pkg.py Staging and build script (called by the CMake `package` target and CI)
sign_rpm.py Signs the built RPMs (called by CI when publishing)
images/
packaging/
Dockerfile Packaging image, built by `build-packaging-images.yml`; installs its tooling with `bin/install-packaging-tools.sh`
publish_pkg.py Uploads built packages to the XRPLF Nexus repositories (called by CI, and shipped in that image)
xrpld/
Dockerfile The xrpld Docker images, installing the built DEB on Ubuntu (see "Docker images")
docker/
Dockerfile Packaging image, built by `build-packaging-images.yml`; installs its tooling with `bin/install-packaging-tools.sh`
publish_pkg.py Uploads built packages to the XRPLF Nexus repositories (called by CI, and shipped in that image)
image/
Dockerfile The xrpld Docker image, installing the built DEB on Ubuntu (see "Docker image")
rpm/
xrpld.spec RPM spec
debian/ Debian control files (control.in, lintian-overrides.in, rules, copyright, docs, links, source/format).
@@ -120,8 +119,8 @@ Caller workflows (`on-pr.yml`, `on-tag.yml`, `on-trigger.yml`) call
in the container of every distro that format targets and running the binaries
there, so one that cannot be installed never reaches Nexus.
3. `publish` uploads both artifacts, or lists what it would upload.
4. `docker` builds the [Docker images](#docker-images) from the tested DEB, and
pushes them when publishing.
4. `docker` builds the [Docker image](#docker-image) from the tested DEB, and
pushes it when publishing.
The packaging script derives the package version from the downloaded binary's
`xrpld --version` output; no CMake configure or build step is needed inside the
@@ -210,36 +209,32 @@ rejects request bodies over 100 MB, so uploads use the DNS-only host instead.
The `release-info` action decides the channel from the event, and
`publish_pkg.py` maps that channel to its repositories:
| Event | Version | Channel | DEB repository | RPM upload repository |
| ------------------------ | ------------------ | --------- | -------------- | --------------------- |
| tag | `X.Y.Z` | `stable` | `deb-stable` | `rpm-stable-hosted` |
| tag | `X.Y.Z-rcN` | `rc` | `deb-rc` | `rpm-rc-hosted` |
| tag | `X.Y.Z-bN` | `beta` | `deb-beta` | `rpm-beta-hosted` |
| tag, any other | `xrpld --version` | `custom` | `deb-custom` | `rpm-custom-hosted` |
| push to `develop` | `0.0.0-dev+<hash>` | `develop` | `deb-develop` | `rpm-develop-hosted` |
| tag, non-public codebase | _any_ | `private` | `deb-private` | `rpm-private-hosted` |
| Event | Version | Channel | DEB repository | RPM upload repository |
| ------------------------ | ----------------- | --------- | -------------- | --------------------- |
| tag | `X.Y.Z` | `stable` | `deb-stable` | `rpm-stable-hosted` |
| tag | `X.Y.Z-rcN` | `rc` | `deb-rc` | `rpm-rc-hosted` |
| tag | `X.Y.Z-bN` | `beta` | `deb-beta` | `rpm-beta-hosted` |
| tag, any other | `xrpld --version` | `custom` | `deb-custom` | `rpm-custom-hosted` |
| push to `develop` | `xrpld --version` | `develop` | `deb-develop` | `rpm-develop-hosted` |
| tag, non-public codebase | _any_ | `private` | `deb-private` | `rpm-private-hosted` |
A variant is published to the same channel under its own name, so
`xrpld-assert` never overwrites `xrpld`.
Only a tag picks a release channel. Versions sort in row order, so moving to a
more mature channel never downgrades. A tag matching none of the release
patterns, such as `X.Y.Z-hotfix1`, publishes to `custom`, which sits outside
that order.
Only a tag names a channel — do not extend that to `develop`, where
`BuildInfo.cpp`'s `versionString` moves through `-bN`, `-rcN` and even the final
version during a release cycle, which would send develop builds into `stable`.
Versions sort in row order, so moving to a more mature channel never downgrades.
A tag matching none of the release patterns, such as `X.Y.Z-hotfix1`, publishes
to `custom`, which sits outside that order.
Every untagged build reports `0.0.0-dev+<hash>`, which the `develop` channel
packages as `0.0.0~dev`, below every release.
The action also picks the package release number:
- A tag: `1`, since a tag's version is never reused.
- Anything else: `<run number>.<commit date>git<commit hash>`, e.g.
`857.20260826gitb6a8995`. Develop packages all share `0.0.0~dev`, so the run
number orders them, and the date and hash name the commit a package on
`packages.xrplf.org` came from.
Both reach the packaging scripts as arguments, so neither script derives
anything itself.
The action decides the package release number on the same split: a tag's version
is unique, so its packages are release 1, while develop repeats the same version
and takes `<run number>.<commit date>git<commit hash>`, e.g.
`857.20260826gitb6a8995` — the leading run number keeps each push superseding
the last, and the date and hash say which commit a package on
`packages.xrplf.org` came from. Both reach the packaging scripts as arguments,
so neither script derives anything itself.
Publishing is its own job, gated behind the install tests, uploading from the same
image that built the packages with the `publish_pkg.py` shipped in it — the
@@ -276,22 +271,14 @@ Nexus owns the repository metadata; nothing here indexes anything. Worth knowing
installs it at `/usr/local/bin/publish_pkg.py` for other XRPLF repositories that
build their packages elsewhere.
## Docker images
## Docker image
The `docker` job installs the tested `xrpld` DEB on `ubuntu:26.04` using
[`images/xrpld/Dockerfile`](images/xrpld/Dockerfile), once per target, and
checks that the server starts in each image. A tag's images are tagged with the
tag name, a develop image as `develop`. With `publish: true`:
- `xrpld` is pushed to `xrplf/xrpld` on Docker Hub using the `DOCKERHUB_TOKEN`
secret, an organization access token for `xrplf`. Private builds are never
pushed there.
- `voidstar` replaces `/usr/bin/xrpld` with the binary of the `voidstar` build
config, adds `libvoidstar.so` and links the binary into `/symbols`, as
Antithesis expects. It is pushed as `xrpld-voidstar` to the Antithesis
registry, `${ANTITHESIS_DOCKER_HOST}/${ANTITHESIS_DOCKER_PATH}`, logging in
with the `ANTITHESIS_DOCKER_CREDENTIALS` service account key. The registry is
private, so private builds are pushed too.
[`image/Dockerfile`](image/Dockerfile), checks that the server starts, and,
with `publish: true`, pushes it to `xrplf/xrpld` on Docker Hub using the
`DOCKERHUB_TOKEN` secret, an organization access token for `xrplf`. A tag's
image is tagged with the tag name, `xrplf/xrpld:<version>`, and a develop image
as `xrplf/xrpld:develop`. Private builds are never pushed.
## How `build_pkg.py` works
@@ -320,30 +307,26 @@ values; DEB writes them as `${pkg_version}-${PKG_RELEASE}` in
With `PKG_RELEASE=1`, the package metadata becomes:
| Input version | RPM version/release | Debian version |
| ------------------- | -------------------------- | ------------------ |
| `3.2.0` | `3.2.0-1%{?dist}` | `3.2.0-1` |
| `3.2.0-b1` | `3.2.0~b1-1%{?dist}` | `3.2.0~b1-1` |
| `3.2.0-rc1` | `3.2.0~rc1-1%{?dist}` | `3.2.0~rc1-1` |
| `3.2.0-custom-1` | `3.2.0~custom.1-1%{?dist}` | `3.2.0~custom.1-1` |
| `0.0.0-dev+abc1234` | `0.0.0~dev-1%{?dist}` | `0.0.0~dev-1` |
| Input version | RPM version/release | Debian version |
| ------------------ | ---------------------------- | -------------------- |
| `3.2.0` | `3.2.0-1%{?dist}` | `3.2.0-1` |
| `3.2.0-b0+abc1234` | `3.2.0~b0+abc1234-1%{?dist}` | `3.2.0~b0+abc1234-1` |
| `3.2.0-b1` | `3.2.0~b1-1%{?dist}` | `3.2.0~b1-1` |
| `3.2.0-rc1` | `3.2.0~rc1-1%{?dist}` | `3.2.0~rc1-1` |
| `3.2.0-custom-1` | `3.2.0~custom.1-1%{?dist}` | `3.2.0~custom.1-1` |
`build_pkg.py` defines `dist` as `.el9` rather than letting rpmbuild take it
from the build host, so the RHEL image can track a newer release without
changing what the packages claim to target.
The Debian changelog entry carries the channel passed as `--channel`: one of
the channels in the table above, or `UNRELEASED`, the Debian convention for a
build that targets no channel, which local and CMake builds pass. Each channel
accepts:
- `stable`, `rc`, `beta`: `X.Y.Z`, or a `bN`/`rcN` pre-release, the only kind
that may carry build metadata. `3.2.0-b1` packages as `3.2.0~b1`.
- `custom`, `private`: any version, with each `-` inside the pre-release or
build metadata switched to `.`. `3.4.0-custom-1` packages as `3.4.0~custom.1`.
- `develop`: only `0.0.0-dev`, without its build metadata. `0.0.0-dev+abc1234`
packages as `0.0.0~dev`.
- `UNRELEASED`: `0.0.0-dev` as `develop` does, anything else as `stable` does.
The Debian changelog entry carries the channel passed as `--channel`, which
only accepts the channels in the table above plus `UNRELEASED`, the Debian
convention for a build that targets no channel at all — what local and CMake
builds pass, since nothing publishes them. An unsupported pre-release, and
build metadata on a final release such as `3.2.0+abc123`, are both rejected,
except in the `custom` and `private` channels, which accept any version and
switch each `-` inside the pre-release or build metadata to `.`, so
`3.4.0-custom-1` packages as `3.4.0~custom.1`.
The RPM path intentionally uses `~` in `Version`, matching the Debian
pre-release ordering convention, so RPM filenames/NVRs begin with forms like
+4 -20
View File
@@ -24,12 +24,6 @@ PRE_RELEASE = re.compile(r"^(b|rc)(0|[1-9][0-9]*)$")
# Channels a tag of any version is published to, rather than only bN/rcN.
ANY_VERSION_CHANNELS = ("custom", "private")
# The version of any untagged build, which is all the develop channel publishes.
DEV_VERSION = "0.0.0-dev"
# Channels an untagged build is packaged for: develop, or none for a local build.
DEV_VERSION_CHANNELS = ("develop", "UNRELEASED")
# The package name a variant suffixes, and the name every variant keeps for its
# on-disk paths (/usr/bin/xrpld, /etc/xrpld, xrpld.service).
BASE_NAME = "xrpld"
@@ -80,23 +74,13 @@ def capture(*command: object) -> str:
def package_version(reported: str, channel: str) -> str:
"""Normalise a reported version into one the package formats accept.
- Release: unchanged (3.2.0 -> 3.2.0).
- bN/rcN pre-release: '-' becomes '~', to sort before the release
(3.2.0-b1 -> 3.2.0~b1).
- custom, private: any version, '-' inside the pre-release and build
metadata becomes '.' (3.4.0-custom-1 -> 3.4.0~custom.1).
- develop, UNRELEASED: 0.0.0-dev without build metadata, so builds sort by
package release, not commit hash (0.0.0-dev+abc1234 -> 0.0.0~dev).
develop accepts nothing else.
A pre-release switches to '~' (3.2.0-b1 -> 3.2.0~b1), which also sorts before
the final 3.2.0; a no-op for a final release. The custom and private
channels accept any pre-release and build metadata, with any '-' inside
either switched to '.' (3.4.0-custom-1 -> 3.4.0~custom.1).
"""
# Metadata first, as it may contain '-' too.
release, plus, metadata = reported.partition("+")
if release == DEV_VERSION and channel in DEV_VERSION_CHANNELS:
return DEV_VERSION.replace("-", "~")
assert channel != "develop", (
f"unsupported version {reported!r}: "
f"the develop channel only accepts {DEV_VERSION}."
)
base, _, pre_release = release.partition("-")
if channel in ANY_VERSION_CHANNELS:
pre_release = pre_release.replace("-", ".")
+10
View File
@@ -0,0 +1,10 @@
ARG BASE_IMAGE=debian:trixie
FROM ${BASE_IMAGE}
# Bind-mounted rather than copied in, so the installer never lands in a layer.
RUN --mount=type=bind,source=bin/install-packaging-tools.sh,target=/install-packaging-tools.sh \
/install-packaging-tools.sh
# See ../README.md, "Publishing from other repositories".
COPY package/docker/publish_pkg.py /usr/local/bin/publish_pkg.py
File renamed without changes.
+15
View File
@@ -0,0 +1,15 @@
FROM ubuntu:26.04
RUN --mount=type=bind,target=/tmp/package \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends /tmp/package/*.deb \
&& rm -rf /var/lib/apt/lists/* \
&& ln -sf /dev/stdout /var/log/xrpld/debug.log
USER xrpld
WORKDIR /var/lib/xrpld
EXPOSE 2459 5005 6006 50051
ENTRYPOINT ["/usr/bin/xrpld"]
CMD ["--net", "--silent", "--conf", "/etc/xrpld/xrpld.cfg"]
-9
View File
@@ -1,9 +0,0 @@
ARG BASE_IMAGE=debian:trixie
FROM ${BASE_IMAGE}
RUN --mount=type=bind,source=bin/install-packaging-tools.sh,target=/install-packaging-tools.sh \
/install-packaging-tools.sh
# See package/README.md, "Publishing from other repositories".
COPY package/images/packaging/publish_pkg.py /usr/local/bin/publish_pkg.py
-31
View File
@@ -1,31 +0,0 @@
FROM ubuntu:26.04 AS xrpld
RUN --mount=type=bind,target=/tmp/package \
apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends /tmp/package/*.deb \
&& rm -rf /var/lib/apt/lists/* \
&& ln -sf /dev/stdout /var/log/xrpld/debug.log
USER xrpld
WORKDIR /var/lib/xrpld
EXPOSE 2459 5005 6006 50051
ENTRYPOINT ["/usr/bin/xrpld"]
CMD ["--net", "--silent", "--conf", "/etc/xrpld/xrpld.cfg"]
# The same image with the Antithesis-instrumented binary, which loads
# libvoidstar.so; Antithesis reads the debug symbols from /symbols.
FROM xrpld AS voidstar
USER root
ADD --chmod=644 --checksum=sha256:d080cc85f1d8d96452bdaf0021a10fa3e4cc3c319840f1fadd2d33904e607095 \
https://antithesis.com/assets/instrumentation/libvoidstar.so /usr/lib/libvoidstar.so
RUN --mount=type=bind,source=xrpld,target=/tmp/xrpld \
install -m 755 /tmp/xrpld /usr/bin/xrpld \
&& mkdir /symbols \
&& ln -s /usr/bin/xrpld /symbols/xrpld
USER xrpld
+16 -11
View File
@@ -13,29 +13,34 @@
#include <string>
#include <string_view>
#ifndef XRPLD_VERSION
#error "XRPLD_VERSION must be defined"
#endif
namespace xrpl::build_info {
namespace {
// Set by cmake/XrplVersion.cmake, see http://semver.org/
constexpr char kVersionString[] = XRPLD_VERSION;
//--------------------------------------------------------------------------
// The build version number. You must edit this for each release
// and follow the format described at http://semver.org/
//------------------------------------------------------------------------------
// clang-format off
// NOLINTNEXTLINE(readability-identifier-naming)
char const* const versionString = "3.5.0-b0"
// clang-format on
;
//
// Don't touch anything below this line
//
std::string
buildVersionString()
{
std::string version = kVersionString;
std::string version = versionString;
#if defined(DEBUG) || defined(SANITIZERS)
// A version derived from git already names its commit in the metadata.
bool const hasMetadata = version.contains('+');
std::string metadata;
std::string const& commitHash = xrpl::git::getCommitHash();
if (!hasMetadata && !commitHash.empty())
if (!commitHash.empty())
metadata += commitHash + ".";
#ifdef DEBUG
@@ -51,7 +56,7 @@ buildVersionString()
#endif
if (!metadata.empty())
version += (hasMetadata ? "." : "+") + metadata;
version += "+" + metadata;
#endif
return version;
+4 -14
View File
@@ -1207,7 +1207,7 @@ public:
auto backend{node_store::Manager::instance().makeBackend(
section,
megabytes(env.app().config().getValueFor(SizedItem::BurstSize, std::nullopt)),
megabytes(env.app().config().getValueFor(SizedItem::BurstSize)),
scheduler,
env.app().getJournal("NodeStoreTest"))};
backend->open();
@@ -1229,22 +1229,12 @@ public:
// Normally, SHAMapStoreImp handles all these details.
auto nscfg = env.app().config().section(Sections::kNodeDatabase);
// Provide default values.
// Provide default values (mirrors SHAMapStoreImp::makeNodeStore).
if (!nscfg.exists(Keys::kCacheSize))
{
nscfg.set(
Keys::kCacheSize,
std::to_string(
env.app().config().getValueFor(SizedItem::TreeCacheSize, std::nullopt)));
}
nscfg.set(Keys::kCacheSize, "16384");
if (!nscfg.exists(Keys::kCacheAge))
{
nscfg.set(
Keys::kCacheAge,
std::to_string(
env.app().config().getValueFor(SizedItem::TreeCacheAge, std::nullopt)));
}
nscfg.set(Keys::kCacheAge, "5");
NodeStoreScheduler scheduler(env.app().getJobQueue());
+104
View File
@@ -595,6 +595,108 @@ main
BEAST_EXPECT(c.networkId == 10000);
}
void
testMemoryLimit()
{
testcase("memory limit");
{
Config c;
c.loadFromString("");
BEAST_EXPECT(!c.memoryLimit);
}
auto const parse = [](std::string const& value) {
Config c;
c.loadFromString("[memory_limit]\n" + value + "\n");
return c;
};
BEAST_EXPECT(parse("16").memoryLimit == std::uint64_t{16} << 30);
BEAST_EXPECT(parse("0").memoryLimit == std::uint64_t{0});
BEAST_EXPECT(parse("0").cacheMemoryBudget() == 0);
// Garbage and out-of-range values are rejected.
expectException([&parse] { parse("banana"); });
expectException([&parse] { parse("2000"); });
// Standalone mode does not change the budget: detected RAM unless
// a limit is configured.
{
Config c;
c.setupControl(true, true, true);
c.loadFromString("[memory_limit]\n8\n");
BEAST_EXPECT(c.cacheMemoryBudget() == std::uint64_t{8} << 30);
}
// Values derive from the budget: half of it at 8 KiB per entry for
// the tree cache; 0 yields the floors.
BEAST_EXPECT(parse("16").getValueFor(SizedItem::TreeCacheSize) == 1048576);
BEAST_EXPECT(parse("64").getValueFor(SizedItem::TreeCacheSize) == 4194304);
BEAST_EXPECT(parse("0").getValueFor(SizedItem::TreeCacheSize) == 16384);
BEAST_EXPECT(parse("16").getValueFor(SizedItem::TxnDbCache) == 32);
BEAST_EXPECT(parse("0").getValueFor(SizedItem::TxnDbCache) == 4);
BEAST_EXPECT(parse("16").getValueFor(SizedItem::SweepInterval) == 30);
BEAST_EXPECT(parse("16").getValueFor(SizedItem::LedgerSize) == 96);
BEAST_EXPECT(parse("16").getValueFor(SizedItem::BurstSize) == 16);
BEAST_EXPECT(parse("1024").getValueFor(SizedItem::BurstSize) == 48);
// Deprecated [node_size] tiers are aliases for budgets (by name,
// case-insensitively, or legacy 0-4 index); an explicit
// [memory_limit] wins.
auto const alias = [](std::string const& value) {
Config c;
c.loadFromString("[node_size]\n" + value + "\n");
return c;
};
BEAST_EXPECT(alias("large").cacheMemoryBudget() == std::uint64_t{64} << 30);
BEAST_EXPECT(alias("large").getValueFor(SizedItem::TreeCacheSize) == 4194304);
BEAST_EXPECT(alias("HUGE").cacheMemoryBudget() == std::uint64_t{128} << 30);
BEAST_EXPECT(alias("3").cacheMemoryBudget() == std::uint64_t{64} << 30);
BEAST_EXPECT(alias("9").cacheMemoryBudget() == std::uint64_t{128} << 30);
{
Config c;
c.loadFromString("[node_size]\nsmall\n\n[memory_limit]\n100\n");
BEAST_EXPECT(c.cacheMemoryBudget() == std::uint64_t{100} << 30);
}
// Policy values are fixed but individually overridable.
{
Config c;
c.loadFromString("[tree_cache_age]\n900\n\n[ledger_fetch_size]\n8\n");
BEAST_EXPECT(c.getValueFor(SizedItem::TreeCacheAge) == 900);
BEAST_EXPECT(c.getValueFor(SizedItem::LedgerFetch) == 8);
BEAST_EXPECT(c.getValueFor(SizedItem::LedgerAge) == 180);
}
expectException([] {
Config c;
c.loadFromString("[ledger_fetch_size]\n100\n");
});
}
void
testMountinfoPath()
{
testcase("mountinfo path");
using detail::decodeMountinfoPath;
// The kernel escapes space, tab, newline and backslash as octal.
BEAST_EXPECT(decodeMountinfoPath("/sys/fs/cgroup") == "/sys/fs/cgroup");
BEAST_EXPECT(decodeMountinfoPath("/var/lib/cgroup\\040mounts") == "/var/lib/cgroup mounts");
BEAST_EXPECT(decodeMountinfoPath("a\\011b\\012c\\134d") == "a\tb\nc\\d");
BEAST_EXPECT(decodeMountinfoPath("\\040\\040") == " ");
// Anything that is not a backslash and three octal digits is kept.
BEAST_EXPECT(decodeMountinfoPath("\\04") == "\\04");
BEAST_EXPECT(decodeMountinfoPath("\\") == "\\");
BEAST_EXPECT(decodeMountinfoPath("\\0x0") == "\\0x0");
BEAST_EXPECT(decodeMountinfoPath("\\400") == "\\400");
BEAST_EXPECT(decodeMountinfoPath("") == "");
}
void
testValidatorsFile()
{
@@ -1676,6 +1778,8 @@ r.ripple.com:51235
testAmendment();
testOverlay();
testNetworkID();
testMemoryLimit();
testMountinfoPath();
}
};
+5
View File
@@ -30,6 +30,11 @@ setupConfigForUnitTests(Config& cfg)
// The Beta API (currently v2) is always available to tests
cfg.betaRpcApi = true;
// Pin the cache memory budget so sized items (and the hard caps derived
// from them) are identical on every host; the detected-RAM default would
// make cache behavior machine-dependent.
cfg.memoryLimit = 64ull << 30;
cfg.overwrite(Sections::kNodeDatabase, Keys::kType, "memory");
cfg.overwrite(Sections::kNodeDatabase, Keys::kPath, "main");
cfg.deprecatedClearSection(Sections::kImportNodeDatabase);
+1 -6
View File
@@ -1,4 +1,3 @@
#include <xrpl/beast/core/SemanticVersion.h>
#include <xrpl/beast/unit_test/suite.h>
#include <xrpl/protocol/BuildInfo.h>
@@ -70,12 +69,8 @@ public:
auto vFF = 0xFFFF'FFFF'FFFF'FFFFLLU;
BEAST_EXPECT(!build_info::isNewerVersion(vFF));
// 1.5.9 is newer than a development build (0.0.0-dev), but not a release.
auto v159 = build_info::encodeSoftwareVersion("1.5.9");
BEAST_EXPECT(
build_info::isNewerVersion(v159) ==
(beast::SemanticVersion(build_info::getVersionString()) <
beast::SemanticVersion("1.5.9")));
BEAST_EXPECT(!build_info::isNewerVersion(v159));
auto vCurrent = build_info::getEncodedVersion();
BEAST_EXPECT(!build_info::isNewerVersion(vCurrent));
-1
View File
@@ -301,7 +301,6 @@ public:
using namespace std::chrono_literals;
Env env{*this, envconfig([](std::unique_ptr<Config> cfg) {
cfg->fees.referenceFee = 10;
cfg->nodeSize = 0;
return cfg;
})};
Account const gw{"gateway"};
+2
View File
@@ -80,6 +80,8 @@ admin = 127.0.0.1
BEAST_EXPECT(result.isMember(jss::info));
auto const& info = result[jss::info];
BEAST_EXPECT(info.isMember(jss::build_version));
// Admin request: reports the cache memory budget in GB.
BEAST_EXPECT(info.isMember(jss::memory_limit));
// Git info is not guaranteed to be present
if (info.isMember(jss::git))
{
+331
View File
@@ -4,15 +4,20 @@
#include <xrpl/basics/IntrusiveRefCounts.h>
#include <xrpl/basics/TaggedCache.ipp> // IWYU pragma: keep
#include <xrpl/basics/chrono.h>
#include <xrpl/beast/insight/NullCollector.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/protocol/Protocol.h>
#include <gtest/gtest.h>
#include <helpers/TestSink.h>
#include <cstddef>
#include <memory>
#include <optional>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
namespace xrpl {
@@ -243,4 +248,330 @@ TEST(TaggedCacheTest, tagged_cache)
}
}
TEST(TaggedCacheTest, hard_cap_enforced_on_insert)
{
using namespace std::chrono_literals;
beast::Journal const journal{TestSink::instance()};
TestStopwatch clock;
clock.set(0);
using Key = LedgerIndex;
using Value = std::string;
using Cache = TaggedCache<Key, Value>;
// A cap-enabled cache must never let the strong-cache count exceed the
// cap, enforced on the insert path alone (no sweep). The large targetSize
// and long age make the periodic sweep irrelevant here, so only
// evictForHardCap can be bounding it.
int const cap = 100;
Cache capped(
"capped", 1'000'000, 3600s, clock, journal, beast::insight::NullCollector::make(), cap);
bool everExceeded = false;
for (Key k = 1; k <= 1000; ++k)
{
capped.insert(k, "v");
if (capped.getCacheSize() > cap)
everExceeded = true;
}
EXPECT_FALSE(everExceeded);
EXPECT_LE(capped.getCacheSize(), cap);
EXPECT_GT(capped.getCacheSize(), 0);
}
TEST(TaggedCacheTest, hard_cap_enforced_across_partitions)
{
using namespace std::chrono_literals;
beast::Journal const journal{TestSink::instance()};
TestStopwatch clock;
clock.set(0);
using Key = LedgerIndex;
using Value = std::string;
using Cache = TaggedCache<Key, Value>;
// The cap is well below the partition count, so with sequential keys
// (partitioned round-robin) the partition a given insert lands in holds
// only that one entry when the count first crosses the cap: eviction
// must fall through to another partition rather than stop at an empty
// home partition and leave cacheCount_ stuck above cacheHardCap_.
int const cap = 4;
std::size_t const partitions = 16;
Cache capped(
"capped-partitions",
1'000'000,
3600s,
clock,
journal,
beast::insight::NullCollector::make(),
cap,
partitions);
bool everExceeded = false;
for (Key k = 1; k <= 2000; ++k)
{
capped.insert(k, "v");
if (capped.getCacheSize() > cap)
everExceeded = true;
}
EXPECT_FALSE(everExceeded);
EXPECT_LE(capped.getCacheSize(), cap);
EXPECT_GT(capped.getCacheSize(), 0);
}
TEST(TaggedCacheTest, hard_cap_enforced_in_sparse_partition)
{
using namespace std::chrono_literals;
beast::Journal const journal{TestSink::instance()};
TestStopwatch clock;
clock.set(0);
using Key = LedgerIndex;
using Value = std::string;
using Cache = TaggedCache<Key, Value>;
// A single partition whose bucket array grows into the thousands while
// an external owner (held in `heldRefs`, the common case for a value
// also referenced from elsewhere) keeps every demoted entry in the
// table as weakly-tracked rather than erased: the handful of strong
// survivors end up sparse among thousands of weak entries, and
// evictForHardCap must keep finding one.
int const cap = 5;
std::size_t const partitions = 1;
Cache capped(
"capped-sparse",
1'000'000,
3600s,
clock,
journal,
beast::insight::NullCollector::make(),
cap,
partitions);
std::vector<std::shared_ptr<Value>> heldRefs;
bool everExceeded = false;
for (Key k = 1; k <= 2000; ++k)
{
capped.insert(k, "v");
heldRefs.push_back(capped.fetch(k));
if (capped.getCacheSize() > cap)
everExceeded = true;
}
EXPECT_FALSE(everExceeded);
EXPECT_LE(capped.getCacheSize(), cap);
EXPECT_GT(capped.getCacheSize(), 0);
}
TEST(TaggedCacheTest, hard_cap_bounded_work_with_weak_entries)
{
using namespace std::chrono_literals;
beast::Journal const journal{TestSink::instance()};
TestStopwatch clock;
clock.set(0);
using Key = LedgerIndex;
using Value = std::string;
using Cache = TaggedCache<Key, Value>;
// Every demoted entry stays weakly tracked because `heldRefs` owns it,
// so weak entries come to outnumber strong ones 100 to 1. Eviction work
// per insert must stay constant rather than grow with the weak count.
int const cap = 100;
Key const inserts = 10'000;
Cache capped(
"capped-weak-heavy",
1'000'000,
3600s,
clock,
journal,
beast::insight::NullCollector::make(),
cap);
std::vector<std::shared_ptr<Value>> heldRefs;
bool everExceeded = false;
for (Key k = 1; k <= inserts; ++k)
{
capped.insert(k, "v");
heldRefs.push_back(capped.fetch(k));
if (capped.getCacheSize() > cap)
everExceeded = true;
}
EXPECT_FALSE(everExceeded);
EXPECT_EQ(capped.getCacheSize(), cap);
EXPECT_EQ(capped.getTrackSize(), inserts);
EXPECT_LE(capped.getEvictVisits(), 4u * inserts);
}
TEST(TaggedCacheTest, hard_cap_enforced_via_fetch_handler)
{
using namespace std::chrono_literals;
beast::Journal const journal{TestSink::instance()};
TestStopwatch clock;
clock.set(0);
using Key = LedgerIndex;
using Value = std::string;
using Cache = TaggedCache<Key, Value>;
// fetch(key, handler) is the miss path a NodeStore- or database-backed
// lookup takes; it must count and cap an inserted entry the same as the
// canonicalize path uses, not insert one uncounted and uncapped.
int const cap = 100;
Cache capped(
"capped-handler",
1'000'000,
3600s,
clock,
journal,
beast::insight::NullCollector::make(),
cap);
bool everExceeded = false;
for (Key k = 1; k <= 1000; ++k)
{
auto const ptr = capped.fetch(k, [] { return std::make_shared<Value>("v"); });
EXPECT_NE(ptr, nullptr);
if (capped.getCacheSize() > cap)
everExceeded = true;
}
EXPECT_FALSE(everExceeded);
EXPECT_LE(capped.getCacheSize(), cap);
EXPECT_GT(capped.getCacheSize(), 0);
}
TEST(TaggedCacheTest, hard_cap_disabled)
{
using namespace std::chrono_literals;
beast::Journal const journal{TestSink::instance()};
TestStopwatch clock;
clock.set(0);
using Key = LedgerIndex;
using Value = std::string;
using Cache = TaggedCache<Key, Value>;
// cacheHardCap = 0: growth is bounded only by the periodic sweep.
Cache uncapped(
"uncapped", 1'000'000, 3600s, clock, journal, beast::insight::NullCollector::make(), 0);
for (Key k = 1; k <= 1000; ++k)
uncapped.insert(k, "v");
EXPECT_EQ(uncapped.getCacheSize(), 1000);
}
TEST(TaggedCacheTest, byte_budget_enforced_on_insert)
{
using namespace std::chrono_literals;
beast::Journal const journal{TestSink::instance()};
TestStopwatch clock;
clock.set(0);
using Key = LedgerIndex;
using Value = std::string;
using Cache = TaggedCache<Key, Value>;
// Each 100-byte value is charged exactly; the 4 KiB budget holds ~40
// entries, enforced as the cache grows, with the sweep unable to fire.
Cache::ByteBudget budget{
.bytes = 4096, .cost = [](std::shared_ptr<Value> const& v) { return v ? v->size() : 0; }};
Cache capped(
"bytes",
1'000'000,
3600s,
clock,
journal,
beast::insight::NullCollector::make(),
0,
std::nullopt,
budget);
bool everExceeded = false;
for (Key k = 1; k <= 200; ++k)
{
capped.insert(k, std::string(100, 'x'));
if (capped.getCacheBytes() > 4096)
everExceeded = true;
}
EXPECT_FALSE(everExceeded);
EXPECT_LE(capped.getCacheBytes(), std::size_t{4096});
EXPECT_GT(capped.getCacheSize(), 0);
EXPECT_LT(capped.getCacheSize(), 50);
}
// A key the cache can hash, compare and partition on but not default
// construct. The requirements TaggedCache places on key_type are those
// three; the eviction path must not add a fourth.
struct ExplicitKey
{
std::size_t value;
explicit ExplicitKey(std::size_t v) : value(v)
{
}
operator std::size_t() const
{
return value;
}
bool
operator==(ExplicitKey const&) const = default;
};
static_assert(!std::is_default_constructible_v<ExplicitKey>);
struct ExplicitKeyHash
{
std::size_t
operator()(ExplicitKey const& key) const
{
return key.value;
}
};
TEST(TaggedCacheTest, hard_cap_with_non_default_constructible_key)
{
using namespace std::chrono_literals;
beast::Journal const journal{TestSink::instance()};
TestStopwatch clock;
clock.set(0);
using Value = std::string;
using Cache = TaggedCache<
ExplicitKey,
Value,
/*IsKeyCache*/ false,
SharedWeakCachePointer<Value>,
std::shared_ptr<Value>,
ExplicitKeyHash>;
int const cap = 100;
Cache capped(
"capped-explicit-key",
1'000'000,
3600s,
clock,
journal,
beast::insight::NullCollector::make(),
cap);
bool everExceeded = false;
for (std::size_t k = 1; k <= 1000; ++k)
{
capped.insert(ExplicitKey{k}, "v");
if (capped.getCacheSize() > cap)
everExceeded = true;
}
EXPECT_FALSE(everExceeded);
EXPECT_LE(capped.getCacheSize(), cap);
EXPECT_GT(capped.getCacheSize(), 0);
}
} // namespace xrpl
+8 -1
View File
@@ -10,6 +10,7 @@
#include <xrpl/basics/chrono.h>
#include <xrpl/basics/contract.h>
#include <xrpl/beast/insight/Collector.h>
#include <xrpl/beast/insight/NullCollector.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/json/json_value.h>
@@ -42,7 +43,13 @@ LedgerHistory::LedgerHistory(beast::insight::Collector::Ptr const& collector, Ap
app_.config().getValueFor(SizedItem::LedgerSize),
std::chrono::seconds{app_.config().getValueFor(SizedItem::LedgerAge)},
stopwatch(),
app_.getJournal("TaggedCache"))
app_.getJournal("TaggedCache"),
beast::insight::NullCollector::make(),
// A ledger byte size is ill-posed (nodes are shared copy-on-write
// across ledgers), so bound this cache by count. 0 = cap off when
// enforcement is disabled.
app_.config().cacheMemoryBudget() != 0 ? app_.config().getValueFor(SizedItem::LedgerSize)
: 0)
, consensusValidated_(
"ConsensusValidated",
64,
+4 -1
View File
@@ -30,6 +30,7 @@
#include <xrpl/basics/safe_cast.h>
#include <xrpl/basics/scope.h>
#include <xrpl/beast/insight/Collector.h>
#include <xrpl/beast/insight/NullCollector.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/Zero.h>
#include <xrpl/beast/utility/instrumentation.h>
@@ -137,7 +138,9 @@ LedgerMaster::LedgerMaster(
65536,
std::chrono::seconds{45},
stopwatch,
app_.getJournal("TaggedCache"))
app_.getJournal("TaggedCache"),
beast::insight::NullCollector::make(),
65536)
, stats_([this] { collectMetrics(); }, collector)
{
}
+44 -15
View File
@@ -52,6 +52,7 @@
#include <xrpl/beast/asio/io_latency_probe.h>
#include <xrpl/beast/core/LexicalCast.h>
#include <xrpl/beast/insight/Event.h>
#include <xrpl/beast/insight/NullCollector.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/PropertyStream.h>
#include <xrpl/beast/utility/instrumentation.h>
@@ -260,6 +261,10 @@ public:
std::unique_ptr<TxQ> txQ_;
ClosureCounter<void, boost::system::error_code const&> waitHandlerCounter_;
boost::asio::steady_timer sweepTimer_;
// Set by doSweep when post-trim RSS exceeds 150% of the memory budget;
// accelerates the sweep cadence until RSS retreats below 130%.
std::atomic<bool> memoryPressure_{false};
boost::asio::steady_timer entropyTimer_;
std::optional<SQLiteDatabase> relationalDatabase_;
@@ -294,9 +299,8 @@ public:
auto const cores = std::thread::hardware_concurrency();
// Use a single thread when running on under-provisioned systems
// or if we are configured to use minimal resources.
if ((cores == 1) || ((config.nodeSize == 0) && (cores == 2)))
// Use a single thread on under-provisioned systems.
if (cores <= 2)
return 1;
// Otherwise, prefer six threads.
@@ -339,14 +343,12 @@ public:
auto count = static_cast<int>(std::thread::hardware_concurrency());
// Be more aggressive about the number of threads to use
// for the job queue if the server is configured as
// "large" or "huge" if there are enough cores.
if (config->nodeSize >= 4 && count >= 16)
// Scale the job queue with the available cores.
if (count >= 16)
{
count = 6 + std::min(count, 8);
}
else if (config->nodeSize >= 3 && count >= 8)
else if (count >= 8)
{
count = 4 + std::min(count, 6);
}
@@ -371,10 +373,12 @@ public:
logs_->journal("TaggedCache"))
, cachedSLEs_(
"Cached SLEs",
0,
config_->getValueFor(SizedItem::SleCacheSize),
std::chrono::minutes(1),
stopwatch(),
logs_->journal("CachedSLEs"))
logs_->journal("CachedSLEs"),
beast::insight::NullCollector::make(),
config_->getValueFor(SizedItem::SleCacheSize))
, networkIDService_(std::make_unique<NetworkIDServiceImpl>(config_->networkId))
, validatorKeys_(*config_, journal_)
, resourceManager_(
@@ -872,7 +876,7 @@ public:
node_store::DummyScheduler dummyScheduler;
std::unique_ptr<node_store::Database> source =
node_store::Manager::instance().makeDatabase(
megabytes(config_->getValueFor(SizedItem::BurstSize, std::nullopt)),
megabytes(config_->getValueFor(SizedItem::BurstSize)),
dummyScheduler,
0,
config_->section(Sections::kImportNodeDatabase),
@@ -928,9 +932,11 @@ public:
}))
{
using namespace std::chrono;
sweepTimer_.expires_after(
seconds{config_->sweepInterval.value_or(
config_->getValueFor(SizedItem::SweepInterval))});
auto interval = seconds{
config_->sweepInterval.value_or(config_->getValueFor(SizedItem::SweepInterval))};
if (memoryPressure_)
interval = std::min(interval, seconds{10});
sweepTimer_.expires_after(interval);
sweepTimer_.async_wait(std::move(*optionalCountedHandler));
}
}
@@ -1099,7 +1105,30 @@ public:
<< "; size after: " << cachedSLEs_.size();
}
mallocTrim("doSweep", journal_);
auto const trim = mallocTrim("doSweep", journal_);
// Circuit breaker, not a control loop: byte-charged caps do the real
// bounding, and RSS legitimately lags eviction (allocator retention),
// so pressure only accelerates sweeps and warns. Hysteresis: trip at
// 150% of the budget, reset below 130%.
if (auto const budget = config_->cacheMemoryBudget(); budget != 0 && trim.rssAfterKB > 0)
{
auto const rss = static_cast<std::uint64_t>(trim.rssAfterKB) * 1024;
if (rss > budget + (budget / 2))
{
if (!memoryPressure_)
{
JLOG(journal_.warn())
<< "memory pressure: RSS " << (rss >> 20) << " MB exceeds 150% of the "
<< (budget >> 20) << " MB memory_limit; sweeping every 10s";
}
memoryPressure_ = true;
}
else if (rss < budget + (budget * 3 / 10))
{
memoryPressure_ = false;
}
}
// Set timer to do another sweep later.
setSweepTimer();
+7 -21
View File
@@ -2882,27 +2882,13 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters)
if (admin)
{
// Note: By default the node size is "tiny". When parsing it's an error if the final
// NODE_SIZE is over 4 so below code should be safe.
// NOLINTNEXTLINE(bugprone-switch-missing-default-case)
switch (registry_.get().getApp().config().nodeSize)
{
case 0:
info[jss::node_size] = "tiny";
break;
case 1:
info[jss::node_size] = "small";
break;
case 2:
info[jss::node_size] = "medium";
break;
case 3:
info[jss::node_size] = "large";
break;
case 4:
info[jss::node_size] = "huge";
break;
}
// The cache memory budget in GB, rounded up: a nonzero sub-GiB
// budget (a cgroup limit under 1 GiB, kept nonzero so enforcement
// stays on) would otherwise truncate to 0, the same value reported
// when enforcement is disabled.
constexpr std::uint64_t oneGiB = std::uint64_t{1} << 30;
auto const budget = registry_.get().getApp().config().cacheMemoryBudget();
info[jss::memory_limit] = static_cast<json::UInt>((budget + oneGiB - 1) >> 30);
auto when = registry_.get().getValidators().expires();
+14 -14
View File
@@ -122,7 +122,9 @@ SHAMapStoreImp::SHAMapStoreImp(
Keys::kCacheMb, std::to_string(config.getValueFor(SizedItem::HashNodeDbCache)));
}
if (!section.exists(Keys::kFilterBits) && (config.nodeSize >= 2))
// 32 GB is the budget the deprecated medium node_size tier maps to,
// preserving the old medium-and-up gate for the bloom filter.
if (!section.exists(Keys::kFilterBits) && config.cacheMemoryBudget() >= (32ull << 30))
section.set(Keys::kFilterBits, "10");
}
@@ -190,20 +192,18 @@ SHAMapStoreImp::makeNodeStore(int readThreads)
{
auto nscfg = app_.config().section(Sections::kNodeDatabase);
// Provide default values.
// Documented defaults: 16384 records, 5 minutes (DatabaseNodeImp reads
// cache_age in minutes).
if (!nscfg.exists(Keys::kCacheSize))
{
nscfg.set(
Keys::kCacheSize,
std::to_string(app_.config().getValueFor(SizedItem::TreeCacheSize, std::nullopt)));
}
nscfg.set(Keys::kCacheSize, "16384");
if (!nscfg.exists(Keys::kCacheAge))
{
nscfg.set(
Keys::kCacheAge,
std::to_string(app_.config().getValueFor(SizedItem::TreeCacheAge, std::nullopt)));
}
nscfg.set(Keys::kCacheAge, "5");
// An eighth of the memory budget bounds the serialized-object cache.
if (auto const budget = app_.config().cacheMemoryBudget();
budget != 0 && !nscfg.exists(Keys::kCacheBytes))
nscfg.set(Keys::kCacheBytes, std::to_string(budget / 8));
std::unique_ptr<node_store::Database> db;
@@ -235,7 +235,7 @@ SHAMapStoreImp::makeNodeStore(int readThreads)
else
{
db = node_store::Manager::instance().makeDatabase(
megabytes(app_.config().getValueFor(SizedItem::BurstSize, std::nullopt)),
megabytes(app_.config().getValueFor(SizedItem::BurstSize)),
scheduler_,
readThreads,
nscfg,
@@ -619,7 +619,7 @@ SHAMapStoreImp::makeBackendRotating(std::string path)
auto backend{node_store::Manager::instance().makeBackend(
section,
megabytes(app_.config().getValueFor(SizedItem::BurstSize, std::nullopt)),
megabytes(app_.config().getValueFor(SizedItem::BurstSize)),
scheduler_,
app_.getJournal(kNodeStoreName))};
backend->open();
+10 -31
View File
@@ -1,9 +1,7 @@
#pragma once
#include <xrpl/basics/Log.h>
#include <xrpl/basics/RangeSet.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/contract.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/protocol/ErrorCodes.h>
#include <xrpl/protocol/LedgerHeader.h>
@@ -18,7 +16,6 @@
#include <map>
#include <memory>
#include <optional>
#include <stdexcept>
#include <utility>
#include <variant>
#include <vector>
@@ -401,6 +398,15 @@ public:
SQLiteDatabase&
operator=(SQLiteDatabase&&) = delete;
/**
* @brief ledgerDbHasSpace Checks if the ledger database has available
* space.
* @param config Config object.
* @return True if space is available.
*/
bool
ledgerDbHasSpace(Config const& config);
/**
* @brief transactionDbHasSpace Checks if the transaction database has
* available space.
@@ -452,27 +458,13 @@ private:
}
/**
* @brief checkoutLedger Checks out and returns node store ledger
* @brief checkoutTransaction Checks out and returns node store ledger
* database.
* @return Session to the node store ledger database.
* @throws std::runtime_error if ledger database is not available.
*
* @note Callers typically guard with existsLedger() before calling
* this method. The explicit null check here provides
* defense-in-depth so that safety does not depend solely on
* an implicit caller contract. See PR #6029 for context on
* the pattern of relying on config settings instead of
* validating actual objects.
*/
auto
checkoutLedger()
{
if (!ledgerDb_)
{
constexpr auto msg = "Ledger database is not available";
JLOG(j_.fatal()) << msg;
Throw<std::runtime_error>(msg);
}
return ledgerDb_->checkoutDb();
}
@@ -480,23 +472,10 @@ private:
* @brief checkoutTransaction Checks out and returns the node store
* transaction database.
* @return Session to the node store transaction database.
* @throws std::runtime_error if transaction database is not available.
*
* @note Callers typically guard with existsTransaction() and/or
* useTxTables_ before calling this method. The explicit null
* check here provides defense-in-depth so that safety does
* not depend solely on an implicit caller contract or config
* settings. See PR #6029 for context.
*/
auto
checkoutTransaction()
{
if (!txdb_)
{
constexpr auto msg = "Transaction database is not available";
JLOG(j_.fatal()) << msg;
Throw<std::runtime_error>(msg);
}
return txdb_->checkoutDb();
}
};
@@ -559,6 +559,18 @@ SQLiteDatabase::SQLiteDatabase(SQLiteDatabase&& rhs) noexcept
std::exchange(txdb_, std::move(rhs.txdb_));
}
bool
SQLiteDatabase::ledgerDbHasSpace(Config const& config)
{
if (existsLedger())
{
auto db = checkoutLedger();
return detail::dbHasSpace(*db, config, j_);
}
return true;
}
bool
SQLiteDatabase::transactionDbHasSpace(Config const& config)
{
+43 -22
View File
@@ -38,10 +38,9 @@ enum class SizedItem : std::size_t {
HashNodeDbCache,
TxnDbCache,
LgrDbCache,
OpenFinalLimit,
BurstSize,
RamSizeGb,
AccountIdCacheSize,
SleCacheSize,
};
/**
@@ -138,7 +137,11 @@ private:
*/
bool signingEnabled_ = false;
// The amount of RAM, in bytes, that we detected on this system.
// The amount of RAM, in bytes, that we detected on this system. Kept at
// byte granularity (not rounded to GiB) so a cgroup limit under 1 GiB
// still reads as a nonzero budget instead of collapsing into the
// detection-failed case below.
// 0 when detection failed.
std::uint64_t const ramSize_;
public:
@@ -209,10 +212,10 @@ public:
std::uint32_t ledgerHistory = 256;
std::uint32_t fetchDepth = 1000000000;
// Tunable that adjusts various parameters, typically associated
// with hardware parameters (RAM size and CPU cores). The default
// is 'tiny'.
std::size_t nodeSize = 0;
// Cache memory budget in bytes, from [memory_limit] (gigabytes). Unset
// defaults to detected physical RAM; 0 disables enforcement. The
// deprecated [node_size] tiers map onto this budget.
std::optional<std::uint64_t> memoryLimit;
bool sslVerify = true;
std::string sslVerifyFile;
@@ -250,6 +253,11 @@ public:
// size, but we allow admins to explicitly set it in the config.
std::optional<int> sweepInterval;
// Optional overrides for the fixed cache policy values.
std::optional<int> treeCacheAge; // [tree_cache_age], seconds
std::optional<int> ledgerCacheAge; // [ledger_cache_age], seconds
std::optional<int> ledgerFetchSize; // [ledger_fetch_size], ledgers per fetch pass
// Reduce-relay - Experimental parameters to control p2p routing algorithms
// Enable base squelching of duplicate validation/proposal messages
@@ -375,25 +383,22 @@ public:
}
/**
* Retrieve the default value for the item at the specified node size
*
* @param item The item for which the default value is needed
* @param node Optional value, used to adjust the result to match the
* size of a node (0: tiny, ..., 4: huge). If unseated,
* uses the configured size (NODE_SIZE).
*
* @throws This method can throw std::out_of_range if you ask for values
* that it does not recognize or request a non-default node-size.
* Retrieve the value for the item, derived from the memory budget.
*
* @param item The item for which the value is needed
* @return The value for the requested item.
*
* @note The defaults are selected so as to be reasonable, but the node
* size is an imprecise metric that combines multiple aspects of
* the underlying system; this means that we can't provide optimal
* defaults in the code for every case.
*/
[[nodiscard]] int
getValueFor(SizedItem item, std::optional<std::size_t> node = std::nullopt) const;
getValueFor(SizedItem item) const;
/**
* The effective cache memory budget in bytes.
*
* [memory_limit] if set, otherwise detected physical RAM. 0 means
* enforcement is disabled (explicit 0, or RAM detection failed).
*/
[[nodiscard]] std::uint64_t
cacheMemoryBudget() const;
[[nodiscard]] beast::Journal
journal() const
@@ -408,4 +413,20 @@ setupFeeVote(Section const& section);
DatabaseCon::Setup
setupDatabaseCon(Config const& c, std::optional<beast::Journal> j = std::nullopt);
namespace detail {
/**
* A /proc/self/mountinfo path field with its octal escapes decoded.
*
* The kernel writes space, tab, newline and backslash in the root and
* mount point fields as `\040`, `\011`, `\012` and `\134`.
*
* @param escaped The field as read from the file.
* @return The path as the kernel names it.
*/
[[nodiscard]] std::string
decodeMountinfoPath(std::string_view escaped);
} // namespace detail
} // namespace xrpl
+364 -101
View File
@@ -35,7 +35,6 @@
#include <filesystem>
#include <format>
#include <iostream>
#include <iterator>
#include <limits>
#include <memory>
#include <optional>
@@ -43,12 +42,41 @@
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <system_error>
#include <thread>
#include <type_traits>
#include <utility>
#include <vector>
namespace xrpl::detail {
std::string
decodeMountinfoPath(std::string_view escaped)
{
// A backslash and three octal digits, the first at most 3, encode one byte.
auto const octal = [](char c, char max = '7') { return c >= '0' && c <= max; };
std::string path;
path.reserve(escaped.size());
for (std::size_t i = 0; i < escaped.size(); ++i)
{
if (escaped[i] == '\\' && i + 3 < escaped.size() && octal(escaped[i + 1], '3') &&
octal(escaped[i + 2]) && octal(escaped[i + 3]))
{
int const byte = ((escaped[i + 1] - '0') << 6) | ((escaped[i + 2] - '0') << 3) |
(escaped[i + 3] - '0');
path.push_back(static_cast<char>(byte));
i += 3;
}
else
{
path.push_back(escaped[i]);
}
}
return path;
}
} // namespace xrpl::detail
#if BOOST_OS_WINDOWS
#include <sysinfoapi.h>
@@ -69,16 +97,204 @@ getMemorySize()
#if BOOST_OS_LINUX
#include <sys/sysinfo.h> // IWYU pragma: keep
#include <sys/types.h>
#include <unistd.h>
#include <fstream>
namespace xrpl::detail {
// This process's cgroup path from /proc/self/cgroup: the v2 line is
// "0::<path>"; a v1 line is "<id>:<controllers>:<path>". Empty when absent.
[[nodiscard]] std::string
getOwnCgroupPath(std::string_view controller)
{
std::ifstream in("/proc/self/cgroup");
std::string line;
while (std::getline(in, line))
{
auto const first = line.find(':');
auto const second = line.find(':', first + 1);
if (first == std::string::npos || second == std::string::npos)
continue;
std::string_view const controllers(line.data() + first + 1, second - first - 1);
if (controller.empty())
{
// The v2 entry is exactly "0::<path>".
if (first == 1 && line[0] == '0' && controllers.empty())
return line.substr(second + 1);
}
else if (controllers.contains(controller))
{
return line.substr(second + 1);
}
}
return {};
}
// The value in a cgroup limit file; 0 when absent or unlimited. "max" (v2)
// fails the read, and the page-counter maximum (v1) both mean unlimited.
[[nodiscard]] std::uint64_t
readCgroupLimit(std::string const& path)
{
std::ifstream in(path);
std::uint64_t limit = 0;
if (in >> limit && limit < (std::uint64_t{1} << 62))
return limit;
return 0;
}
// Whether the cgroup directory contains this process. /proc/self/cgroup
// paths are namespace-relative, so a resolved directory can name-collide
// with a different cgroup when the cgroup mount shows another view; only
// trust a directory this process is actually in.
[[nodiscard]] bool
cgroupContainsSelf(std::string const& dir)
{
std::ifstream in(dir + "/cgroup.procs");
pid_t const self = ::getpid();
pid_t pid = 0;
while (in >> pid)
{
if (pid == self)
return true;
}
return false;
}
// The smallest numeric limit in `file` from the leaf cgroup up through its
// ancestors (the effective limit is the minimum over the hierarchy); 0 when
// none is set or the leaf does not belong to this process.
[[nodiscard]] std::uint64_t
minCgroupLimit(std::string const& mount, std::string path, char const* file)
{
if (!cgroupContainsSelf(mount + path))
return 0;
std::uint64_t best = 0;
auto const consider = [&best](std::uint64_t limit) {
if (limit != 0 && (best == 0 || limit < best))
best = limit;
};
while (!path.empty() && path != "/")
{
consider(readCgroupLimit(mount + path + "/" + file));
auto const slash = path.find_last_of('/');
if (slash == std::string::npos)
break;
path.resize(slash);
}
consider(readCgroupLimit(mount + "/" + file));
return best;
}
// The mount point of the cgroup v2 unified hierarchy (`controller` empty) or
// of the cgroup v1 hierarchy whose co-mounted controllers include
// `controller`; empty when neither is mounted. Read from /proc/self/mountinfo
// rather than a fixed path, since a container runtime or an init system can
// mount either hierarchy somewhere other than /sys/fs/cgroup. Field 5, the
// mount point, is already expressed relative to this process's own root
// (man 5 proc), so no further namespace translation is needed; its octal
// escapes are decoded so the path opens as the kernel names it.
[[nodiscard]] std::string
findCgroupMount(std::string_view controller)
{
std::ifstream in("/proc/self/mountinfo");
std::string line;
while (std::getline(in, line))
{
auto const dash = line.find(" - ");
if (dash == std::string::npos)
continue;
std::istringstream prefix(line.substr(0, dash));
std::string id, parentId, majorMinor, root, mountPoint;
prefix >> id >> parentId >> majorMinor >> root >> mountPoint;
std::istringstream suffix(line.substr(dash + 3));
std::string fsType, source, superOptions;
suffix >> fsType >> source >> superOptions;
if (controller.empty())
{
if (fsType == "cgroup2")
return decodeMountinfoPath(mountPoint);
continue;
}
if (fsType != "cgroup")
continue;
std::stringstream opts(superOptions);
std::string opt;
while (std::getline(opts, opt, ','))
{
if (opt == controller)
return decodeMountinfoPath(mountPoint);
}
}
return {};
}
// The cgroup (v2, then v1) memory limit in bytes; 0 when absent or
// unlimited. Checks this process's own cgroup and its ancestors (covering
// nested limits such as systemd MemoryMax=) before the root-level files
// containers expose.
[[nodiscard]] std::uint64_t
getCgroupMemoryLimit()
{
if (auto const mount = findCgroupMount(""); !mount.empty())
{
if (auto const path = getOwnCgroupPath(""); !path.empty() && path != "/")
{
if (auto const limit = minCgroupLimit(mount, path, "memory.max"))
return limit;
}
if (auto const limit = readCgroupLimit(mount + "/memory.max"))
return limit;
}
if (auto const mount = findCgroupMount("memory"); !mount.empty())
{
if (auto const path = getOwnCgroupPath("memory"); !path.empty() && path != "/")
{
if (auto const limit = minCgroupLimit(mount, path, "memory.limit_in_bytes"))
return limit;
}
if (auto const limit = readCgroupLimit(mount + "/memory.limit_in_bytes"))
return limit;
}
return 0;
}
[[nodiscard]] std::uint64_t
getMemorySize()
{
if (struct sysinfo si{}; sysinfo(&si) == 0)
return static_cast<std::uint64_t>(si.totalram) * si.mem_unit;
std::uint64_t ram = 0;
return 0;
if (struct sysinfo si{}; sysinfo(&si) == 0)
ram = static_cast<std::uint64_t>(si.totalram) * si.mem_unit;
if (auto const limit = getCgroupMemoryLimit(); limit != 0 && (ram == 0 || limit < ram))
return limit;
return ram;
}
} // namespace xrpl::detail
@@ -109,50 +325,6 @@ getMemorySize()
namespace xrpl {
// clang-format off
// The configurable node sizes are "tiny", "small", "medium", "large", "huge"
inline constexpr std::array<std::pair<SizedItem, std::array<int, 5>>, 13>
kSizedItems
{{
// FIXME: We should document each of these items, explaining exactly
// what they control and whether there exists an explicit
// config option that can be used to override the default.
// tiny small medium large huge
{SizedItem::SweepInterval, {{ 10, 30, 60, 90, 120 }}},
{SizedItem::TreeCacheSize, {{ 262144, 524288, 2097152, 4194304, 8388608 }}},
{SizedItem::TreeCacheAge, {{ 30, 60, 90, 120, 900 }}},
{SizedItem::LedgerSize, {{ 32, 32, 64, 256, 384 }}},
{SizedItem::LedgerAge, {{ 30, 60, 180, 300, 600 }}},
{SizedItem::LedgerFetch, {{ 2, 3, 4, 5, 8 }}},
{SizedItem::HashNodeDbCache, {{ 4, 12, 24, 64, 128 }}},
{SizedItem::TxnDbCache, {{ 4, 12, 24, 64, 128 }}},
{SizedItem::LgrDbCache, {{ 4, 8, 16, 32, 128 }}},
{SizedItem::OpenFinalLimit, {{ 8, 16, 32, 64, 128 }}},
{SizedItem::BurstSize, {{ 4, 8, 16, 32, 48 }}},
{SizedItem::RamSizeGb, {{ 6, 8, 12, 24, 0 }}},
{SizedItem::AccountIdCacheSize, {{ 20047, 50053, 77081, 150061, 300007 }}}
}};
// clang-format on
// Ensure that the order of entries in the table corresponds to the
// order of entries in the enum:
static_assert(
[]() constexpr -> bool {
std::underlying_type_t<SizedItem> idx = 0;
for (auto const& i : kSizedItems)
{
if (static_cast<std::underlying_type_t<SizedItem>>(i.first) != idx)
return false;
++idx;
}
return true;
}(),
"Mismatch between sized item enum & array indices");
//
// TODO: Check permissions on config file before using it.
//
@@ -257,44 +429,16 @@ getEnvVar(char const* name)
return value;
}
Config::Config()
: j_(beast::Journal::getNullSink()), ramSize_(detail::getMemorySize() / (1024 * 1024 * 1024))
Config::Config() : j_(beast::Journal::getNullSink()), ramSize_(detail::getMemorySize())
{
}
void
Config::setupControl(bool bQuiet, bool bSilent, bool bStandalone)
{
XRPL_ASSERT(nodeSize == 0, "xrpl::Config::setupControl : node size not set");
quiet_ = bQuiet || bSilent;
silent_ = bSilent;
runStandalone_ = bStandalone;
// We try to autodetect the appropriate node size by checking available
// RAM and CPU resources. We default to "tiny" for standalone mode.
if (!bStandalone)
{
// First, check against 'minimum' RAM requirements per node size:
auto const& threshold =
kSizedItems[std::underlying_type_t<SizedItem>(SizedItem::RamSizeGb)];
auto ns = std::ranges::find_if(threshold.second, [this](std::size_t limit) {
return (limit == 0) || (ramSize_ < limit);
});
XRPL_ASSERT(ns != threshold.second.end(), "xrpl::Config::setupControl : valid node size");
if (ns != threshold.second.end())
nodeSize = std::distance(threshold.second.begin(), ns);
// Adjust the size based on the number of hardware threads of
// execution available to us:
if (auto const hc = std::thread::hardware_concurrency(); hc != 0)
nodeSize = std::min<std::size_t>(hc / 2, nodeSize);
}
XRPL_ASSERT(nodeSize <= 4, "xrpl::Config::setupControl : node size is set");
}
void
@@ -577,31 +721,65 @@ Config::loadFromString(std::string const& fileContents)
}
}
if (getSingleSection(secConfig, Sections::kMemoryLimit, strTemp, j_))
{
// Gigabytes; 0 disables enforcement.
auto const gb = beast::lexicalCastThrow<std::uint64_t>(strTemp);
if (gb > 1024)
{
Throw<std::runtime_error>(
"Invalid value '" + strTemp + "' for key '" + Sections::kMemoryLimit +
"'; the limit is in gigabytes and may not exceed 1024");
}
memoryLimit = gb << 30;
}
if (getSingleSection(secConfig, Sections::kNodeSize, strTemp, j_))
{
if (boost::iequals(strTemp, "tiny"))
// Deprecated: each tier (by name or its legacy 0-4 index) is an
// alias for a memory budget. [memory_limit], when present, wins.
static constexpr std::array<std::pair<std::string_view, std::uint64_t>, 5> kTiers{
{{"tiny", 4}, {"small", 8}, {"medium", 32}, {"large", 64}, {"huge", 128}}};
auto const tier = std::ranges::find_if(
kTiers, [&strTemp](auto const& t) { return boost::iequals(strTemp, t.first); });
std::uint64_t const budgetGb = tier != kTiers.end()
? tier->second
: kTiers[std::min<std::size_t>(4, beast::lexicalCastThrow<std::size_t>(strTemp))]
.second;
if (!memoryLimit)
memoryLimit = budgetGb << 30;
if (!quiet_)
{
nodeSize = 0;
std::cerr << "WARNING: [node_size] is deprecated and will be removed "
"in a future release. Set [memory_limit] instead; thread "
"counts derive from the core count and [workers] / "
"[io_workers].\n";
}
else if (boost::iequals(strTemp, "small"))
}
// A budget beyond the detected memory (physical RAM, or the cgroup limit
// when one is set) cannot be honored and recreates the oversized-preset
// OOM this setting exists to prevent.
if (memoryLimit && ramSize_ != 0 && *memoryLimit > ramSize_ && !quiet_)
{
std::cerr << "WARNING: the configured memory budget (" << (*memoryLimit >> 30)
<< " GB) exceeds detected memory (" << ramSize_
<< " bytes, RAM or cgroup limit); ";
// A whole-GB floor of 0 here is a nonzero byte budget rounded down,
// not the "disabled" value: recommending it would turn enforcement
// off, the opposite of this warning's point.
if (auto const detectedGb = ramSize_ >> 30)
{
nodeSize = 1;
}
else if (boost::iequals(strTemp, "medium"))
{
nodeSize = 2;
}
else if (boost::iequals(strTemp, "large"))
{
nodeSize = 3;
}
else if (boost::iequals(strTemp, "huge"))
{
nodeSize = 4;
std::cerr << "set [memory_limit] to " << detectedGb << " or less.\n";
}
else
{
nodeSize = std::min<std::size_t>(4, beast::lexicalCastThrow<std::size_t>(strTemp));
std::cerr << "[memory_limit] is in whole gigabytes and cannot represent it; "
"remove the setting to use the detected budget automatically.\n";
}
}
@@ -744,6 +922,42 @@ Config::loadFromString(std::string const& fileContents)
}
}
if (getSingleSection(secConfig, Sections::kTreeCacheAge, strTemp, j_))
{
treeCacheAge = beast::lexicalCastThrow<int>(strTemp);
if (*treeCacheAge < 10 || *treeCacheAge > 3600)
{
Throw<std::runtime_error>(
std::string("Invalid ") + Sections::kTreeCacheAge +
": must be between 10 and 3600 inclusive");
}
}
if (getSingleSection(secConfig, Sections::kLedgerCacheAge, strTemp, j_))
{
ledgerCacheAge = beast::lexicalCastThrow<int>(strTemp);
if (*ledgerCacheAge < 10 || *ledgerCacheAge > 3600)
{
Throw<std::runtime_error>(
std::string("Invalid ") + Sections::kLedgerCacheAge +
": must be between 10 and 3600 inclusive");
}
}
if (getSingleSection(secConfig, Sections::kLedgerFetchSize, strTemp, j_))
{
ledgerFetchSize = beast::lexicalCastThrow<int>(strTemp);
if (*ledgerFetchSize < 1 || *ledgerFetchSize > 16)
{
Throw<std::runtime_error>(
std::string("Invalid ") + Sections::kLedgerFetchSize +
": must be between 1 and 16 inclusive");
}
}
if (getSingleSection(secConfig, Sections::kWorkers, strTemp, j_))
{
workers = beast::lexicalCastThrow<int>(strTemp);
@@ -1224,12 +1438,61 @@ Config::getDebugLogFile() const
}
int
Config::getValueFor(SizedItem item, std::optional<std::size_t> node) const
Config::getValueFor(SizedItem item) const
{
auto const index = static_cast<std::underlying_type_t<SizedItem>>(item);
XRPL_ASSERT(index < kSizedItems.size(), "xrpl::Config::getValueFor : valid index input");
XRPL_ASSERT(!node || *node <= 4, "xrpl::Config::getValueFor : unset or valid node");
return kSizedItems.at(index).second.at(node.value_or(nodeSize));
// Memory-shaped items scale linearly with the budget between a floor and
// a ceiling; time and policy items are fixed. A budget of 0 (enforcement
// disabled) yields the floors. The 1024 bound keeps gb * 65536 within
// int range (the config parser enforces it too).
auto const gb = static_cast<int>(std::min<std::uint64_t>(cacheMemoryBudget() >> 30, 1024));
switch (item)
{
case SizedItem::SweepInterval:
return 30;
case SizedItem::TreeCacheSize:
// Half the budget at an estimated 8 KiB per entry (the node plus
// its weak-tracking entry, hash buckets, and control block):
// 1 GiB / 2 / 8 KiB = 65536 entries per budget GB.
return std::max(16384, gb * 65536);
case SizedItem::TreeCacheAge:
return treeCacheAge.value_or(300);
case SizedItem::LedgerSize:
return std::clamp(gb * 6, 32, 384);
case SizedItem::LedgerAge:
return ledgerCacheAge.value_or(180);
case SizedItem::LedgerFetch:
return ledgerFetchSize.value_or(4);
case SizedItem::HashNodeDbCache:
case SizedItem::TxnDbCache:
case SizedItem::LgrDbCache:
// HashNodeDbCache is consumed in MB (RocksDB cache_mb); the two
// SQLite page caches are consumed in KB.
return std::clamp(gb * 2, 4, 128);
case SizedItem::BurstSize:
return std::clamp(gb, 4, 48);
case SizedItem::AccountIdCacheSize:
// Fixed regardless of budget: ~22 MB at 72 bytes per slot, and
// the value stays prime for hash distribution.
return 300007;
case SizedItem::SleCacheSize:
// Closed-ledger SLEs pulled by RPC and pathfinding, a few KB each.
return std::clamp(gb * 1024, 4096, 262144);
}
UNREACHABLE("xrpl::Config::getValueFor : invalid item");
return 0;
}
std::uint64_t
Config::cacheMemoryBudget() const
{
if (memoryLimit)
return *memoryLimit;
// ramSize_ is in bytes; 0 when detection failed, which disables
// enforcement.
return ramSize_;
}
FeeSetup
@@ -1,6 +1,5 @@
#include <xrpl/core/PeerReservationTable.h>
#include <xrpl/basics/contract.h>
#include <xrpl/json/json_value.h>
#include <xrpl/protocol/PublicKey.h>
#include <xrpl/protocol/jss.h>
@@ -11,7 +10,6 @@
#include <iterator>
#include <mutex>
#include <optional>
#include <stdexcept>
#include <string>
#include <vector>
@@ -89,15 +87,6 @@ PeerReservationTable::insertOrAssign(PeerReservation const& reservation)
}
table_.insert(hint, reservation);
// connection_ is set by load() during two-phase init. Validate
// before dereferencing to guard against use-before-load or a reset
// connection. See PR #6029 for the general pattern discussion.
if (connection_ == nullptr)
{
Throw<std::runtime_error>(
"PeerReservationTable::insertOrAssign: database connection is "
"not available");
}
auto db = connection_->checkoutDb();
insertPeerReservation(*db, reservation.nodeId, reservation.description);
@@ -116,14 +105,6 @@ PeerReservationTable::erase(PublicKey const& nodeId)
{
previous = *it;
table_.erase(it);
// Validate connection_ before dereferencing — see comment in
// insertOrAssign above.
if (connection_ == nullptr)
{
Throw<std::runtime_error>(
"PeerReservationTable::erase: database connection is not "
"available");
}
auto db = connection_->checkoutDb();
deletePeerReservation(*db, nodeId);
}
+13 -1
View File
@@ -11,6 +11,7 @@
#include <xrpl/basics/Log.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/chrono.h>
#include <xrpl/beast/insight/NullCollector.h>
#include <xrpl/shamap/FullBelowCache.h>
#include <xrpl/shamap/TreeNodeCache.h>
@@ -39,8 +40,19 @@ NodeFamily::NodeFamily(Application& app, CollectorManager& cm)
app.config().getValueFor(SizedItem::TreeCacheSize),
std::chrono::seconds(app.config().getValueFor(SizedItem::TreeCacheAge)),
stopwatch(),
j_))
j_,
beast::insight::NullCollector::make(),
// Hard cap: the clamped target, enforced on insert; 0 = off.
app.config().cacheMemoryBudget() != 0
? app.config().getValueFor(SizedItem::TreeCacheSize)
: 0))
{
// Bytes, not whole GB: a cgroup limit under 1 GiB is a nonzero budget
// with the cap on, and must not read as the disabled value.
auto const budget = app.config().cacheMemoryBudget();
JLOG(j_.info()) << "TreeNodeCache sizing: target="
<< app.config().getValueFor(SizedItem::TreeCacheSize) << " entries, budget "
<< budget << " bytes" << (budget == 0 ? " (enforcement disabled)" : "");
}
void