Compare commits

..

20 Commits

Author SHA1 Message Date
Michael Legleux
cb5354e4fe PR comments 2026-04-28 16:52:40 -07:00
Michael Legleux
19da83abf3 package: refactor packaging; make builds reproducible; stop requiring git 2026-04-28 16:22:22 -07:00
Michael Legleux
8a02c904a6 fix indentation 2026-04-28 16:22:22 -07:00
Michael Legleux
e7ca96362f rm rpm template; try to get version from binary 2026-04-28 16:22:21 -07:00
Michael Legleux
4593754347 f 2026-04-28 16:22:21 -07:00
Michael Legleux
2caa18ddd5 update matrix to build packages 2026-04-28 16:22:21 -07:00
Michael Legleux
8dbf9b8ab4 add a little more info to the result of the update check 2026-04-28 16:22:21 -07:00
Michael Legleux
94ef68090c Migrate updater from cron to systemd timer; use journald + flock
- systemd oneshot service + timer replaces cron job
- randomized scheduling and persistent timers
- journald logging with SyslogIdentifier
- flock-based concurrency control
- remove legacy update.log file
2026-04-28 16:22:21 -07:00
Michael Legleux
8fa81e24ed guard more 2026-04-28 16:22:21 -07:00
Michael Legleux
4e1c3b0922 Pin packaging image versions 2026-04-28 16:22:21 -07:00
Michael Legleux
f148d9a4e7 pr comments 2026-04-28 16:22:20 -07:00
Michael Legleux
453e3efcc9 PR comments and more tweaks 2026-04-28 16:22:20 -07:00
Michael Legleux
8b4ff1c171 pr comments 2026-04-28 16:22:20 -07:00
Michael Legleux
c33630f859 more tweaks 2026-04-28 16:22:20 -07:00
Michael Legleux
94103e97d7 cp lic/readme 2026-04-28 16:22:20 -07:00
Michael Legleux
cfcb09933a more fixes/updates 2026-04-28 16:22:20 -07:00
Michael Legleux
e36ba02762 update download-artifacts version 2026-04-28 16:22:19 -07:00
Michael Legleux
fe47e5adb2 handle tmpfiles in rpm 2026-04-28 16:22:19 -07:00
Michael Legleux
96a6922bce address review comments 2026-04-28 16:22:19 -07:00
Michael Legleux
7721bda445 feat: Build Linux packages in GitHub 2026-04-28 16:22:19 -07:00
66 changed files with 1532 additions and 922 deletions

View File

@@ -93,6 +93,7 @@ test.core > xrpl.basics
test.core > xrpl.core
test.core > xrpld.core
test.core > xrpl.json
test.core > xrpl.protocol
test.core > xrpl.rdb
test.core > xrpl.server
test.csf > xrpl.basics

View File

@@ -4,14 +4,15 @@ import itertools
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any
THIS_DIR = Path(__file__).parent.resolve()
@dataclass
class Config:
architecture: list[dict]
os: list[dict]
architecture: list[dict[str, Any]]
os: list[dict[str, Any]]
build_type: list[str]
cmake_args: list[str]
@@ -32,7 +33,52 @@ We will further set additional CMake arguments as follows:
"""
def generate_strategy_matrix(all: bool, config: Config) -> list:
def build_config_name(
os_entry: dict[str, Any], architecture: dict[str, Any], build_type: str
) -> str:
parts = [os_entry["distro_name"]]
for key in ("distro_version", "compiler_name", "compiler_version"):
if value := os_entry[key]:
parts.append(value)
platform = architecture["platform"]
parts.append(platform[platform.find("/") + 1 :])
parts.append(build_type.lower())
return "-".join(parts)
def build_container_image(os_entry: dict[str, Any]) -> str:
image = f"ghcr.io/xrplf/ci/{os_entry['distro_name']}-{os_entry['distro_version']}"
tag = f"{os_entry['compiler_name']}-{os_entry['compiler_version']}-sha-{os_entry['image_sha']}"
return f"{image}:{tag}"
def generate_packaging_matrix(config: Config) -> list[dict[str, str]]:
"""Emit packaging entries for each os entry with `package: true`.
Packaging always uses Release build on linux/amd64. The package format
(deb or rpm) is inferred at runtime from the container's package manager.
"""
architecture = next(
(a for a in config.architecture if a["platform"] == "linux/amd64"),
None,
)
if architecture is None:
raise Exception("linux/amd64 architecture required for packaging")
entries = []
for os_entry in config.os:
if not os_entry.get("package", False):
continue
config_name = build_config_name(os_entry, architecture, "Release")
entries.append(
{
"artifact_name": f"xrpld-{config_name}",
"container_image": build_container_image(os_entry),
}
)
return entries
def generate_strategy_matrix(all: bool, config: Config) -> list[dict[str, Any]]:
configurations = []
for architecture, os, build_type, cmake_args in itertools.product(
config.architecture, config.os, config.build_type, config.cmake_args
@@ -100,14 +146,15 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
continue
# RHEL:
# - 9 using GCC 12: Debug on linux/amd64.
# - 9 using GCC 12: Debug and Release on linux/amd64
# (Release is required for RPM packaging).
# - 10 using Clang: Release on linux/amd64.
if os["distro_name"] == "rhel":
skip = True
if os["distro_version"] == "9":
if (
f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12"
and build_type == "Debug"
and build_type in ["Debug", "Release"]
and architecture["platform"] == "linux/amd64"
):
skip = False
@@ -122,7 +169,8 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
continue
# Ubuntu:
# - Jammy using GCC 12: Debug on linux/arm64.
# - Jammy using GCC 12: Debug on linux/arm64, Release on
# linux/amd64 (Release is required for DEB packaging).
# - Noble using GCC 14: Release on linux/amd64.
# - Noble using Clang 18: Debug on linux/amd64.
# - Noble using Clang 19: Release on linux/arm64.
@@ -135,6 +183,12 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
and architecture["platform"] == "linux/arm64"
):
skip = False
if (
f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-12"
and build_type == "Release"
and architecture["platform"] == "linux/amd64"
):
skip = False
elif os["distro_version"] == "noble":
if (
f"{os['compiler_name']}-{os['compiler_version']}" == "gcc-14"
@@ -216,17 +270,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
# Generate a unique name for the configuration, e.g. macos-arm64-debug
# or debian-bookworm-gcc-12-amd64-release.
config_name = os["distro_name"]
if (n := os["distro_version"]) != "":
config_name += f"-{n}"
if (n := os["compiler_name"]) != "":
config_name += f"-{n}"
if (n := os["compiler_version"]) != "":
config_name += f"-{n}"
config_name += (
f"-{architecture['platform'][architecture['platform'].find('/')+1:]}"
)
config_name += f"-{build_type.lower()}"
config_name = build_config_name(os, architecture, build_type)
if "-Dcoverage=ON" in cmake_args:
config_name += "-coverage"
if "-Dunity=ON" in cmake_args:
@@ -330,10 +374,19 @@ if __name__ == "__main__":
required=False,
type=Path,
)
parser.add_argument(
"-p",
"--packaging",
help="Emit the packaging matrix (derived from the 'package' field on os entries) instead of the build/test matrix.",
action="store_true",
)
args = parser.parse_args()
matrix = []
if args.config is None or args.config == "":
if args.packaging:
config_path = args.config if args.config else THIS_DIR / "linux.json"
matrix += generate_packaging_matrix(read_config(config_path))
elif args.config is None or args.config == "":
matrix += generate_strategy_matrix(
args.all, read_config(THIS_DIR / "linux.json")
)

View File

@@ -120,7 +120,8 @@
"distro_version": "9",
"compiler_name": "gcc",
"compiler_version": "12",
"image_sha": "ab4d1f0"
"image_sha": "ab4d1f0",
"package": true
},
{
"distro_name": "rhel",
@@ -162,7 +163,8 @@
"distro_version": "jammy",
"compiler_name": "gcc",
"compiler_version": "12",
"image_sha": "ab4d1f0"
"image_sha": "ab4d1f0",
"package": true
},
{
"distro_name": "ubuntu",

View File

@@ -67,6 +67,7 @@ jobs:
.github/workflows/reusable-build-test.yml
.github/workflows/reusable-clang-tidy.yml
.github/workflows/reusable-clang-tidy-files.yml
.github/workflows/reusable-package.yml
.github/workflows/reusable-strategy-matrix.yml
.github/workflows/reusable-test.yml
.github/workflows/reusable-upload-recipe.yml
@@ -81,6 +82,8 @@ jobs:
CMakeLists.txt
conanfile.py
conan.lock
package/**
- name: Check whether to run
# This step determines whether the rest of the workflow should
# run. The rest of the workflow will run if this job runs AND at
@@ -137,6 +140,30 @@ jobs:
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
generate-version:
needs: should-run
if: ${{ needs.should-run.outputs.go == 'true' }}
runs-on: ubuntu-latest
outputs:
version: ${{ steps.version.outputs.version }}
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: |
.github/actions/generate-version
src/libxrpl/protocol/BuildInfo.cpp
- name: Generate version
id: version
uses: ./.github/actions/generate-version
package:
needs: [should-run, build-test, generate-version]
if: ${{ needs.should-run.outputs.go == 'true' }}
uses: ./.github/workflows/reusable-package.yml
with:
version: ${{ needs.generate-version.outputs.version }}
upload-recipe:
needs:
- should-run

View File

@@ -1,5 +1,5 @@
# This workflow uploads the libxrpl recipe to the Conan remote when a versioned
# tag is pushed.
# This workflow uploads the libxrpl recipe to the Conan remote and builds
# release packages when a versioned tag is pushed.
name: Tag
on:
@@ -22,3 +22,39 @@ jobs:
secrets:
remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }}
remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }}
build-test:
if: ${{ github.repository == 'XRPLF/rippled' }}
uses: ./.github/workflows/reusable-build-test.yml
strategy:
fail-fast: true
matrix:
os: [linux]
with:
ccache_enabled: false
os: ${{ matrix.os }}
strategy_matrix: minimal
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
generate-version:
if: ${{ github.repository == 'XRPLF/rippled' }}
runs-on: ubuntu-latest
outputs:
version: ${{ steps.version.outputs.version }}
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: |
.github/actions/generate-version
src/libxrpl/protocol/BuildInfo.cpp
- name: Generate version
id: version
uses: ./.github/actions/generate-version
package:
needs: [build-test, generate-version]
uses: ./.github/workflows/reusable-package.yml
with:
version: ${{ needs.generate-version.outputs.version }}

View File

@@ -24,6 +24,7 @@ on:
- ".github/workflows/reusable-build-test.yml"
- ".github/workflows/reusable-clang-tidy.yml"
- ".github/workflows/reusable-clang-tidy-files.yml"
- ".github/workflows/reusable-package.yml"
- ".github/workflows/reusable-strategy-matrix.yml"
- ".github/workflows/reusable-test.yml"
- ".github/workflows/reusable-upload-recipe.yml"
@@ -38,6 +39,7 @@ on:
- "CMakeLists.txt"
- "conanfile.py"
- "conan.lock"
- "package/**"
# Run at 06:32 UTC on every day of the week from Monday through Friday. This
# will force all dependencies to be rebuilt, which is useful to verify that
@@ -98,3 +100,24 @@ jobs:
secrets:
remote_username: ${{ secrets.CONAN_REMOTE_USERNAME }}
remote_password: ${{ secrets.CONAN_REMOTE_PASSWORD }}
generate-version:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.version.outputs.version }}
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: |
.github/actions/generate-version
src/libxrpl/protocol/BuildInfo.cpp
- name: Generate version
id: version
uses: ./.github/actions/generate-version
package:
needs: [build-test, generate-version]
uses: ./.github/workflows/reusable-package.yml
with:
version: ${{ needs.generate-version.outputs.version }}

91
.github/workflows/reusable-package.yml vendored Normal file
View File

@@ -0,0 +1,91 @@
# Build Linux packages (DEB and RPM) from pre-built binary artifacts.
# Discovers which configurations to package from linux.json (entries with
# "package": true) and fans out one job per entry.
name: Package
on:
workflow_call:
inputs:
pkg_release:
description: "Package release number. Increment when repackaging the same executable."
required: false
type: string
default: "1"
version:
description: "Version string used for naming the output artifact."
required: true
type: string
defaults:
run:
shell: bash
env:
BUILD_DIR: build
jobs:
generate-matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.generate.outputs.matrix }}
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: 3.13
- name: Generate packaging matrix
id: generate
working-directory: .github/scripts/strategy-matrix
run: |
./generate.py --packaging --config=linux.json >> "${GITHUB_OUTPUT}"
package:
needs: generate-matrix
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
name: "${{ matrix.artifact_name }}"
permissions:
contents: read
runs-on: ["self-hosted", "Linux", "X64", "heavy"]
container: ${{ matrix.container_image }}
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set source date epoch
run: |
echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct "$GITHUB_SHA")" >> "$GITHUB_ENV"
- name: Download pre-built binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ matrix.artifact_name }}
path: ${{ env.BUILD_DIR }}
- name: Make binary executable
run: chmod +x "${BUILD_DIR}/xrpld"
- name: Build package
env:
PKG_VERSION: ${{ inputs.version }}
PKG_RELEASE: ${{ inputs.pkg_release }}
SOURCE_DATE_EPOCH: ${{ env.SOURCE_DATE_EPOCH }}
run: |
./package/build_pkg.sh . "$BUILD_DIR" "$PKG_VERSION" "$PKG_RELEASE"
- name: Upload package artifact
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: ${{ matrix.artifact_name }}-pkg-${{ inputs.version }}
path: |
${{ env.BUILD_DIR }}/debbuild/*.deb
${{ env.BUILD_DIR }}/debbuild/*.ddeb
${{ env.BUILD_DIR }}/rpmbuild/RPMS/**/*.rpm
if-no-files-found: error

View File

@@ -39,7 +39,8 @@ jobs:
- name: Generate strategy matrix
working-directory: .github/scripts/strategy-matrix
id: generate
env:
GENERATE_CONFIG: ${{ inputs.os != '' && format('--config={0}.json', inputs.os) || '' }}
GENERATE_OPTION: ${{ inputs.strategy_matrix == 'all' && '--all' || '' }}
run: ./generate.py ${GENERATE_OPTION} ${GENERATE_CONFIG} >> "${GITHUB_OUTPUT}"
run: |
./generate.py \
${{ inputs.strategy_matrix == 'all' && '--all' || '' }} \
${{ inputs.os != '' && format('--config={0}.json', inputs.os) || '' }} \
>> "${GITHUB_OUTPUT}"

View File

@@ -134,6 +134,7 @@ endif()
include(XrplCore)
include(XrplProtocolAutogen)
include(XrplInstall)
include(XrplPackaging)
include(XrplValidatorKeys)
if(tests)

View File

@@ -12,14 +12,14 @@ if(is_root_project AND TARGET xrpld)
install(
FILES "${CMAKE_CURRENT_SOURCE_DIR}/cfg/xrpld-example.cfg"
DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}/xrpld"
DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}"
RENAME xrpld.cfg
COMPONENT runtime
)
install(
FILES "${CMAKE_CURRENT_SOURCE_DIR}/cfg/validators-example.txt"
DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}/xrpld"
DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}"
RENAME validators.txt
COMPONENT runtime
)

136
cmake/XrplPackaging.cmake Normal file
View File

@@ -0,0 +1,136 @@
#[===================================================================[
Linux packaging support: RPM and Debian targets + install tests
#]===================================================================]
if(NOT CMAKE_INSTALL_PREFIX STREQUAL "/opt/xrpld")
message(
STATUS
"Packaging targets require -DCMAKE_INSTALL_PREFIX=/opt/xrpld "
"(current: '${CMAKE_INSTALL_PREFIX}'); skipping."
)
return()
endif()
if(NOT DEFINED pkg_release)
set(pkg_release 1)
endif()
find_program(RPMBUILD_EXECUTABLE rpmbuild)
find_program(DPKG_BUILDPACKAGE_EXECUTABLE dpkg-buildpackage)
if(RPMBUILD_EXECUTABLE OR DPKG_BUILDPACKAGE_EXECUTABLE)
add_custom_target(
package
COMMAND
${CMAKE_SOURCE_DIR}/package/build_pkg.sh ${CMAKE_SOURCE_DIR}
${CMAKE_BINARY_DIR} "${xrpld_version}" ${pkg_release}
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
DEPENDS xrpld
COMMENT "Building Linux package (deb/rpm inferred from host tooling)"
VERBATIM
)
else()
message(
STATUS
"Neither rpmbuild nor dpkg-buildpackage found; 'package' target not available"
)
endif()
#[===================================================================[
CTest fixtures for package install verification (requires docker)
#]===================================================================]
find_program(DOCKER_EXECUTABLE docker)
if(NOT DOCKER_EXECUTABLE)
message(STATUS "docker not found; package install tests not available")
return()
endif()
set(DEB_TEST_IMAGE
"geerlingguy/docker-ubuntu2204-ansible@sha256:bbe4c56c16c57c902554b9a47833590926b7a7d4440aef3d9851473b9f7be9d4"
)
set(RPM_TEST_IMAGE
"geerlingguy/docker-rockylinux9-ansible@sha256:790c2db9add93c0daa903ace816f352c9c04abb046ecfa12c581e8d4c59f41d6"
)
# Only register install-test fixtures for package formats the host can build,
# since the smoketest needs a corresponding .deb/.rpm artifact in build/.
set(PKG_TYPES "")
if(DPKG_BUILDPACKAGE_EXECUTABLE)
list(APPEND PKG_TYPES deb)
endif()
if(RPMBUILD_EXECUTABLE)
list(APPEND PKG_TYPES rpm)
endif()
foreach(PKG IN LISTS PKG_TYPES)
if(PKG STREQUAL "deb")
set(IMAGE ${DEB_TEST_IMAGE})
else()
set(IMAGE ${RPM_TEST_IMAGE})
endif()
# This image runs systemd for full testing xrpld.service
add_test(
NAME ${PKG}_container_start
COMMAND
sh -c
"docker rm -f xrpld_${PKG}_install_test 2>/dev/null || true && \
docker run -d \
--name xrpld_${PKG}_install_test \
--cgroupns host \
--volume '${CMAKE_SOURCE_DIR}:/root:ro' \
--volume /sys/fs/cgroup:/sys/fs/cgroup:rw \
--tmpfs /run/lock \
${IMAGE}"
)
set_tests_properties(
${PKG}_container_start
PROPERTIES FIXTURES_SETUP ${PKG}_container LABELS packaging
)
# On CI: always stop. Locally: leave running on failure for diagnosis.
add_test(
NAME ${PKG}_container_stop
COMMAND
sh -c
"if [ -n \"$CI\" ] || ! docker exec xrpld_${PKG}_install_test test -f /tmp/test_failed 2>/dev/null; then \
docker rm -f xrpld_${PKG}_install_test; \
else \
echo 'Tests failed — leaving xrpld_${PKG}_install_test running for diagnosis'; \
echo 'Clean up with: docker rm -f xrpld_${PKG}_install_test'; \
fi"
)
set_tests_properties(
${PKG}_container_stop
PROPERTIES FIXTURES_CLEANUP ${PKG}_container LABELS packaging
)
add_test(
NAME ${PKG}_install
COMMAND
docker exec -w /root xrpld_${PKG}_install_test bash
/root/package/test/smoketest.sh local
)
set_tests_properties(
${PKG}_install
PROPERTIES
FIXTURES_REQUIRED ${PKG}_container
FIXTURES_SETUP ${PKG}_installed
LABELS packaging
TIMEOUT 600
)
add_test(
NAME ${PKG}_install_paths
COMMAND
docker exec -w /root xrpld_${PKG}_install_test sh
/root/package/test/check_install_paths.sh
)
set_tests_properties(
${PKG}_install_paths
PROPERTIES
FIXTURES_REQUIRED "${PKG}_container;${PKG}_installed"
LABELS packaging
TIMEOUT 60
)
endforeach()

View File

@@ -97,12 +97,15 @@ words:
- desync
- desynced
- determ
- disablerepo
- distro
- doxyfile
- dxrpl
- enabled
- enablerepo
- endmacro
- exceptioned
- EXPECT_STREQ
- Falco
- fcontext
- finalizers
@@ -158,6 +161,7 @@ words:
- Merkle
- Metafuncton
- misprediction
- missingok
- mptbalance
- MPTDEX
- mptflags
@@ -189,7 +193,9 @@ words:
- NOLINT
- NOLINTNEXTLINE
- nonxrp
- noreplace
- noripple
- notifempty
- nudb
- nullptr
- nunl
@@ -209,6 +215,7 @@ words:
- preauthorize
- preauthorizes
- preclaim
- preun
- protobuf
- protos
- ptrs
@@ -243,12 +250,14 @@ words:
- sfields
- shamap
- shamapitem
- shlibs
- sidechain
- SIGGOOD
- sle
- sles
- soci
- socidb
- SRPMS
- sslws
- statsd
- STATSDCOLLECTOR
@@ -276,8 +285,8 @@ words:
- txn
- txns
- txs
- UBSAN
- ubsan
- UBSAN
- umant
- unacquired
- unambiguity
@@ -314,7 +323,6 @@ words:
- xbridge
- xchain
- ximinez
- EXPECT_STREQ
- XMACRO
- xrpkuwait
- xrpl

View File

@@ -11,7 +11,6 @@
#include <limits>
#include <stdexcept>
#include <string>
#include <string_view>
#include <type_traits>
#include <vector>
@@ -232,11 +231,4 @@ makeSlice(std::basic_string<char, Traits, Alloc> const& s)
return Slice(s.data(), s.size());
}
template <class Traits>
Slice
makeSlice(std::basic_string_view<char, Traits> s)
{
return Slice(s.data(), s.size());
}
} // namespace xrpl

View File

@@ -246,15 +246,7 @@ message TMGetObjectByHash {
message TMLedgerNode {
required bytes nodedata = 1;
// Used when protocol version <2.3. Not set for ledger base data.
optional bytes nodeid = 2;
// Used when protocol version >=2.3. Neither value is set for ledger base data.
oneof reference {
bytes id = 3; // Set for inner nodes.
uint32 depth = 4; // Set for leaf nodes.
}
optional bytes nodeid = 2; // missing for ledger base data
}
enum TMLedgerInfoType {

View File

@@ -16,7 +16,6 @@
#include <set>
#include <stack>
#include <tuple>
#include <vector>
namespace xrpl {
@@ -74,22 +73,6 @@ enum class SHAMapState {
See https://en.wikipedia.org/wiki/Merkle_tree
*/
/** Holds a SHAMap node's identity, serialized data, and leaf status.
Used by getNodeFat to return node data for peer synchronization.
*/
struct SHAMapNodeData
{
SHAMapNodeID nodeID;
Blob data;
bool isLeaf;
SHAMapNodeData(SHAMapNodeID const& id, Blob d, bool leaf)
: nodeID(id), data(std::move(d)), isLeaf(leaf)
{
}
};
class SHAMap
{
private:
@@ -102,7 +85,7 @@ private:
/** The sequence of the ledger that this map references, if any. */
std::uint32_t ledgerSeq_ = 0;
SHAMapTreeNodePtr root_;
intr_ptr::SharedPtr<SHAMapTreeNode> root_;
mutable SHAMapState state_;
SHAMapType const type_;
bool backed_ = true; // Map is backed by the database
@@ -267,10 +250,10 @@ public:
std::vector<std::pair<SHAMapNodeID, uint256>>
getMissingNodes(int maxNodes, SHAMapSyncFilter* filter);
[[nodiscard]] bool
bool
getNodeFat(
SHAMapNodeID const& wanted,
std::vector<SHAMapNodeData>& data,
std::vector<std::pair<SHAMapNodeID, Blob>>& data,
bool fatLeaves,
std::uint32_t depth) const;
@@ -297,42 +280,10 @@ public:
void
serializeRoot(Serializer& s) const;
/** Add a root node to the SHAMap during synchronization.
*
* This function is used when receiving the root node of a SHAMap from a peer during ledger
* synchronization. The node must already have been deserialized.
*
* @param hash The expected hash of the root node.
* @param rootNode A deserialized root node to add.
* @param filter Optional sync filter to track received nodes.
* @return Status indicating whether the node was useful, duplicate, or invalid.
*
* @note This function expects the rootNode to be a valid, deserialized SHAMapTreeNode. The
* caller is responsible for deserialization and basic validation before calling this
* function.
*/
SHAMapAddNode
addRootNode(SHAMapHash const& hash, SHAMapTreeNodePtr rootNode, SHAMapSyncFilter const* filter);
/** Add a known node at a specific position in the SHAMap during synchronization.
*
* This function is used when receiving nodes from peers during ledger synchronization. The node
* is inserted at the position specified by nodeID. The node must already have been
* deserialized.
*
* @param nodeID The position in the tree where this node belongs.
* @param treeNode A deserialized tree node to add.
* @param filter Optional sync filter to track received nodes.
* @return Status indicating whether the node was useful, duplicate, or invalid.
*
* @note This function expects that the caller has already validated that the nodeID is
* consistent with the node's content.
*/
addRootNode(SHAMapHash const& hash, Slice const& rootNode, SHAMapSyncFilter* filter);
SHAMapAddNode
addKnownNode(
SHAMapNodeID const& nodeID,
SHAMapTreeNodePtr treeNode,
SHAMapSyncFilter const* filter);
addKnownNode(SHAMapNodeID const& nodeID, Slice const& rawNode, SHAMapSyncFilter* filter);
// status functions
void
@@ -375,32 +326,36 @@ public:
invariants() const;
private:
using SharedPtrNodeStack = std::stack<std::pair<SHAMapTreeNodePtr, SHAMapNodeID>>;
using SharedPtrNodeStack =
std::stack<std::pair<intr_ptr::SharedPtr<SHAMapTreeNode>, SHAMapNodeID>>;
using DeltaRef =
std::pair<boost::intrusive_ptr<SHAMapItem const>, boost::intrusive_ptr<SHAMapItem const>>;
// tree node cache operations
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
cacheLookup(SHAMapHash const& hash) const;
void
canonicalize(SHAMapHash const& hash, SHAMapTreeNodePtr&) const;
canonicalize(SHAMapHash const& hash, intr_ptr::SharedPtr<SHAMapTreeNode>&) const;
// database operations
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNodeFromDB(SHAMapHash const& hash) const;
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNodeNT(SHAMapHash const& hash) const;
SHAMapTreeNodePtr
fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const;
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNode(SHAMapHash const& hash) const;
SHAMapTreeNodePtr
checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const;
intr_ptr::SharedPtr<SHAMapTreeNode>
checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
/** Update hashes up to the root */
void
dirtyUp(SharedPtrNodeStack& stack, uint256 const& target, SHAMapTreeNodePtr terminal);
dirtyUp(
SharedPtrNodeStack& stack,
uint256 const& target,
intr_ptr::SharedPtr<SHAMapTreeNode> terminal);
/** Walk towards the specified id, returning the node. Caller must check
if the return is nullptr, and if not, if the node->peekItem()->key() ==
@@ -422,21 +377,25 @@ private:
preFlushNode(intr_ptr::SharedPtr<Node> node) const;
/** write and canonicalize modified node */
SHAMapTreeNodePtr
writeNode(NodeObjectType t, SHAMapTreeNodePtr node) const;
intr_ptr::SharedPtr<SHAMapTreeNode>
writeNode(NodeObjectType t, intr_ptr::SharedPtr<SHAMapTreeNode> node) const;
// returns the first item at or below this node
SHAMapLeafNode*
firstBelow(SHAMapTreeNodePtr node, SharedPtrNodeStack& stack, int branch = 0) const;
firstBelow(intr_ptr::SharedPtr<SHAMapTreeNode>, SharedPtrNodeStack& stack, int branch = 0)
const;
// returns the last item at or below this node
SHAMapLeafNode*
lastBelow(SHAMapTreeNodePtr node, SharedPtrNodeStack& stack, int branch = branchFactor) const;
lastBelow(
intr_ptr::SharedPtr<SHAMapTreeNode> node,
SharedPtrNodeStack& stack,
int branch = branchFactor) const;
// helper function for firstBelow and lastBelow
SHAMapLeafNode*
belowHelper(
SHAMapTreeNodePtr node,
intr_ptr::SharedPtr<SHAMapTreeNode> node,
SharedPtrNodeStack& stack,
int branch,
std::tuple<int, std::function<bool(int)>, std::function<void(int&)>> const& loopParams)
@@ -448,19 +407,20 @@ private:
descend(SHAMapInnerNode*, int branch) const;
SHAMapTreeNode*
descendThrow(SHAMapInnerNode*, int branch) const;
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
descend(SHAMapInnerNode&, int branch) const;
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
descendThrow(SHAMapInnerNode&, int branch) const;
// Descend with filter
// If pending, callback is called as if it called fetchNodeNT
using descendCallback = std::function<void(SHAMapTreeNodePtr, SHAMapHash const&)>;
using descendCallback =
std::function<void(intr_ptr::SharedPtr<SHAMapTreeNode>, SHAMapHash const&)>;
SHAMapTreeNode*
descendAsync(
SHAMapInnerNode* parent,
int branch,
SHAMapSyncFilter const* filter,
SHAMapSyncFilter* filter,
bool& pending,
descendCallback&&) const;
@@ -469,11 +429,11 @@ private:
SHAMapInnerNode* parent,
SHAMapNodeID const& parentID,
int branch,
SHAMapSyncFilter const* filter) const;
SHAMapSyncFilter* filter) const;
// Non-storing
// Does not hook the returned node to its parent
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
descendNoStore(SHAMapInnerNode&, int branch) const;
/** If there is only one leaf below this node, get its contents */
@@ -535,10 +495,10 @@ private:
// nodes we may have acquired from deferred reads
using DeferredNode = std::tuple<
SHAMapInnerNode*, // parent node
SHAMapNodeID, // parent node ID
int, // branch
SHAMapTreeNodePtr>; // node
SHAMapInnerNode*, // parent node
SHAMapNodeID, // parent node ID
int, // branch
intr_ptr::SharedPtr<SHAMapTreeNode>>; // node
int deferred_;
std::mutex deferLock_;
@@ -564,7 +524,7 @@ private:
gmn_ProcessDeferredReads(MissingNodes&);
// fetch from DB helper function
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
finishFetch(SHAMapHash const& hash, std::shared_ptr<NodeObject> const& object) const;
};

View File

@@ -27,7 +27,7 @@ public:
{
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
clone(std::uint32_t cowid) const final
{
return intr_ptr::make_shared<SHAMapAccountStateLeafNode>(item_, cowid, hash_);

View File

@@ -87,7 +87,7 @@ public:
void
partialDestructor() override;
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
clone(std::uint32_t cowid) const override;
SHAMapNodeType
@@ -121,19 +121,19 @@ public:
getChildHash(int m) const;
void
setChild(int m, SHAMapTreeNodePtr child);
setChild(int m, intr_ptr::SharedPtr<SHAMapTreeNode> child);
void
shareChild(int m, SHAMapTreeNodePtr const& child);
shareChild(int m, intr_ptr::SharedPtr<SHAMapTreeNode> const& child);
SHAMapTreeNode*
getChildPointer(int branch);
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
getChild(int branch);
SHAMapTreeNodePtr
canonicalizeChild(int branch, SHAMapTreeNodePtr node);
intr_ptr::SharedPtr<SHAMapTreeNode>
canonicalizeChild(int branch, intr_ptr::SharedPtr<SHAMapTreeNode> node);
// sync functions
bool
@@ -161,10 +161,10 @@ public:
void
invariants(bool is_root = false) const override;
static SHAMapTreeNodePtr
static intr_ptr::SharedPtr<SHAMapTreeNode>
makeFullInner(Slice data, SHAMapHash const& hash, bool hashValid);
static SHAMapTreeNodePtr
static intr_ptr::SharedPtr<SHAMapTreeNode>
makeCompressedInner(Slice data);
};

View File

@@ -166,6 +166,4 @@ private:
makeTransactionWithMeta(Slice data, SHAMapHash const& hash, bool hashValid);
};
using SHAMapTreeNodePtr = intr_ptr::SharedPtr<SHAMapTreeNode>;
} // namespace xrpl

View File

@@ -26,7 +26,7 @@ public:
{
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
clone(std::uint32_t cowid) const final
{
return intr_ptr::make_shared<SHAMapTxLeafNode>(item_, cowid, hash_);

View File

@@ -27,7 +27,7 @@ public:
{
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
clone(std::uint32_t cowid) const override
{
return intr_ptr::make_shared<SHAMapTxPlusMetaLeafNode>(item_, cowid, hash_);

View File

@@ -11,5 +11,5 @@ using TreeNodeCache = TaggedCache<
SHAMapTreeNode,
/*IsKeyCache*/ false,
intr_ptr::SharedWeakUnionPtr<SHAMapTreeNode>,
SHAMapTreeNodePtr>;
intr_ptr::SharedPtr<SHAMapTreeNode>>;
} // namespace xrpl

View File

@@ -148,7 +148,7 @@ public:
/** Get the number of elements in each array and a pointer to the start
of each array.
*/
[[nodiscard]] std::tuple<std::uint8_t, SHAMapHash*, SHAMapTreeNodePtr*>
[[nodiscard]] std::tuple<std::uint8_t, SHAMapHash*, intr_ptr::SharedPtr<SHAMapTreeNode>*>
getHashesAndChildren() const;
/** Get the `hashes` array */
@@ -156,7 +156,7 @@ public:
getHashes() const;
/** Get the `children` array */
[[nodiscard]] SHAMapTreeNodePtr*
[[nodiscard]] intr_ptr::SharedPtr<SHAMapTreeNode>*
getChildren() const;
/** Call the `f` callback for all 16 (branchFactor) branches - even if

View File

@@ -25,7 +25,8 @@ static_assert(
// Terminology: A chunk is the memory being allocated from a block. A block
// contains multiple chunks. This is the terminology the boost documentation
// uses. Pools use "Simple Segregated Storage" as their storage format.
constexpr size_t elementSizeBytes = (sizeof(SHAMapHash) + sizeof(SHAMapTreeNodePtr));
constexpr size_t elementSizeBytes =
(sizeof(SHAMapHash) + sizeof(intr_ptr::SharedPtr<SHAMapTreeNode>));
constexpr size_t blockSizeBytes = kilobytes(512);
@@ -362,7 +363,8 @@ inline TaggedPointer::TaggedPointer(
// keep
new (&dstHashes[dstIndex]) SHAMapHash{srcHashes[srcIndex]};
new (&dstChildren[dstIndex]) SHAMapTreeNodePtr{std::move(srcChildren[srcIndex])};
new (&dstChildren[dstIndex])
intr_ptr::SharedPtr<SHAMapTreeNode>{std::move(srcChildren[srcIndex])};
++dstIndex;
++srcIndex;
}
@@ -373,7 +375,7 @@ inline TaggedPointer::TaggedPointer(
if (dstIsDense)
{
new (&dstHashes[dstIndex]) SHAMapHash{};
new (&dstChildren[dstIndex]) SHAMapTreeNodePtr{};
new (&dstChildren[dstIndex]) intr_ptr::SharedPtr<SHAMapTreeNode>{};
++dstIndex;
}
}
@@ -381,7 +383,7 @@ inline TaggedPointer::TaggedPointer(
{
// add
new (&dstHashes[dstIndex]) SHAMapHash{};
new (&dstChildren[dstIndex]) SHAMapTreeNodePtr{};
new (&dstChildren[dstIndex]) intr_ptr::SharedPtr<SHAMapTreeNode>{};
++dstIndex;
if (srcIsDense)
{
@@ -394,7 +396,7 @@ inline TaggedPointer::TaggedPointer(
if (dstIsDense)
{
new (&dstHashes[dstIndex]) SHAMapHash{};
new (&dstChildren[dstIndex]) SHAMapTreeNodePtr{};
new (&dstChildren[dstIndex]) intr_ptr::SharedPtr<SHAMapTreeNode>{};
++dstIndex;
}
if (srcIsDense)
@@ -411,7 +413,7 @@ inline TaggedPointer::TaggedPointer(
for (int i = dstIndex; i < dstNumAllocated; ++i)
{
new (&dstHashes[i]) SHAMapHash{};
new (&dstChildren[i]) SHAMapTreeNodePtr{};
new (&dstChildren[i]) intr_ptr::SharedPtr<SHAMapTreeNode>{};
}
*this = std::move(dst);
}
@@ -431,7 +433,7 @@ inline TaggedPointer::TaggedPointer(
// allocate hashes and children, but do not run constructors
TaggedPointer newHashesAndChildren{RawAllocateTag{}, toAllocate};
SHAMapHash *newHashes, *oldHashes;
SHAMapTreeNodePtr *newChildren, *oldChildren;
intr_ptr::SharedPtr<SHAMapTreeNode>*newChildren, *oldChildren;
std::uint8_t newNumAllocated;
// structured bindings can't be captured in c++ 17; use tie instead
std::tie(newNumAllocated, newHashes, newChildren) = newHashesAndChildren.getHashesAndChildren();
@@ -442,7 +444,8 @@ inline TaggedPointer::TaggedPointer(
// new arrays are dense, old arrays are sparse
iterNonEmptyChildIndexes(isBranch, [&](auto branchNum, auto indexNum) {
new (&newHashes[branchNum]) SHAMapHash{oldHashes[indexNum]};
new (&newChildren[branchNum]) SHAMapTreeNodePtr{std::move(oldChildren[indexNum])};
new (&newChildren[branchNum])
intr_ptr::SharedPtr<SHAMapTreeNode>{std::move(oldChildren[indexNum])};
});
// Run the constructors for the remaining elements
for (int i = 0; i < SHAMapInnerNode::branchFactor; ++i)
@@ -450,7 +453,7 @@ inline TaggedPointer::TaggedPointer(
if ((1 << i) & isBranch)
continue;
new (&newHashes[i]) SHAMapHash{};
new (&newChildren[i]) SHAMapTreeNodePtr{};
new (&newChildren[i]) intr_ptr::SharedPtr<SHAMapTreeNode>{};
}
}
else
@@ -460,14 +463,14 @@ inline TaggedPointer::TaggedPointer(
iterNonEmptyChildIndexes(isBranch, [&](auto branchNum, auto indexNum) {
new (&newHashes[curCompressedIndex]) SHAMapHash{oldHashes[indexNum]};
new (&newChildren[curCompressedIndex])
SHAMapTreeNodePtr{std::move(oldChildren[indexNum])};
intr_ptr::SharedPtr<SHAMapTreeNode>{std::move(oldChildren[indexNum])};
++curCompressedIndex;
});
// Run the constructors for the remaining elements
for (int i = curCompressedIndex; i < newNumAllocated; ++i)
{
new (&newHashes[i]) SHAMapHash{};
new (&newChildren[i]) SHAMapTreeNodePtr{};
new (&newChildren[i]) intr_ptr::SharedPtr<SHAMapTreeNode>{};
}
}
@@ -481,7 +484,7 @@ inline TaggedPointer::TaggedPointer(std::uint8_t numChildren)
for (std::size_t i = 0; i < numAllocated; ++i)
{
new (&hashes[i]) SHAMapHash{};
new (&children[i]) SHAMapTreeNodePtr{};
new (&children[i]) intr_ptr::SharedPtr<SHAMapTreeNode>{};
}
}
@@ -519,13 +522,14 @@ TaggedPointer::isDense() const
return (tp_ & tagMask) == boundaries.size() - 1;
}
[[nodiscard]] inline std::tuple<std::uint8_t, SHAMapHash*, SHAMapTreeNodePtr*>
[[nodiscard]] inline std::tuple<std::uint8_t, SHAMapHash*, intr_ptr::SharedPtr<SHAMapTreeNode>*>
TaggedPointer::getHashesAndChildren() const
{
auto const [tag, ptr] = decode();
auto const hashes = reinterpret_cast<SHAMapHash*>(ptr);
std::uint8_t numAllocated = boundaries[tag];
auto const children = reinterpret_cast<SHAMapTreeNodePtr*>(hashes + numAllocated);
auto const children =
reinterpret_cast<intr_ptr::SharedPtr<SHAMapTreeNode>*>(hashes + numAllocated);
return {numAllocated, hashes, children};
};
@@ -535,7 +539,7 @@ TaggedPointer::getHashes() const
return reinterpret_cast<SHAMapHash*>(tp_ & ptrMask);
};
[[nodiscard]] inline SHAMapTreeNodePtr*
[[nodiscard]] inline intr_ptr::SharedPtr<SHAMapTreeNode>*
TaggedPointer::getChildren() const
{
auto [unused1, unused2, result] = getHashesAndChildren();

160
package/README.md Normal file
View File

@@ -0,0 +1,160 @@
# Linux Packaging
This directory contains all files needed to build RPM and Debian packages for `xrpld`.
## Directory layout
```
package/
build_pkg.sh Staging and build script (called by CMake targets and CI)
rpm/
xrpld.spec RPM spec (xrpld_version/pkg_release passed via rpmbuild --define)
debian/ Debian control files (control, rules, install, links, conffiles, ...)
shared/
xrpld.service systemd unit file (used by both RPM and DEB)
xrpld.sysusers sysusers.d config (used by both RPM and DEB)
xrpld.tmpfiles tmpfiles.d config (used by both RPM and DEB)
xrpld.logrotate logrotate config (installed to /opt/xrpld/bin/, user activates)
update-xrpld.sh auto-update script (installed to /opt/xrpld/bin/)
update-xrpld-cron cron entry for auto-update (installed to /opt/xrpld/bin/)
test/
smoketest.sh Package install smoke test
check_install_paths.sh Verify install paths and compat symlinks
```
## Prerequisites
Packaging targets and their container images are declared in
[`.github/scripts/strategy-matrix/linux.json`](../.github/scripts/strategy-matrix/linux.json)
via a `"package": true` field on specific os entries. The package format
(deb or rpm) is inferred at build time from the container's package manager
(`apt-get` -> deb, `dnf`/`yum` -> rpm). The image tag is composed as
`ghcr.io/xrplf/ci/{distro}-{version}:{compiler}-{cver}-sha-{image_sha}`
the same scheme used by `reusable-build-test.yml`. Bump `image_sha` in
`linux.json` and both CI and local builds pick up the new image with no
workflow edits.
| Package type | Image (derived from `linux.json`) | Tool required |
| ------------ | ---------------------------------------------------- | --------------------------------------------------------------- |
| RPM | `ghcr.io/xrplf/ci/rhel-9:gcc-12-sha-<git_sha>` | `rpmbuild` |
| DEB | `ghcr.io/xrplf/ci/ubuntu-jammy:gcc-12-sha-<git_sha>` | `dpkg-buildpackage`, `debhelper (>= 13)`, `dh-sequence-systemd` |
To print the exact image tags for the current `linux.json`:
```bash
./.github/scripts/strategy-matrix/generate.py --packaging --config=.github/scripts/strategy-matrix/linux.json
```
## Building packages
### Via CI
Caller workflows (`on-pr.yml`, `on-tag.yml`, `on-trigger.yml`) call
`reusable-strategy-matrix.yml` with `mode: packaging` to generate the matrix of
`{artifact_name, container_image}` entries, then fan out to
`reusable-package.yml` per entry. That workflow downloads the pre-built `xrpld`
binary artifact, detects the package format from the container, and calls
`build_pkg.sh` directly — no CMake configure or build step is needed inside
the packaging job.
### Locally (mirrors CI)
With an `xrpld` binary already built at `build/xrpld`, run the packaging step
inside the same container CI uses. The image tag is derived from `linux.json`
so you don't need to hardcode a SHA.
```bash
# From the repo root. Pick any image flagged with `"package": true` in
# linux.json; the package format is inferred from the container's package
# manager. Example for the rpm-producing image:
IMAGE=$(jq -r '
.os | map(select(.package == true))[0] |
"ghcr.io/xrplf/ci/\(.distro_name)-\(.distro_version):\(.compiler_name)-\(.compiler_version)-sha-\(.image_sha)"
' .github/scripts/strategy-matrix/linux.json)
VERSION=2.4.0-local
PKG_RELEASE=1
docker run --rm \
-v "$(pwd):/src" \
-w /src \
"$IMAGE" \
./package/build_pkg.sh . build "$VERSION" "$PKG_RELEASE"
# Output:
# build/debbuild/*.deb (DEB + dbgsym .ddeb)
# build/rpmbuild/RPMS/x86_64/*.rpm
```
### Via CMake (host-side target)
If you run CMake configure on a host that has `rpmbuild` or `dpkg-buildpackage`
installed natively, you can use the CMake target directly — no container
needed, but the host toolchain replaces the pinned CI image:
```bash
cmake \
-DCMAKE_INSTALL_PREFIX=/opt/xrpld \
-Dxrpld=ON \
-Dxrpld_version=2.4.0-local \
-Dtests=OFF \
..
cmake --build . --target package # deb on Debian/Ubuntu, rpm on RHEL
```
The `cmake/XrplPackaging.cmake` module defines the target only if at least one
of `rpmbuild` / `dpkg-buildpackage` is present; `build_pkg.sh` then infers the
package format from the host's package manager. `CMAKE_INSTALL_PREFIX` must
be `/opt/xrpld`; if it is not, both targets are skipped with a `STATUS`
message.
## How `build_pkg.sh` works
`build_pkg.sh <src_dir> <build_dir> [version] [pkg_release]` stages
all files and invokes the platform build tool. It resolves `src_dir` and
`build_dir` to absolute paths, then calls `stage_common()` to copy the binary,
config files, and shared support files into the staging area. The package
format is taken from the `PKG_TYPE` env var if set; otherwise it is inferred
from the available package manager (`apt-get` -> deb, `dnf`/`yum` -> rpm).
### RPM
1. Creates the standard `rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}` tree inside the build directory.
2. Copies `xrpld.spec` and all source files (binary, configs, service files) into `SOURCES/`.
3. Runs `rpmbuild -bb --define "xrpld_version ..." --define "pkg_release ..."`. The spec uses manual `install` commands to place files.
4. Output: `rpmbuild/RPMS/x86_64/xrpld-*.rpm`
### DEB
1. Creates a staging source tree at `debbuild/source/` inside the build directory.
2. Stages the binary, configs, `README.md`, and `LICENSE.md`.
3. Copies `package/deb/debian/` control files into `debbuild/source/debian/`.
4. Copies shared service/sysusers/tmpfiles into `debian/` where `dh_installsystemd`, `dh_installsysusers`, and `dh_installtmpfiles` pick them up automatically.
5. Generates a minimal `debian/changelog` (pre-release versions use `~` instead of `-`).
6. Runs `dpkg-buildpackage -b --no-sign`. `debian/rules` uses manual `install` commands.
7. Output: `debbuild/*.deb` and `debbuild/*.ddeb` (dbgsym package)
## Post-build verification
```bash
# DEB
dpkg-deb -c debbuild/*.deb | grep -E 'systemd|sysusers|tmpfiles'
lintian -I debbuild/*.deb
# RPM
rpm -qlp rpmbuild/RPMS/x86_64/*.rpm
```
## Reproducibility
The following environment variables improve build reproducibility. They are not
set automatically by `build_pkg.sh`; set them manually if needed:
```bash
export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
export TZ=UTC
export LC_ALL=C.UTF-8
export GZIP=-n
export DEB_BUILD_OPTIONS="noautodbgsym reproducible=+fixfilepath"
```

134
package/build_pkg.sh Executable file
View File

@@ -0,0 +1,134 @@
#!/usr/bin/env bash
set -euo pipefail
# Build an RPM or Debian package from a pre-built xrpld binary.
#
# Usage: build_pkg.sh <src_dir> <build_dir> [version] [pkg_release]
# src_dir : path to repository root
# build_dir : directory containing the pre-built xrpld binary
# version : package version string (e.g. 3.2.0-b1)
# pkg_release : package release number (default: 1)
#
# The package format is taken from the PKG_TYPE env var if set; otherwise it
# is inferred from the available package manager (apt-get -> deb, dnf/yum -> rpm).
SRC_DIR="$(cd "${1:?src_dir required}" && pwd)"
BUILD_DIR="$(cd "${2:?build_dir required}" && pwd)"
VERSION="${3:-$("${BUILD_DIR}/xrpld" --version | awk 'NR==1 {print $3}')}"
PKG_RELEASE="${4:-1}"
if [[ -z "${PKG_TYPE:-}" ]]; then
if command -v apt-get >/dev/null 2>&1; then
PKG_TYPE=deb
elif command -v dnf >/dev/null 2>&1 || command -v yum >/dev/null 2>&1; then
PKG_TYPE=rpm
else
echo "Cannot infer PKG_TYPE: no apt-get, dnf, or yum on PATH." >&2
exit 1
fi
fi
if [[ -z "${SOURCE_DATE_EPOCH:-}" ]]; then
if git -C "$SRC_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
SOURCE_DATE_EPOCH="$(git -C "$SRC_DIR" log -1 --format=%ct)"
else
SOURCE_DATE_EPOCH="$(date +%s)"
fi
fi
export SOURCE_DATE_EPOCH
CHANGELOG_DATE="$(date -u -R -d "@$SOURCE_DATE_EPOCH")"
# Split VERSION at the first '-' into base and optional pre-release suffix.
# Examples: "3.2.0" -> ("3.2.0", ""); "3.2.0-b1" -> ("3.2.0", "b1").
VER_BASE="${VERSION%%-*}"
VER_SUFFIX="${VERSION#*-}"
[[ "${VER_SUFFIX}" == "${VERSION}" ]] && VER_SUFFIX=""
SHARED="${SRC_DIR}/package/shared"
DEBIAN_DIR="${SRC_DIR}/package/debian"
# Stage files that both packaging systems consume using the same filenames.
stage_common() {
local dest="$1"
mkdir -p "${dest}"
cp "${BUILD_DIR}/xrpld" "${dest}/xrpld"
cp "${SRC_DIR}/cfg/xrpld-example.cfg" "${dest}/xrpld.cfg"
cp "${SRC_DIR}/cfg/validators-example.txt" "${dest}/validators.txt"
cp "${SRC_DIR}/LICENSE.md" "${dest}/LICENSE.md"
cp "${SRC_DIR}/README.md" "${dest}/README.md"
cp "${SHARED}/xrpld.service" "${dest}/xrpld.service"
cp "${SHARED}/xrpld.sysusers" "${dest}/xrpld.sysusers"
cp "${SHARED}/xrpld.tmpfiles" "${dest}/xrpld.tmpfiles"
cp "${SHARED}/xrpld.logrotate" "${dest}/xrpld.logrotate"
cp "${SHARED}/update-xrpld.sh" "${dest}/update-xrpld.sh"
cp "${SHARED}/update-xrpld-cron" "${dest}/update-xrpld-cron"
cp "${SHARED}/update-xrpld.service" "${dest}/update-xrpld.service"
cp "${SHARED}/update-xrpld.timer" "${dest}/update-xrpld.timer"
cp "${SHARED}/50-xrpld.preset" "${dest}/50-xrpld.preset"
}
build_rpm() {
local topdir="${BUILD_DIR}/rpmbuild"
rm -rf "${topdir}"
mkdir -p "${topdir}"/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}
cp "${SRC_DIR}/package/rpm/xrpld.spec" "${topdir}/SPECS/xrpld.spec"
stage_common "${topdir}/SOURCES"
# RPM Version can't contain '-'. A pre-release goes in Release with a
# leading "0." so 3.2.0-b1 sorts before the final 3.2.0-<pkg_release>.
local rpm_release="${PKG_RELEASE}"
[[ -n "${VER_SUFFIX}" ]] && rpm_release="0.${VER_SUFFIX}.${PKG_RELEASE}"
set -x
rpmbuild -bb \
--define "_topdir ${topdir}" \
--define "xrpld_version ${VER_BASE}" \
--define "xrpld_release ${rpm_release}" \
"${topdir}/SPECS/xrpld.spec"
}
build_deb() {
local staging="${BUILD_DIR}/debbuild/source"
rm -rf "${staging}"
mkdir -p "${staging}"
stage_common "${staging}"
cp -r "${DEBIAN_DIR}" "${staging}/debian"
# Debhelper auto-discovers these only from debian/.
cp "${staging}/xrpld.service" "${staging}/debian/xrpld.service"
cp "${staging}/xrpld.sysusers" "${staging}/debian/xrpld.sysusers"
cp "${staging}/xrpld.tmpfiles" "${staging}/debian/xrpld.tmpfiles"
cp "${staging}/update-xrpld.service" "${staging}/debian/xrpld.update-xrpld.service"
cp "${staging}/update-xrpld.timer" "${staging}/debian/xrpld.update-xrpld.timer"
# Debian '~' marks a pre-release; 3.2.0~b1 sorts before 3.2.0.
local deb_full_version="${VER_BASE}${VER_SUFFIX:+~${VER_SUFFIX}}-${PKG_RELEASE}"
# Derive release channel from the version suffix:
# (none) -> stable (tagged release)
# b0 -> develop (develop-branch build)
# b<N>, rc<N> -> unstable (pre-release)
local deb_distribution
case "${VER_SUFFIX}" in
"") deb_distribution="stable" ;;
b0) deb_distribution="develop" ;;
*) deb_distribution="unstable" ;;
esac
cat > "${staging}/debian/changelog" <<EOF
xrpld (${deb_full_version}) ${deb_distribution}; urgency=medium
* Release ${VERSION}.
-- XRPL Foundation <contact@xrplf.org> ${CHANGELOG_DATE}
EOF
chmod +x "${staging}/debian/rules"
set -x
( cd "${staging}" && dpkg-buildpackage -b --no-sign -d )
}
"build_${PKG_TYPE}"

23
package/debian/control Normal file
View File

@@ -0,0 +1,23 @@
Source: xrpld
Section: net
Priority: optional
Maintainer: XRPL Foundation <contact@xrplf.org>
Rules-Requires-Root: no
Build-Depends:
debhelper-compat (= 13)
Standards-Version: 4.7.0
Homepage: https://github.com/XRPLF/rippled
Vcs-Git: https://github.com/XRPLF/rippled.git
Vcs-Browser: https://github.com/XRPLF/rippled
Package: xrpld
Section: net
Priority: optional
Architecture: any
Depends:
${shlibs:Depends},
${misc:Depends}
Description: XRP Ledger daemon
Reference implementation of the XRP Ledger protocol. Participates
in the peer-to-peer network, processes transactions, and maintains
a local ledger copy.

20
package/debian/copyright Normal file
View File

@@ -0,0 +1,20 @@
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: rippled
Source: https://github.com/XRPLF/rippled
Files: *
Copyright: 2012-2026 Ripple Labs Inc.
License: ISC
License: ISC
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

34
package/debian/rules Normal file
View File

@@ -0,0 +1,34 @@
#!/usr/bin/make -f
export DH_VERBOSE = 1
export DH_OPTIONS = -v
%:
dh $@
override_dh_auto_configure override_dh_auto_build override_dh_auto_test:
@:
override_dh_auto_install:
install -Dm0755 xrpld debian/tmp/opt/xrpld/bin/xrpld
install -Dm0644 xrpld.cfg debian/tmp/opt/xrpld/etc/xrpld.cfg
install -Dm0644 validators.txt debian/tmp/opt/xrpld/etc/validators.txt
install -Dm0644 xrpld.logrotate debian/tmp/opt/xrpld/bin/xrpld.logrotate
install -Dm0755 update-xrpld.sh debian/tmp/opt/xrpld/bin/update-xrpld.sh
install -Dm0644 update-xrpld-cron debian/tmp/opt/xrpld/bin/update-xrpld-cron
install -Dm0644 README.md debian/tmp/usr/share/doc/xrpld/README.md
install -Dm0644 LICENSE.md debian/tmp/usr/share/doc/xrpld/LICENSE.md
# update-xrpld.service is a Type=oneshot fired by update-xrpld.timer; installing
# it without --no-start would run the update on package install.
override_dh_installsystemd:
dh_installsystemd xrpld.service
dh_installsystemd --name=update-xrpld --no-enable --no-start update-xrpld.service update-xrpld.timer
execute_before_dh_installtmpfiles:
dh_installsysusers
override_dh_installsysusers:
override_dh_dwz:
@:

View File

@@ -0,0 +1 @@
3.0 (quilt)

View File

@@ -0,0 +1,2 @@
/opt/xrpld/etc/xrpld.cfg
/opt/xrpld/etc/validators.txt

View File

@@ -0,0 +1,10 @@
opt/xrpld/bin/xrpld
opt/xrpld/bin/xrpld.logrotate
opt/xrpld/bin/update-xrpld.sh
opt/xrpld/bin/update-xrpld-cron
opt/xrpld/etc/xrpld.cfg
opt/xrpld/etc/validators.txt
usr/share/doc/xrpld/README.md
usr/share/doc/xrpld/LICENSE.md

View File

@@ -0,0 +1,10 @@
opt/xrpld/etc etc/opt/xrpld
opt/xrpld/bin/xrpld usr/bin/xrpld
# remove when "rippled" deprecated
opt/xrpld/bin/xrpld opt/xrpld/bin/rippled
opt/xrpld/bin/xrpld usr/local/bin/rippled
opt/xrpld/etc/xrpld.cfg opt/xrpld/etc/rippled.cfg
opt/xrpld opt/ripple
etc/opt/xrpld etc/opt/ripple

111
package/rpm/xrpld.spec Normal file
View File

@@ -0,0 +1,111 @@
Name: xrpld
Version: %{xrpld_version}
Release: %{xrpld_release}%{?dist}
Summary: XRP Ledger daemon
License: ISC
URL: https://github.com/XRPLF/rippled
ExclusiveArch: x86_64 aarch64
BuildRequires: systemd-rpm-macros
%undefine _debugsource_packages
%debug_package
%build_mtime_policy clamp_to_source_date_epoch
%{?systemd_requires}
%{?sysusers_requires_compat}
%description
xrpld is the reference implementation of the XRP Ledger protocol. It
participates in the peer-to-peer XRP Ledger network, processes
transactions, and maintains the ledger database.
%prep
:
%install
rm -rf %{buildroot}
SRC=%{_sourcedir}
install -Dm0755 ${SRC}/xrpld %{buildroot}/opt/xrpld/bin/xrpld
install -Dm0644 ${SRC}/xrpld.cfg %{buildroot}/opt/xrpld/etc/xrpld.cfg
install -Dm0644 ${SRC}/validators.txt %{buildroot}/opt/xrpld/etc/validators.txt
mkdir -p %{buildroot}/etc/opt %{buildroot}/usr/bin %{buildroot}/usr/local/bin
ln -s /opt/xrpld/etc %{buildroot}/etc/opt/xrpld
ln -s /opt/xrpld/bin/xrpld %{buildroot}/usr/bin/xrpld
# TODO: remove when rippled deprecated
ln -s xrpld %{buildroot}/opt/xrpld/bin/rippled
ln -s /opt/xrpld/bin/xrpld %{buildroot}/usr/local/bin/rippled
ln -s xrpld.cfg %{buildroot}/opt/xrpld/etc/rippled.cfg
ln -s /opt/xrpld %{buildroot}/opt/ripple
ln -s /etc/opt/xrpld %{buildroot}/etc/opt/ripple
install -Dm0644 ${SRC}/xrpld.service %{buildroot}%{_unitdir}/xrpld.service
install -Dm0644 ${SRC}/update-xrpld.service %{buildroot}%{_unitdir}/update-xrpld.service
install -Dm0644 ${SRC}/update-xrpld.timer %{buildroot}%{_unitdir}/update-xrpld.timer
install -Dm0644 ${SRC}/xrpld.sysusers %{buildroot}%{_sysusersdir}/xrpld.conf
install -Dm0644 ${SRC}/xrpld.tmpfiles %{buildroot}%{_tmpfilesdir}/xrpld.conf
install -Dm0644 ${SRC}/50-xrpld.preset %{buildroot}%{_presetdir}/50-xrpld.preset
install -Dm0755 ${SRC}/update-xrpld.sh %{buildroot}/opt/xrpld/bin/update-xrpld.sh
install -Dm0644 ${SRC}/update-xrpld-cron %{buildroot}/opt/xrpld/bin/update-xrpld-cron
install -Dm0644 ${SRC}/xrpld.logrotate %{buildroot}/opt/xrpld/bin/xrpld.logrotate
install -Dm0644 ${SRC}/LICENSE.md %{buildroot}/opt/xrpld/share/LICENSE.md
install -Dm0644 ${SRC}/README.md %{buildroot}/opt/xrpld/share/README.md
%pre
%sysusers_create_package xrpld %{_sourcedir}/xrpld.sysusers
%post
systemd-tmpfiles --create %{_tmpfilesdir}/xrpld.conf || :
%systemd_post xrpld.service update-xrpld.timer
%preun
%systemd_preun xrpld.service update-xrpld.timer
%postun
%systemd_postun_with_restart xrpld.service
%files
%license /opt/xrpld/share/LICENSE.md
%doc /opt/xrpld/share/README.md
%dir /opt/xrpld
%dir /opt/xrpld/bin
%dir /opt/xrpld/etc
/opt/xrpld/bin/xrpld
/opt/xrpld/bin/xrpld.logrotate
/opt/xrpld/bin/update-xrpld.sh
/opt/xrpld/bin/update-xrpld-cron
/usr/bin/xrpld
/etc/opt/xrpld
%config(noreplace) /opt/xrpld/etc/xrpld.cfg
%config(noreplace) /opt/xrpld/etc/validators.txt
%{_unitdir}/xrpld.service
%{_unitdir}/update-xrpld.service
%{_unitdir}/update-xrpld.timer
%{_presetdir}/50-xrpld.preset
%{_sysusersdir}/xrpld.conf
%{_tmpfilesdir}/xrpld.conf
%ghost %dir /var/lib/xrpld
%ghost %dir /var/log/xrpld
# TODO: remove when rippled deprecated
/opt/xrpld/bin/rippled
/usr/local/bin/rippled
/opt/xrpld/etc/rippled.cfg
/etc/opt/ripple
/opt/ripple

View File

@@ -0,0 +1,4 @@
# /usr/lib/systemd/system-preset/50-xrpld.preset
enable xrpld.service
# Don't enable automatic updates
disable update-xrpld.timer

View File

@@ -0,0 +1,9 @@
# For automatic updates, symlink this file to /etc/cron.d/
# Do not remove the newline at the end of this cron script
# bash required for use of RANDOM below.
SHELL=/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin
# invoke check/update script with random delay up to 59 mins
0 * * * * root sleep $((RANDOM*3540/32768)) && /opt/xrpld/bin/update-xrpld.sh

View File

@@ -0,0 +1,16 @@
[Unit]
Description=Check for and install xrpld package updates
Documentation=man:systemd.service(5)
Wants=network-online.target
After=network-online.target
ConditionPathExists=/opt/xrpld/bin/update-xrpld.sh
ConditionPathExists=/opt/xrpld/bin/xrpld
[Service]
Type=oneshot
ExecStart=/usr/bin/flock -n /run/lock/xrpld-update.lock /opt/xrpld/bin/update-xrpld.sh
StandardOutput=journal
StandardError=journal
SyslogIdentifier=update-xrpld
TimeoutStartSec=30min
PrivateTmp=true

145
package/shared/update-xrpld.sh Executable file
View File

@@ -0,0 +1,145 @@
#!/usr/bin/env bash
set -euo pipefail
# Optional: also write logs to a legacy file in addition to journald.
# By default, this script logs to systemd/journald, viewable via:
# journalctl -t update-xrpld
#
# Uncomment the line below if you need a flat file for compatibility with
# external tooling, manual inspection, or environments where journald logs
# are not persisted or easily accessible.
#
# Note: This duplicates all output (stdout/stderr) to both journald and the file.
# It is generally not needed on modern systems and may cause log file growth
# if left enabled long-term.
#
# Requires /var/log/xrpld/ to exist and be writable by the service (root).
#
# exec > >(tee -a /var/log/xrpld/update.log) 2>&1
PATH=/usr/sbin:/usr/bin:/sbin:/bin
PKG_NAME=${PKG_NAME:-xrpld}
log() {
# If running under systemd/journald, let it handle timestamps.
if [[ -n "${JOURNAL_STREAM:-}" ]]; then
printf '%s\n' "$*"
else
printf '%s %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*"
fi
}
require_root() {
if [[ ${EUID:-$(id -u)} -ne 0 ]]; then
log "RESULT: failed reason=not-root"
exit 1
fi
}
get_installed_version() {
if command -v dpkg-query >/dev/null 2>&1; then
dpkg-query -W -f='${Version}' "$PKG_NAME" 2>/dev/null || printf 'unknown'
elif command -v rpm >/dev/null 2>&1; then
rpm -q --qf '%{VERSION}-%{RELEASE}' "$PKG_NAME" 2>/dev/null || printf 'unknown'
else
printf 'unknown'
fi
}
trap 'log "RESULT: failed reason=script-error exit_code=$?"' ERR
apt_can_update() {
apt-get update -qq
apt-get -s --only-upgrade install "$PKG_NAME" 2>/dev/null | grep -q "^Inst ${PKG_NAME}\b"
}
apt_apply_update() {
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq \
-o Dpkg::Options::="--force-confdef" \
-o Dpkg::Options::="--force-confold" \
"$PKG_NAME"
}
get_rpm_pm() {
if command -v dnf >/dev/null 2>&1; then
printf 'dnf\n'
elif command -v yum >/dev/null 2>&1; then
printf 'yum\n'
else
return 1
fi
}
rpm_refresh_metadata() {
local pm=$1
if [[ "$pm" == "dnf" ]]; then
dnf makecache --refresh -q >/dev/null
else
yum clean expire-cache -q >/dev/null
fi
}
rpm_can_update() {
local pm=$1
rpm_refresh_metadata "$pm"
local rc=0
set +e
"$pm" check-update -q "$PKG_NAME" >/dev/null 2>&1
rc=$?
set -e
if [[ $rc -eq 100 ]]; then
return 0
elif [[ $rc -eq 0 ]]; then
return 1
else
log "$pm check-update failed with exit code $rc"
exit 1
fi
}
rpm_apply_update() {
local pm=$1
"$pm" update -y "$PKG_NAME"
}
restart_service() {
systemctl restart "${PKG_NAME}.service"
log "${PKG_NAME} service restarted successfully"
}
main() {
require_root
if command -v apt-get >/dev/null 2>&1; then
log "Checking for ${PKG_NAME} updates via apt"
if apt_can_update; then
log "Update available; installing"
apt_apply_update
restart_service
log "RESULT: updated ${PKG_NAME}=$(get_installed_version)"
else
log "RESULT: no-update ${PKG_NAME}=$(get_installed_version)"
fi
return
fi
local rpm_pm=""
if rpm_pm="$(get_rpm_pm)"; then
log "Checking for ${PKG_NAME} updates via ${rpm_pm}"
if rpm_can_update "$rpm_pm"; then
log "Update available; installing"
rpm_apply_update "$rpm_pm"
restart_service
log "RESULT: updated ${PKG_NAME}=$(get_installed_version)"
else
log "RESULT: no-update ${PKG_NAME}=$(get_installed_version)"
fi
return
fi
log "RESULT: failed reason=no-package-manager"
exit 1
}
main "$@"

View File

@@ -0,0 +1,10 @@
[Unit]
Description=Daily xrpld update check
[Timer]
OnCalendar=*-*-* 00:00:00
RandomizedDelaySec=1h
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,14 @@
/var/log/xrpld/*.log {
daily
minsize 200M
rotate 7
nocreate
missingok
notifempty
compress
compresscmd /usr/bin/gzip
compressext .gz
postrotate
/opt/xrpld/bin/xrpld --conf /etc/opt/xrpld/xrpld.cfg logrotate
endscript
}

View File

@@ -0,0 +1,22 @@
[Unit]
Description=XRP Ledger Daemon
After=network-online.target
Wants=network-online.target
StartLimitIntervalSec=300
StartLimitBurst=5
[Service]
Type=simple
ExecStart=/opt/xrpld/bin/xrpld --net --silent --conf /etc/opt/xrpld/xrpld.cfg
Restart=always
RestartSec=5s
NoNewPrivileges=true
ProtectSystem=full
ProtectHome=true
PrivateTmp=true
User=xrpld
Group=xrpld
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1 @@
u xrpld - "XRP Ledger daemon" /var/lib/xrpld /sbin/nologin

View File

@@ -0,0 +1,2 @@
d /var/lib/xrpld 0750 xrpld xrpld -
d /var/log/xrpld 0750 xrpld xrpld -

View File

@@ -0,0 +1,45 @@
#!/usr/bin/env sh
# Validate installed paths and compat symlinks for xrpld packages.
set -e
set -x
trap 'test $? -ne 0 && touch /tmp/test_failed' EXIT
check() { test $1 "$2" || { echo "FAIL: $1 $2"; exit 1; }; }
check_resolves_to() {
actual=$(readlink -f "$1")
[ "$actual" = "$2" ] || { echo "FAIL: $1 resolves to $actual, expected $2"; exit 1; }
}
# compat directory symlinks — existence and resolved target
check -L /opt/ripple
check_resolves_to /opt/ripple /opt/xrpld
check -L /etc/opt/xrpld
check_resolves_to /etc/opt/xrpld /opt/xrpld/etc
check -L /etc/opt/ripple
check_resolves_to /etc/opt/ripple /opt/xrpld/etc
# config accessible via all expected paths
check -f /opt/xrpld/etc/xrpld.cfg
check -f /opt/xrpld/etc/rippled.cfg
check -f /etc/opt/xrpld/xrpld.cfg
check -f /etc/opt/xrpld/rippled.cfg
check -f /etc/opt/ripple/xrpld.cfg
check -f /etc/opt/ripple/rippled.cfg
if systemctl is-system-running >/dev/null 2>&1; then
# service file sanity check
SERVICE=$(systemctl cat xrpld)
echo "$SERVICE" | grep -q 'ExecStart=/opt/xrpld/bin/xrpld' || { echo "FAIL: ExecStart wrong"; echo "$SERVICE"; exit 1; }
echo "$SERVICE" | grep -q 'User=xrpld' || { echo "FAIL: User not xrpld"; echo "$SERVICE"; exit 1; }
fi
# binary accessible via all expected paths
/opt/xrpld/bin/xrpld --version
/opt/xrpld/bin/rippled --version
/opt/ripple/bin/xrpld --version
/opt/ripple/bin/rippled --version
/usr/bin/xrpld --version
/usr/local/bin/rippled --version

91
package/test/smoketest.sh Executable file
View File

@@ -0,0 +1,91 @@
#!/usr/bin/env bash
# Install a locally-built package and run basic verification.
#
# Usage: smoketest.sh local
# Expects packages in build/{dpkg,rpm}/packages/ or build/debbuild/ / build/rpmbuild/RPMS/
set -o pipefail
set -x
rm -f /tmp/test_failed /tmp/unittest_results
trap 'test $? -ne 0 && touch /tmp/test_failed' EXIT
install_from=$1
. /etc/os-release
case ${ID} in
ubuntu|debian)
pkgtype="dpkg"
;;
fedora|centos|rhel|rocky|almalinux)
pkgtype="rpm"
;;
*)
echo "unrecognized distro!"
exit 1
;;
esac
if [ "${install_from}" != "local" ]; then
echo "only 'local' install mode is supported"
exit 1
fi
# Install the package
if [ "${pkgtype}" = "dpkg" ] ; then
apt-get -y update
# Find .deb files — check both possible output locations
mapfile -t debs < <(find build/debbuild/ build/dpkg/packages/ -name '*.deb' ! -name '*dbgsym*' 2>/dev/null)
if [ ${#debs[@]} -eq 0 ]; then
echo "No .deb files found"
exit 1
fi
dpkg --no-debsig -i "${debs[@]}" || apt-get -y install -f || { echo "DEB install failed"; exit 1; }
elif [ "${pkgtype}" = "rpm" ] ; then
# Find .rpm files — check both possible output locations
mapfile -t rpms < <(find build/rpmbuild/RPMS/ build/rpm/packages/ -name '*.rpm' \
! -name '*debug*' ! -name '*devel*' ! -name '*.src.rpm' 2>/dev/null)
if [ ${#rpms[@]} -eq 0 ]; then
echo "No .rpm files found"
exit 1
fi
rpm -i "${rpms[@]}" || { echo "RPM install failed"; exit 1; }
fi
# Verify installed version
if ! VERSION_OUTPUT=$(/opt/xrpld/bin/xrpld --version); then
echo "xrpld --version failed; binary not installed correctly"
exit 1
fi
INSTALLED=$(echo "$VERSION_OUTPUT" | head -1 | awk '{print $NF}')
echo "Installed version: ${INSTALLED}"
# Run unit tests
if [ -n "${CI:-}" ]; then
unittest_jobs=$(nproc)
else
unittest_jobs=16
fi
(
cd /tmp
/opt/xrpld/bin/xrpld --unittest --unittest-jobs "${unittest_jobs}" > /tmp/unittest_results || true
)
if [ ! -s /tmp/unittest_results ]; then
echo "Unit test results file is empty — xrpld may have crashed"
exit 1
fi
num_failures=$(tail /tmp/unittest_results -n1 | grep -oP '\d+(?= failures)')
if [ -z "$num_failures" ]; then
echo "Could not parse unit test results — expected summary line not found"
exit 1
fi
if [ "${num_failures}" -ne 0 ]; then
echo "$num_failures unit test(s) failed:"
grep 'failed:' /tmp/unittest_results
exit 1
fi
# Compat path checks
"$(dirname "${BASH_SOURCE[0]}")/check_install_paths.sh"

View File

@@ -97,7 +97,10 @@ SHAMap::snapShot(bool isMutable) const
}
void
SHAMap::dirtyUp(SharedPtrNodeStack& stack, uint256 const& target, SHAMapTreeNodePtr child)
SHAMap::dirtyUp(
SharedPtrNodeStack& stack,
uint256 const& target,
intr_ptr::SharedPtr<SHAMapTreeNode> child)
{
// walk the tree up from through the inner nodes to the root_
// update hashes and links
@@ -162,7 +165,7 @@ SHAMap::findKey(uint256 const& id) const
return leaf;
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const
{
XRPL_ASSERT(backed_, "xrpl::SHAMap::fetchNodeFromDB : is backed");
@@ -170,7 +173,7 @@ SHAMap::fetchNodeFromDB(SHAMapHash const& hash) const
return finishFetch(hash, obj);
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::finishFetch(SHAMapHash const& hash, std::shared_ptr<NodeObject> const& object) const
{
XRPL_ASSERT(backed_, "xrpl::SHAMap::finishFetch : is backed");
@@ -205,8 +208,8 @@ SHAMap::finishFetch(SHAMapHash const& hash, std::shared_ptr<NodeObject> const& o
}
// See if a sync filter has a node
SHAMapTreeNodePtr
SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
{
if (auto nodeData = filter->getNode(hash))
{
@@ -231,8 +234,8 @@ SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) cons
// Get a node without throwing
// Used on maps where missing nodes are expected
SHAMapTreeNodePtr
SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
{
auto node = cacheLookup(hash);
if (node)
@@ -254,7 +257,7 @@ SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter const* filter) cons
return node;
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::fetchNodeNT(SHAMapHash const& hash) const
{
auto node = cacheLookup(hash);
@@ -266,7 +269,7 @@ SHAMap::fetchNodeNT(SHAMapHash const& hash) const
}
// Throw if the node is missing
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::fetchNode(SHAMapHash const& hash) const
{
auto node = fetchNodeNT(hash);
@@ -288,10 +291,10 @@ SHAMap::descendThrow(SHAMapInnerNode* parent, int branch) const
return ret;
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::descendThrow(SHAMapInnerNode& parent, int branch) const
{
SHAMapTreeNodePtr ret = descend(parent, branch);
intr_ptr::SharedPtr<SHAMapTreeNode> ret = descend(parent, branch);
if (!ret && !parent.isEmptyBranch(branch))
Throw<SHAMapMissingNode>(type_, parent.getChildHash(branch));
@@ -306,7 +309,7 @@ SHAMap::descend(SHAMapInnerNode* parent, int branch) const
if ((ret != nullptr) || !backed_)
return ret;
SHAMapTreeNodePtr node = fetchNodeNT(parent->getChildHash(branch));
intr_ptr::SharedPtr<SHAMapTreeNode> node = fetchNodeNT(parent->getChildHash(branch));
if (!node)
return nullptr;
@@ -314,10 +317,10 @@ SHAMap::descend(SHAMapInnerNode* parent, int branch) const
return node.get();
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::descend(SHAMapInnerNode& parent, int branch) const
{
SHAMapTreeNodePtr node = parent.getChild(branch);
intr_ptr::SharedPtr<SHAMapTreeNode> node = parent.getChild(branch);
if (node || !backed_)
return node;
@@ -331,10 +334,10 @@ SHAMap::descend(SHAMapInnerNode& parent, int branch) const
// Gets the node that would be hooked to this branch,
// but doesn't hook it up.
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::descendNoStore(SHAMapInnerNode& parent, int branch) const
{
SHAMapTreeNodePtr ret = parent.getChild(branch);
intr_ptr::SharedPtr<SHAMapTreeNode> ret = parent.getChild(branch);
if (!ret && backed_)
ret = fetchNode(parent.getChildHash(branch));
return ret;
@@ -345,7 +348,7 @@ SHAMap::descend(
SHAMapInnerNode* parent,
SHAMapNodeID const& parentID,
int branch,
SHAMapSyncFilter const* filter) const
SHAMapSyncFilter* filter) const
{
XRPL_ASSERT(parent->isInner(), "xrpl::SHAMap::descend : valid parent input");
XRPL_ASSERT(
@@ -358,7 +361,7 @@ SHAMap::descend(
if (child == nullptr)
{
auto const& childHash = parent->getChildHash(branch);
SHAMapTreeNodePtr childNode = fetchNodeNT(childHash, filter);
intr_ptr::SharedPtr<SHAMapTreeNode> childNode = fetchNodeNT(childHash, filter);
if (childNode)
{
@@ -374,7 +377,7 @@ SHAMapTreeNode*
SHAMap::descendAsync(
SHAMapInnerNode* parent,
int branch,
SHAMapSyncFilter const* filter,
SHAMapSyncFilter* filter,
bool& pending,
descendCallback&& callback) const
{
@@ -431,7 +434,7 @@ SHAMap::unshareNode(intr_ptr::SharedPtr<Node> node, SHAMapNodeID const& nodeID)
SHAMapLeafNode*
SHAMap::belowHelper(
SHAMapTreeNodePtr node,
intr_ptr::SharedPtr<SHAMapTreeNode> node,
SharedPtrNodeStack& stack,
int branch,
std::tuple<int, std::function<bool(int)>, std::function<void(int&)>> const& loopParams) const
@@ -476,7 +479,8 @@ SHAMap::belowHelper(
return nullptr;
}
SHAMapLeafNode*
SHAMap::lastBelow(SHAMapTreeNodePtr node, SharedPtrNodeStack& stack, int branch) const
SHAMap::lastBelow(intr_ptr::SharedPtr<SHAMapTreeNode> node, SharedPtrNodeStack& stack, int branch)
const
{
auto init = branchFactor - 1;
auto cmp = [](int i) { return i >= 0; };
@@ -485,7 +489,8 @@ SHAMap::lastBelow(SHAMapTreeNodePtr node, SharedPtrNodeStack& stack, int branch)
return belowHelper(node, stack, branch, {init, cmp, incr});
}
SHAMapLeafNode*
SHAMap::firstBelow(SHAMapTreeNodePtr node, SharedPtrNodeStack& stack, int branch) const
SHAMap::firstBelow(intr_ptr::SharedPtr<SHAMapTreeNode> node, SharedPtrNodeStack& stack, int branch)
const
{
auto init = 0;
auto cmp = [](int i) { return i <= branchFactor; };
@@ -694,8 +699,10 @@ SHAMap::delItem(uint256 const& id)
SHAMapNodeType const type = leaf->getType();
using TreeNodeType = intr_ptr::SharedPtr<SHAMapTreeNode>;
// What gets attached to the end of the chain (For now, nothing, since we deleted the leaf)
SHAMapTreeNodePtr prevNode;
TreeNodeType prevNode;
while (!stack.empty())
{
@@ -721,7 +728,7 @@ SHAMap::delItem(uint256 const& id)
// no children below this branch
//
// Note: This is unnecessary due to the std::move above but left here for safety
prevNode = SHAMapTreeNodePtr{};
prevNode = TreeNodeType{};
}
else if (bc == 1)
{
@@ -734,7 +741,7 @@ SHAMap::delItem(uint256 const& id)
{
if (!node->isEmptyBranch(i))
{
node->setChild(i, SHAMapTreeNodePtr{});
node->setChild(i, TreeNodeType{});
break;
}
}
@@ -930,8 +937,8 @@ SHAMap::fetchRoot(SHAMapHash const& hash, SHAMapSyncFilter* filter)
@note The node must have already been unshared by having the caller
first call SHAMapTreeNode::unshare().
*/
SHAMapTreeNodePtr
SHAMap::writeNode(NodeObjectType t, SHAMapTreeNodePtr node) const
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::writeNode(NodeObjectType t, intr_ptr::SharedPtr<SHAMapTreeNode> node) const
{
XRPL_ASSERT(node->cowid() == 0, "xrpl::SHAMap::writeNode : valid input node");
XRPL_ASSERT(backed_, "xrpl::SHAMap::writeNode : is backed");
@@ -1148,7 +1155,7 @@ SHAMap::dump(bool hash) const
JLOG(journal_.info()) << leafCount << " resident leaves";
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::cacheLookup(SHAMapHash const& hash) const
{
auto ret = f_.getTreeNodeCache()->fetch(hash.as_uint256());
@@ -1157,7 +1164,7 @@ SHAMap::cacheLookup(SHAMapHash const& hash) const
}
void
SHAMap::canonicalize(SHAMapHash const& hash, SHAMapTreeNodePtr& node) const
SHAMap::canonicalize(SHAMapHash const& hash, intr_ptr::SharedPtr<SHAMapTreeNode>& node) const
{
XRPL_ASSERT(backed_, "xrpl::SHAMap::canonicalize : is backed");
XRPL_ASSERT(node->cowid() == 0, "xrpl::SHAMap::canonicalize : valid node input");

View File

@@ -261,7 +261,7 @@ SHAMap::walkMap(std::vector<SHAMapMissingNode>& missingNodes, int maxMissing) co
{
if (!node->isEmptyBranch(i))
{
SHAMapTreeNodePtr const nextNode = descendNoStore(*node, i);
intr_ptr::SharedPtr<SHAMapTreeNode> const nextNode = descendNoStore(*node, i);
if (nextNode)
{
@@ -286,7 +286,7 @@ SHAMap::walkMapParallel(std::vector<SHAMapMissingNode>& missingNodes, int maxMis
return false;
using StackEntry = intr_ptr::SharedPtr<SHAMapInnerNode>;
std::array<SHAMapTreeNodePtr, 16> topChildren;
std::array<intr_ptr::SharedPtr<SHAMapTreeNode>, 16> topChildren;
{
auto const& innerRoot = intr_ptr::static_pointer_cast<SHAMapInnerNode>(root_);
for (int i = 0; i < 16; ++i)
@@ -331,7 +331,8 @@ SHAMap::walkMapParallel(std::vector<SHAMapMissingNode>& missingNodes, int maxMis
{
if (node->isEmptyBranch(i))
continue;
SHAMapTreeNodePtr const nextNode = descendNoStore(*node, i);
intr_ptr::SharedPtr<SHAMapTreeNode> const nextNode =
descendNoStore(*node, i);
if (nextNode)
{

View File

@@ -37,7 +37,7 @@ SHAMapInnerNode::~SHAMapInnerNode() = default;
void
SHAMapInnerNode::partialDestructor()
{
SHAMapTreeNodePtr* children = nullptr;
intr_ptr::SharedPtr<SHAMapTreeNode>* children = nullptr;
// structured bindings can't be captured in c++ 17; use tie instead
std::tie(std::ignore, std::ignore, children) = hashesAndChildren_.getHashesAndChildren();
iterNonEmptyChildIndexes([&](auto branchNum, auto indexNum) { children[indexNum].reset(); });
@@ -69,7 +69,7 @@ SHAMapInnerNode::getChildIndex(int i) const
return hashesAndChildren_.getChildIndex(isBranch_, i);
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapInnerNode::clone(std::uint32_t cowid) const
{
auto const branchCount = getBranchCount();
@@ -79,7 +79,7 @@ SHAMapInnerNode::clone(std::uint32_t cowid) const
p->isBranch_ = isBranch_;
p->fullBelowGen_ = fullBelowGen_;
SHAMapHash *cloneHashes = nullptr, *thisHashes = nullptr;
SHAMapTreeNodePtr *cloneChildren = nullptr, *thisChildren = nullptr;
intr_ptr::SharedPtr<SHAMapTreeNode>*cloneChildren = nullptr, *thisChildren = nullptr;
// structured bindings can't be captured in c++ 17; use tie instead
std::tie(std::ignore, cloneHashes, cloneChildren) =
p->hashesAndChildren_.getHashesAndChildren();
@@ -118,7 +118,7 @@ SHAMapInnerNode::clone(std::uint32_t cowid) const
return p;
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapInnerNode::makeFullInner(Slice data, SHAMapHash const& hash, bool hashValid)
{
// A full inner node is serialized as 16 256-bit hashes, back to back:
@@ -153,7 +153,7 @@ SHAMapInnerNode::makeFullInner(Slice data, SHAMapHash const& hash, bool hashVali
return ret;
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapInnerNode::makeCompressedInner(Slice data)
{
// A compressed inner node is serialized as a series of 33 byte chunks,
@@ -207,7 +207,7 @@ void
SHAMapInnerNode::updateHashDeep()
{
SHAMapHash* hashes = nullptr;
SHAMapTreeNodePtr* children = nullptr;
intr_ptr::SharedPtr<SHAMapTreeNode>* children = nullptr;
// structured bindings can't be captured in c++ 17; use tie instead
std::tie(std::ignore, hashes, children) = hashesAndChildren_.getHashesAndChildren();
iterNonEmptyChildIndexes([&](auto branchNum, auto indexNum) {
@@ -265,7 +265,7 @@ SHAMapInnerNode::getString(SHAMapNodeID const& id) const
// We are modifying an inner node
void
SHAMapInnerNode::setChild(int m, SHAMapTreeNodePtr child)
SHAMapInnerNode::setChild(int m, intr_ptr::SharedPtr<SHAMapTreeNode> child)
{
XRPL_ASSERT(
(m >= 0) && (m < branchFactor), "xrpl::SHAMapInnerNode::setChild : valid branch input");
@@ -307,7 +307,7 @@ SHAMapInnerNode::setChild(int m, SHAMapTreeNodePtr child)
// finished modifying, now make shareable
void
SHAMapInnerNode::shareChild(int m, SHAMapTreeNodePtr const& child)
SHAMapInnerNode::shareChild(int m, intr_ptr::SharedPtr<SHAMapTreeNode> const& child)
{
XRPL_ASSERT(
(m >= 0) && (m < branchFactor), "xrpl::SHAMapInnerNode::shareChild : valid branch input");
@@ -337,7 +337,7 @@ SHAMapInnerNode::getChildPointer(int branch)
return hashesAndChildren_.getChildren()[index].get();
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapInnerNode::getChild(int branch)
{
XRPL_ASSERT(
@@ -364,8 +364,8 @@ SHAMapInnerNode::getChildHash(int m) const
return zeroSHAMapHash;
}
SHAMapTreeNodePtr
SHAMapInnerNode::canonicalizeChild(int branch, SHAMapTreeNodePtr node)
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapInnerNode::canonicalizeChild(int branch, intr_ptr::SharedPtr<SHAMapTreeNode> node)
{
XRPL_ASSERT(
branch >= 0 && branch < branchFactor,

View File

@@ -129,9 +129,7 @@ selectBranch(SHAMapNodeID const& id, uint256 const& hash)
SHAMapNodeID
SHAMapNodeID::createID(int depth, uint256 const& key)
{
XRPL_ASSERT(
depth >= 0 && depth <= SHAMap::leafDepth,
"xrpl::SHAMapNodeID::createID : valid branch input");
XRPL_ASSERT((depth >= 0) && (depth < 65), "xrpl::SHAMapNodeID::createID : valid branch input");
return SHAMapNodeID(depth, key & depthMask(depth));
}

View File

@@ -66,7 +66,7 @@ SHAMap::visitNodes(std::function<bool(SHAMapTreeNode&)> const& function) const
{
if (!node->isEmptyBranch(pos))
{
SHAMapTreeNodePtr const child = descendNoStore(*node, pos);
intr_ptr::SharedPtr<SHAMapTreeNode> const child = descendNoStore(*node, pos);
if (!function(*child))
return;
@@ -204,7 +204,8 @@ SHAMap::gmn_ProcessNodes(MissingNodes& mn, MissingNodes::StackEntry& se)
branch,
mn.filter_,
pending,
[node, nodeID, branch, &mn](SHAMapTreeNodePtr found, SHAMapHash const&) {
[node, nodeID, branch, &mn](
intr_ptr::SharedPtr<SHAMapTreeNode> found, SHAMapHash const&) {
// a read completed asynchronously
std::unique_lock<std::mutex> const lock{mn.deferLock_};
mn.finishedReads_.emplace_back(node, nodeID, branch, std::move(found));
@@ -267,7 +268,8 @@ SHAMap::gmn_ProcessDeferredReads(MissingNodes& mn)
int complete = 0;
while (complete != mn.deferred_)
{
std::tuple<SHAMapInnerNode*, SHAMapNodeID, int, SHAMapTreeNodePtr> deferredNode;
std::tuple<SHAMapInnerNode*, SHAMapNodeID, int, intr_ptr::SharedPtr<SHAMapTreeNode>>
deferredNode;
{
std::unique_lock<std::mutex> lock{mn.deferLock_};
@@ -415,7 +417,7 @@ SHAMap::getMissingNodes(int max, SHAMapSyncFilter* filter)
bool
SHAMap::getNodeFat(
SHAMapNodeID const& wanted,
std::vector<SHAMapNodeData>& data,
std::vector<std::pair<SHAMapNodeID, Blob>>& data,
bool fatLeaves,
std::uint32_t depth) const
{
@@ -461,7 +463,7 @@ SHAMap::getNodeFat(
// Add this node to the reply
s.erase();
node->serializeForWire(s);
data.emplace_back(nodeID, s.getData(), node->isLeaf());
data.emplace_back(nodeID, s.getData());
if (node->isInner())
{
@@ -491,7 +493,7 @@ SHAMap::getNodeFat(
// Just include this node
s.erase();
childNode->serializeForWire(s);
data.emplace_back(childID, s.getData(), childNode->isLeaf());
data.emplace_back(childID, s.getData());
}
}
}
@@ -509,18 +511,8 @@ SHAMap::serializeRoot(Serializer& s) const
}
SHAMapAddNode
SHAMap::addRootNode(
SHAMapHash const& hash,
SHAMapTreeNodePtr rootNode,
SHAMapSyncFilter const* filter)
SHAMap::addRootNode(SHAMapHash const& hash, Slice const& rootNode, SHAMapSyncFilter* filter)
{
XRPL_ASSERT(rootNode, "xrpl::SHAMap::addRootNode : non-null root node");
if (!rootNode)
{
JLOG(journal_.error()) << "Null node received";
return SHAMapAddNode::invalid();
}
// we already have a root_ node
if (root_->getHash().isNonZero())
{
@@ -530,16 +522,14 @@ SHAMap::addRootNode(
}
XRPL_ASSERT(cowid_ >= 1, "xrpl::SHAMap::addRootNode : valid cowid");
if (rootNode->getHash() != hash)
{
JLOG(journal_.warn()) << "Corrupt node received";
auto node = SHAMapTreeNode::makeFromWire(rootNode);
if (!node || node->getHash() != hash)
return SHAMapAddNode::invalid();
}
if (backed_)
canonicalize(hash, rootNode);
canonicalize(hash, node);
root_ = std::move(rootNode);
root_ = node;
if (root_->isLeaf())
clearSynching();
@@ -556,23 +546,9 @@ SHAMap::addRootNode(
}
SHAMapAddNode
SHAMap::addKnownNode(
SHAMapNodeID const& nodeID,
SHAMapTreeNodePtr treeNode,
SHAMapSyncFilter const* filter)
SHAMap::addKnownNode(SHAMapNodeID const& node, Slice const& rawNode, SHAMapSyncFilter* filter)
{
XRPL_ASSERT(!nodeID.isRoot(), "xrpl::SHAMap::addKnownNode : valid node input");
if (nodeID.isRoot())
{
JLOG(journal_.error()) << "Root node received";
return SHAMapAddNode::invalid();
}
XRPL_ASSERT(treeNode, "xrpl::SHAMap::addKnownNode : non-null tree node");
if (!treeNode)
{
JLOG(journal_.error()) << "Null node received";
return SHAMapAddNode::invalid();
}
XRPL_ASSERT(!node.isRoot(), "xrpl::SHAMap::addKnownNode : valid node input");
if (!isSynching())
{
@@ -586,14 +562,14 @@ SHAMap::addKnownNode(
while (currNode->isInner() &&
!safe_downcast<SHAMapInnerNode*>(currNode)->isFullBelow(generation) &&
(currNodeID.getDepth() < nodeID.getDepth()))
(currNodeID.getDepth() < node.getDepth()))
{
int const branch = selectBranch(currNodeID, nodeID.getNodeID());
int const branch = selectBranch(currNodeID, node.getNodeID());
XRPL_ASSERT(branch >= 0, "xrpl::SHAMap::addKnownNode : valid branch");
auto inner = safe_downcast<SHAMapInnerNode*>(currNode);
if (inner->isEmptyBranch(branch))
{
JLOG(journal_.warn()) << "Add known node for empty branch" << nodeID;
JLOG(journal_.warn()) << "Add known node for empty branch" << node;
return SHAMapAddNode::invalid();
}
@@ -609,44 +585,67 @@ SHAMap::addKnownNode(
if (currNode != nullptr)
continue;
if (childHash != treeNode->getHash())
auto newNode = SHAMapTreeNode::makeFromWire(rawNode);
if (!newNode || childHash != newNode->getHash())
{
JLOG(journal_.warn()) << "Corrupt node received";
return SHAMapAddNode::invalid();
}
// In rare cases, a node can still be corrupt even after hash
// validation. For leaf nodes, we perform an additional check to
// ensure the node's position in the tree is consistent with its
// content to prevent inconsistencies that could
// propagate further down the line.
if (newNode->isLeaf())
{
auto const& actualKey =
safe_downcast<SHAMapLeafNode const*>(newNode.get())->peekItem()->key();
// Validate that this leaf belongs at the target position
auto const expectedNodeID = SHAMapNodeID::createID(node.getDepth(), actualKey);
if (expectedNodeID.getNodeID() != node.getNodeID())
{
JLOG(journal_.debug())
<< "Leaf node position mismatch: "
<< "expected=" << expectedNodeID.getNodeID() << ", actual=" << node.getNodeID();
return SHAMapAddNode::invalid();
}
}
// Inner nodes must be at a level strictly less than 64
// but leaf nodes (while notionally at level 64) can be
// at any depth up to and including 64:
if ((currNodeID.getDepth() > leafDepth) ||
(treeNode->isInner() && currNodeID.getDepth() == leafDepth))
(newNode->isInner() && currNodeID.getDepth() == leafDepth))
{
// Map is provably invalid
state_ = SHAMapState::Invalid;
return SHAMapAddNode::useful();
}
if (currNodeID != nodeID)
if (currNodeID != node)
{
// Either this node is broken or we didn't request it (yet)
JLOG(journal_.warn()) << "unable to hook node " << nodeID;
JLOG(journal_.warn()) << "unable to hook node " << node;
JLOG(journal_.info()) << " stuck at " << currNodeID;
JLOG(journal_.info()) << "got depth=" << nodeID.getDepth()
JLOG(journal_.info()) << "got depth=" << node.getDepth()
<< ", walked to= " << currNodeID.getDepth();
return SHAMapAddNode::useful();
}
if (backed_)
canonicalize(childHash, treeNode);
canonicalize(childHash, newNode);
treeNode = prevNode->canonicalizeChild(branch, std::move(treeNode));
newNode = prevNode->canonicalizeChild(branch, std::move(newNode));
if (filter != nullptr)
{
Serializer s;
treeNode->serializeWithPrefix(s);
newNode->serializeWithPrefix(s);
filter->gotNode(
false, childHash, ledgerSeq_, std::move(s.modData()), treeNode->getType());
false, childHash, ledgerSeq_, std::move(s.modData()), newNode->getType());
}
return SHAMapAddNode::useful();

View File

@@ -25,7 +25,7 @@
namespace xrpl {
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapTreeNode::makeTransaction(Slice data, SHAMapHash const& hash, bool hashValid)
{
auto item = make_shamapitem(sha512Half(HashPrefix::transactionID, data), data);
@@ -36,7 +36,7 @@ SHAMapTreeNode::makeTransaction(Slice data, SHAMapHash const& hash, bool hashVal
return intr_ptr::make_shared<SHAMapTxLeafNode>(std::move(item), 0);
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapTreeNode::makeTransactionWithMeta(Slice data, SHAMapHash const& hash, bool hashValid)
{
Serializer s(data.data(), data.size());
@@ -60,7 +60,7 @@ SHAMapTreeNode::makeTransactionWithMeta(Slice data, SHAMapHash const& hash, bool
return intr_ptr::make_shared<SHAMapTxPlusMetaLeafNode>(std::move(item), 0);
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapTreeNode::makeAccountState(Slice data, SHAMapHash const& hash, bool hashValid)
{
Serializer s(data.data(), data.size());
@@ -87,7 +87,7 @@ SHAMapTreeNode::makeAccountState(Slice data, SHAMapHash const& hash, bool hashVa
return intr_ptr::make_shared<SHAMapAccountStateLeafNode>(std::move(item), 0);
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapTreeNode::makeFromWire(Slice rawNode)
{
if (rawNode.empty())
@@ -118,7 +118,7 @@ SHAMapTreeNode::makeFromWire(Slice rawNode)
Throw<std::runtime_error>("wire: Unknown type (" + std::to_string(type) + ")");
}
SHAMapTreeNodePtr
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMapTreeNode::makeFromPrefix(Slice rawNode, SHAMapHash const& hash)
{
if (rawNode.size() < 4)

View File

@@ -1,350 +0,0 @@
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpl/basics/IntrusivePointer.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/beast/unit_test/suite.h>
#include <xrpl/protocol/Serializer.h>
#include <xrpl/shamap/SHAMap.h>
#include <xrpl/shamap/SHAMapAccountStateLeafNode.h>
#include <xrpl/shamap/SHAMapInnerNode.h>
#include <xrpl/shamap/SHAMapItem.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <boost/smart_ptr/intrusive_ptr.hpp>
#include <xrpl.pb.h>
#include <bit>
#include <cstdint>
#include <string>
namespace xrpl::tests {
class LedgerNodeHelpers_test : public beast::unit_test::suite
{
static boost::intrusive_ptr<SHAMapItem>
makeTestItem(std::uint32_t seed)
{
Serializer s;
s.add32(seed);
s.add32(seed + 1);
s.add32(seed + 2);
return make_shamapitem(s.getSHA512Half(), s.slice());
}
static std::string
serializeNode(SHAMapTreeNodePtr const& node)
{
Serializer s;
node->serializeForWire(s);
auto const slice = s.slice();
return std::string(std::bit_cast<char const*>(slice.data()), slice.size());
}
void
testValidateLedgerNode()
{
// In the tests below the validity of the content of the node data and ID fields is not
// checked - only that the fields have values when expected. The content of the fields is
// verified in the other tests in this file.
testcase("validateLedgerNode");
// Invalid: missing all fields.
{
protocol::TMLedgerNode const node;
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: missing `nodedata` field.
{
protocol::TMLedgerNode node;
node.set_nodeid("test_nodeid");
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: missing `nodedata` field.
{
protocol::TMLedgerNode node;
node.set_id("test_nodeid");
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: missing `nodedata` field.
{
protocol::TMLedgerNode node;
node.set_depth(1);
BEAST_EXPECT(!validateLedgerNode(node));
}
// Valid: legacy `nodeid` field.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_nodeid("test_nodeid");
BEAST_EXPECT(validateLedgerNode(node));
}
// Invalid: has both legacy `nodeid` and new `id` fields.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_nodeid("test_nodeid");
node.set_id("test_nodeid");
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: has both legacy `nodeid` and new `depth` fields.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_nodeid("test_nodeid");
node.set_depth(5);
BEAST_EXPECT(!validateLedgerNode(node));
}
// Valid: new `id` field.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_id("test_id");
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: new `depth` field.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(5);
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: `depth` at minimum depth.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(0);
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: `depth` at arbitrary depth between minimum and maximum.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(10);
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: `depth` at maximum depth.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(SHAMap::leafDepth);
BEAST_EXPECT(validateLedgerNode(node));
}
// Invalid: `depth` is greater than maximum depth.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(SHAMap::leafDepth + 1);
BEAST_EXPECT(!validateLedgerNode(node));
}
}
void
testGetTreeNode()
{
testcase("getTreeNode");
// Valid: inner node. It must have at least one child for `serializeNode` to work.
{
auto const innerNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
auto const childNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
innerNode->setChild(0, childNode);
auto const innerData = serializeNode(innerNode);
auto const result = getTreeNode(innerData);
BEAST_EXPECT(result && result->isInner());
}
// Valid: leaf node.
{
auto const leafItem = makeTestItem(12345);
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
auto const leafData = serializeNode(leafNode);
auto const result = getTreeNode(leafData);
BEAST_EXPECT(result && result->isLeaf());
}
// Invalid: empty data.
{
auto const result = getTreeNode("");
BEAST_EXPECT(!result);
}
// Invalid: garbage data.
{
auto const result = getTreeNode("invalid");
BEAST_EXPECT(!result);
}
// Invalid: truncated data.
{
auto const leafItem = makeTestItem(54321);
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
// Truncate the data to trigger an exception in SHAMapTreeNode::makeAccountState when
// the data is used to deserialize the node.
uint256 const tag;
auto const leafData = serializeNode(leafNode).substr(0, tag.bytes - 1);
auto const result = getTreeNode(leafData);
BEAST_EXPECT(!result);
}
}
void
testGetSHAMapNodeID()
{
testcase("getSHAMapNodeID");
{
// Tests using inner nodes at various depths.
auto const innerNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
auto const childNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
innerNode->setChild(0, childNode);
auto const innerData = serializeNode(innerNode);
// Valid: legacy `nodeid` field at arbitrary depth.
{
auto const innerDepth = 3;
auto const innerID = SHAMapNodeID::createID(innerDepth, uint256{});
protocol::TMLedgerNode node;
node.set_nodedata(innerData);
node.set_nodeid(innerID.getRawString());
auto const result = getSHAMapNodeID(node, innerNode);
BEAST_EXPECT(result == innerID);
}
// Valid: new `id` field at minimum depth.
{
auto const innerDepth = 0;
auto const innerID = SHAMapNodeID::createID(innerDepth, uint256{});
protocol::TMLedgerNode node;
node.set_nodedata(innerData);
node.set_id(innerID.getRawString());
auto const result = getSHAMapNodeID(node, innerNode);
BEAST_EXPECT(result == innerID);
}
// Invalid: new `depth` field should not be used for inner nodes.
{
protocol::TMLedgerNode node;
node.set_nodedata(innerData);
node.set_depth(10);
auto const result = getSHAMapNodeID(node, innerNode);
BEAST_EXPECT(!result);
}
}
{
// Tests using leaf nodes at various depths.
auto const leafItem = makeTestItem(12345);
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
auto const leafData = serializeNode(leafNode);
auto const leafKey = leafItem->key();
// Valid: legacy `nodeid` field at arbitrary depth.
{
auto const leafDepth = 5;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(leafData);
ledgerNode.set_nodeid(leafID.getRawString());
auto const result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(result == leafID);
}
// Invalid: new `id` field should not be used for leaf nodes.
{
auto const leafDepth = 5;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(leafData);
ledgerNode.set_id(leafID.getRawString());
auto const result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(!result);
}
// Valid: new `depth` field at minimum depth.
{
auto const leafDepth = 0;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode node;
node.set_nodedata(leafData);
node.set_depth(leafDepth);
auto const result = getSHAMapNodeID(node, leafNode);
BEAST_EXPECT(result == leafID);
}
// Valid: new `depth` field at arbitrary depth between minimum and maximum.
{
auto const leafDepth = 10;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(leafData);
ledgerNode.set_depth(leafDepth);
auto const result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(result == leafID);
}
// Valid: new `depth` field at maximum depth.
// Note that we do not test a depth greater than the maximum depth, because the proto
// message is assumed to have been validated by the time the getSHAMapNodeID function is
// called.
{
auto const leafDepth = SHAMap::leafDepth;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode node;
node.set_nodedata(leafData);
node.set_depth(leafDepth);
auto const result = getSHAMapNodeID(node, leafNode);
BEAST_EXPECT(result == leafID);
}
// Invalid: legacy `nodeid` field where the node ID is inconsistent with the key.
{
auto const otherItem = makeTestItem(54321);
auto const otherNode =
intr_ptr::make_shared<SHAMapAccountStateLeafNode>(otherItem, 1);
auto const otherData = serializeNode(otherNode);
auto const otherKey = otherItem->key();
auto const otherDepth = 1;
auto const otherID = SHAMapNodeID::createID(otherDepth, otherKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(otherData);
ledgerNode.set_nodeid(otherID.getRawString());
auto const result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(!result);
}
}
}
public:
void
run() override
{
testValidateLedgerNode();
testGetTreeNode();
testGetSHAMapNodeID();
}
};
BEAST_DEFINE_TESTSUITE(LedgerNodeHelpers, app, xrpl);
} // namespace xrpl::tests

View File

@@ -7,6 +7,7 @@
#include <xrpl/basics/BasicConfig.h>
#include <xrpl/beast/unit_test/suite.h>
#include <xrpl/beast/utility/temp_dir.h>
#include <xrpl/protocol/SystemParameters.h> // IWYU pragma: keep
#include <xrpl/server/Port.h>
#include <boost/filesystem/operations.hpp>

View File

@@ -63,8 +63,8 @@ public:
negotiateProtocolVersion("RTXP/1.2, XRPL/2.0, XRPL/2.1") == make_protocol(2, 1));
BEAST_EXPECT(negotiateProtocolVersion("XRPL/2.2") == make_protocol(2, 2));
BEAST_EXPECT(
negotiateProtocolVersion("RTXP/1.2, XRPL/2.3, XRPL/2.4, XRPL/999.999") ==
make_protocol(2, 3));
negotiateProtocolVersion("RTXP/1.2, XRPL/2.2, XRPL/2.3, XRPL/999.999") ==
make_protocol(2, 2));
BEAST_EXPECT(negotiateProtocolVersion("XRPL/999.999, WebSocket/1.0") == std::nullopt);
BEAST_EXPECT(negotiateProtocolVersion("") == std::nullopt);
}

View File

@@ -1,6 +1,7 @@
#include <test/shamap/common.h>
#include <test/unit_test/SuiteJournal.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/SHAMapHash.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/base_uint.h>
@@ -114,17 +115,14 @@ public:
destination.setSynching();
{
std::vector<SHAMapNodeData> a;
std::vector<std::pair<SHAMapNodeID, Blob>> a;
BEAST_EXPECT(source.getNodeFat(SHAMapNodeID(), a, rand_bool(eng_), rand_int(eng_, 2)));
unexpected(a.empty(), "NodeSize");
auto node = SHAMapTreeNode::makeFromWire(makeSlice(a[0].data));
if (!node)
fail("", __FILE__, __LINE__);
BEAST_EXPECT(
destination.addRootNode(source.getHash(), std::move(node), nullptr).isGood());
BEAST_EXPECT(destination.addRootNode(source.getHash(), makeSlice(a[0].second), nullptr)
.isGood());
}
do
@@ -138,7 +136,7 @@ public:
break;
// get as many nodes as possible based on this information
std::vector<SHAMapNodeData> b;
std::vector<std::pair<SHAMapNodeID, Blob>> b;
for (auto& it : nodesMissing)
{
@@ -160,10 +158,8 @@ public:
// Don't use BEAST_EXPECT here b/c it will be called a
// non-deterministic number of times and the number of tests run
// should be deterministic
auto node = SHAMapTreeNode::makeFromWire(makeSlice(b[i].data));
if (!node)
fail("", __FILE__, __LINE__);
if (!destination.addKnownNode(b[i].nodeID, std::move(node), nullptr).isUseful())
if (!destination.addKnownNode(b[i].first, makeSlice(b[i].second), nullptr)
.isUseful())
fail("", __FILE__, __LINE__);
}
} while (true);

View File

@@ -9,7 +9,6 @@
#include <mutex>
#include <set>
#include <string_view>
#include <utility>
namespace xrpl {
@@ -132,16 +131,16 @@ private:
processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& data);
bool
takeHeader(std::string_view data);
takeHeader(std::string const& data);
void
receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san);
receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode&);
bool
takeTxRootNode(std::string_view data, SHAMapAddNode& san);
takeTxRootNode(Slice const& data, SHAMapAddNode&);
bool
takeAsRootNode(std::string_view data, SHAMapAddNode& san);
takeAsRootNode(Slice const& data, SHAMapAddNode&);
std::vector<uint256>
neededTxHashes(int max, SHAMapSyncFilter* filter) const;

View File

@@ -4,7 +4,6 @@
#include <xrpld/app/ledger/InboundLedgers.h>
#include <xrpld/app/ledger/LedgerMaster.h>
#include <xrpld/app/ledger/TransactionStateSF.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpld/app/ledger/detail/TimeoutCounter.h>
#include <xrpld/app/main/Application.h>
#include <xrpld/overlay/Message.h>
@@ -43,8 +42,8 @@
#include <mutex>
#include <random>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <tuple>
#include <unordered_map>
#include <utility>
@@ -794,7 +793,7 @@ InboundLedger::filterNodes(
*/
// data must not have hash prefix
bool
InboundLedger::takeHeader(std::string_view data)
InboundLedger::takeHeader(std::string const& data)
{
// Return value: true=normal, false=bad data
JLOG(journal_.trace()) << "got header acquiring ledger " << hash_;
@@ -882,31 +881,20 @@ InboundLedger::receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san)
{
auto const f = filter.get();
for (auto const& ledgerNode : packet.nodes())
for (auto const& node : packet.nodes())
{
auto treeNode = getTreeNode(ledgerNode.nodedata());
if (!treeNode)
{
JLOG(journal_.warn()) << "Got invalid node data";
san.incInvalid();
return;
}
auto const nodeID = deserializeSHAMapNodeID(node.nodeid());
auto const nodeID = getSHAMapNodeID(ledgerNode, treeNode);
if (!nodeID)
{
JLOG(journal_.warn()) << "Got invalid node id";
san.incInvalid();
return;
}
throw std::runtime_error("data does not properly deserialize");
if (nodeID->isRoot())
{
san += map.addRootNode(rootHash, std::move(treeNode), f);
san += map.addRootNode(rootHash, makeSlice(node.nodedata()), f);
}
else
{
san += map.addKnownNode(*nodeID, std::move(treeNode), f);
san += map.addKnownNode(*nodeID, makeSlice(node.nodedata()), f);
}
if (!san.isGood())
@@ -946,7 +934,7 @@ InboundLedger::receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san)
Call with a lock
*/
bool
InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
InboundLedger::takeAsRootNode(Slice const& data, SHAMapAddNode& san)
{
if (failed_ || mHaveState)
{
@@ -962,17 +950,9 @@ InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
// LCOV_EXCL_STOP
}
auto treeNode = getTreeNode(data);
if (!treeNode)
{
JLOG(journal_.warn()) << "Got invalid node data";
san.incInvalid();
return false;
}
AccountStateSF filter(mLedger->stateMap().family().db(), app_.getLedgerMaster());
san += mLedger->stateMap().addRootNode(
SHAMapHash{mLedger->header().accountHash}, std::move(treeNode), &filter);
san +=
mLedger->stateMap().addRootNode(SHAMapHash{mLedger->header().accountHash}, data, &filter);
return san.isGood();
}
@@ -980,7 +960,7 @@ InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
Call with a lock
*/
bool
InboundLedger::takeTxRootNode(std::string_view data, SHAMapAddNode& san)
InboundLedger::takeTxRootNode(Slice const& data, SHAMapAddNode& san)
{
if (failed_ || mHaveTransactions)
{
@@ -996,17 +976,8 @@ InboundLedger::takeTxRootNode(std::string_view data, SHAMapAddNode& san)
// LCOV_EXCL_STOP
}
auto treeNode = getTreeNode(data);
if (!treeNode)
{
JLOG(journal_.warn()) << "Got invalid node data";
san.incInvalid();
return false;
}
TransactionStateSF filter(mLedger->txMap().family().db(), app_.getLedgerMaster());
san += mLedger->txMap().addRootNode(
SHAMapHash{mLedger->header().txHash}, std::move(treeNode), &filter);
san += mLedger->txMap().addRootNode(SHAMapHash{mLedger->header().txHash}, data, &filter);
return san.isGood();
}
@@ -1103,13 +1074,13 @@ InboundLedger::processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& p
}
if (!mHaveState && (packet.nodes().size() > 1) &&
!takeAsRootNode(packet.nodes(1).nodedata(), san))
!takeAsRootNode(makeSlice(packet.nodes(1).nodedata()), san))
{
JLOG(journal_.warn()) << "Included AS root invalid";
}
if (!mHaveTransactions && (packet.nodes().size() > 2) &&
!takeTxRootNode(packet.nodes(2).nodedata(), san))
!takeTxRootNode(makeSlice(packet.nodes(2).nodedata()), san))
{
JLOG(journal_.warn()) << "Included TX root invalid";
}
@@ -1140,13 +1111,13 @@ InboundLedger::processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& p
ScopedLockType const sl(mtx_);
// Verify nodes are complete
for (auto const& ledgerNode : packet.nodes())
// Verify node IDs and data are complete
for (auto const& node : packet.nodes())
{
if (!validateLedgerNode(ledgerNode))
if (!node.has_nodeid() || !node.has_nodedata())
{
JLOG(journal_.warn()) << "Got malformed ledger node";
peer->charge(Resource::feeMalformedRequest, "ledgerNode");
JLOG(journal_.warn()) << "Got bad node";
peer->charge(Resource::feeMalformedRequest, "ledger_data bad node");
return -1;
}
}

View File

@@ -2,13 +2,13 @@
#include <xrpld/app/ledger/InboundLedger.h>
#include <xrpld/app/ledger/LedgerMaster.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpld/app/main/Application.h>
#include <xrpld/overlay/PeerSet.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/DecayingSample.h>
#include <xrpl/basics/Log.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/UnorderedContainers.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/scope.h>
@@ -248,20 +248,23 @@ public:
Serializer s;
try
{
for (auto const& ledgerNode : packet_ptr->nodes())
for (int i = 0; i < packet_ptr->nodes().size(); ++i)
{
if (!validateLedgerNode(ledgerNode))
auto const& node = packet_ptr->nodes(i);
if (!node.has_nodeid() || !node.has_nodedata())
return;
auto const treeNode = getTreeNode(ledgerNode.nodedata());
if (!treeNode)
auto newNode = SHAMapTreeNode::makeFromWire(makeSlice(node.nodedata()));
if (!newNode)
return;
s.erase();
treeNode->serializeWithPrefix(s);
newNode->serializeWithPrefix(s);
app_.getLedgerMaster().addFetchPack(
treeNode->getHash().as_uint256(), std::make_shared<Blob>(s.begin(), s.end()));
newNode->getHash().as_uint256(), std::make_shared<Blob>(s.begin(), s.end()));
}
}
catch (std::exception const&) // NOLINT(bugprone-empty-catch)

View File

@@ -1,11 +1,11 @@
#include <xrpld/app/ledger/InboundTransactions.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpld/app/ledger/detail/TransactionAcquire.h>
#include <xrpld/app/main/Application.h>
#include <xrpld/overlay/PeerSet.h>
#include <xrpl/basics/Log.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/UnorderedContainers.h>
#include <xrpl/beast/insight/Collector.h>
#include <xrpl/protocol/RippleLedgerHash.h>
@@ -14,7 +14,6 @@
#include <xrpl/shamap/SHAMap.h>
#include <xrpl/shamap/SHAMapMissingNode.h>
#include <xrpl/shamap/SHAMapNodeID.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <xrpl.pb.h>
@@ -145,38 +144,29 @@ public:
return;
}
std::vector<std::pair<SHAMapNodeID, SHAMapTreeNodePtr>> data;
std::vector<std::pair<SHAMapNodeID, Slice>> data;
data.reserve(packet.nodes().size());
for (auto const& ledgerNode : packet.nodes())
for (auto const& node : packet.nodes())
{
if (!validateLedgerNode(ledgerNode))
if (!node.has_nodeid() || !node.has_nodedata())
{
JLOG(j_.warn()) << "Got malformed ledger node";
peer->charge(Resource::feeMalformedRequest, "ledgerNode");
peer->charge(Resource::feeMalformedRequest, "ledger_data");
return;
}
auto treeNode = getTreeNode(ledgerNode.nodedata());
if (!treeNode)
auto const id = deserializeSHAMapNodeID(node.nodeid());
if (!id)
{
JLOG(j_.warn()) << "Got invalid node data";
peer->charge(Resource::feeInvalidData, "node_data");
peer->charge(Resource::feeInvalidData, "ledger_data");
return;
}
auto const nodeID = getSHAMapNodeID(ledgerNode, treeNode);
if (!nodeID)
{
JLOG(j_.warn()) << "Got invalid node id";
peer->charge(Resource::feeInvalidData, "node_id");
return;
}
data.emplace_back(*nodeID, std::move(treeNode));
data.emplace_back(*id, makeSlice(node.nodedata()));
}
if (!ta->takeNodes(std::move(data), peer).isUseful())
if (!ta->takeNodes(data, peer).isUseful())
peer->charge(Resource::feeUselessData, "ledger_data not useful");
}

View File

@@ -1,91 +0,0 @@
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/safe_cast.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/shamap/SHAMap.h>
#include <xrpl/shamap/SHAMapLeafNode.h>
#include <xrpl/shamap/SHAMapNodeID.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <xrpl.pb.h>
#include <exception>
#include <optional>
#include <string_view>
namespace xrpl {
bool
validateLedgerNode(protocol::TMLedgerNode const& ledgerNode)
{
if (!ledgerNode.has_nodedata())
return false;
if (ledgerNode.has_nodeid())
return !ledgerNode.has_id() && !ledgerNode.has_depth();
return ledgerNode.has_id() ||
(ledgerNode.has_depth() && ledgerNode.depth() <= SHAMap::leafDepth);
}
SHAMapTreeNodePtr
getTreeNode(std::string_view data)
{
auto const slice = makeSlice(data);
try
{
return SHAMapTreeNode::makeFromWire(slice);
}
catch (std::exception const&)
{
return {};
}
}
std::optional<SHAMapNodeID>
getSHAMapNodeID(protocol::TMLedgerNode const& ledgerNode, SHAMapTreeNodePtr const& treeNode)
{
if (ledgerNode.has_id() || ledgerNode.has_depth())
{
if (treeNode->isInner())
{
if (!ledgerNode.has_id())
return std::nullopt;
return deserializeSHAMapNodeID(ledgerNode.id());
}
if (treeNode->isLeaf())
{
if (!ledgerNode.has_depth())
return std::nullopt;
auto const key =
safe_downcast<SHAMapLeafNode const*>(treeNode.get())->peekItem()->key();
return SHAMapNodeID::createID(ledgerNode.depth(), key);
}
UNREACHABLE("xrpl::getSHAMapNodeID : tree node is neither inner nor leaf");
return std::nullopt;
}
if (!ledgerNode.has_nodeid())
return std::nullopt;
auto nodeID = deserializeSHAMapNodeID(ledgerNode.nodeid());
if (!nodeID.has_value())
return std::nullopt;
if (treeNode->isLeaf())
{
auto const key = safe_downcast<SHAMapLeafNode const*>(treeNode.get())->peekItem()->key();
auto const expected_id = SHAMapNodeID::createID(static_cast<int>(nodeID->getDepth()), key);
if (nodeID->getNodeID() != expected_id.getNodeID())
return std::nullopt;
}
return nodeID;
}
} // namespace xrpl

View File

@@ -1,72 +0,0 @@
#pragma once
#include <xrpl/basics/IntrusivePointer.h>
#include <xrpl/shamap/SHAMapNodeID.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <optional>
#include <string_view>
namespace protocol {
class TMLedgerNode;
} // namespace protocol
namespace xrpl {
/**
* @brief Validates a ledger node proto message.
*
* This function checks whether a ledger node has the expected fields (for non-ledger base data):
* - The node must have `nodedata`.
* - If the legacy `nodeid` field is present then the new `id` and `depth` fields must not be
* present.
* - If the new `id` or `depth` fields are present (it is a oneof field, so only one of the two can
* be set) then the legacy `nodeid` must not be present.
* - If the `depth` field is present then it must be between 0 and SHAMap::leafDepth (inclusive).
*
* @param ledgerNode The ledger node to validate.
* @return true if the ledger node has the expected fields, false otherwise.
*/
[[nodiscard]] bool
validateLedgerNode(protocol::TMLedgerNode const& ledgerNode);
/**
* @brief Deserializes a SHAMapTreeNode from wire format data.
*
* This function attempts to create a SHAMapTreeNode from the provided data string. If the data is
* malformed or deserialization fails, the function returns a nullptr instead of throwing an
* exception.
*
* @param data The serialized node data in wire format.
* @return The deserialized tree node if successful, or a nullptr if deserialization fails.
*/
[[nodiscard]] SHAMapTreeNodePtr
getTreeNode(std::string_view data);
/**
* @brief Extracts or reconstructs the SHAMapNodeID from a ledger node proto message.
*
* This function retrieves the SHAMapNodeID for a tree node, with behavior that depends on which
* field is set and the node type (inner vs. leaf).
*
* When the legacy `nodeid` field is set in the message:
* - For all nodes: Deserializes the node ID from the field.
* - For leaf nodes: Validates that the node ID is consistent with the leaf's key.
*
* When the new `id` or `depth` field is set in the message:
* - For inner nodes: Deserializes the node ID from the `id` field.
* - For leaf nodes: Reconstructs the node ID using both the depth from the `depth` field and the
* key from the leaf node's item.
* Note that root nodes may be inner nodes or leaf nodes.
*
* @param ledgerNode The validated protocol message containing the ledger node data.
* @param treeNode The deserialized tree node (inner or leaf node).
* @return An optional containing the node ID if extraction/reconstruction succeeds, or std::nullopt
* if the required fields are missing or validation fails.
* @note This function expects that the caller has already validated the ledger node by calling the
* `validateLedgerNode` function and obtained a valid tree node by calling `getTreeNode`.
*/
[[nodiscard]] std::optional<SHAMapNodeID>
getSHAMapNodeID(protocol::TMLedgerNode const& ledgerNode, SHAMapTreeNodePtr const& treeNode);
} // namespace xrpl

View File

@@ -7,13 +7,13 @@
#include <xrpld/overlay/PeerSet.h>
#include <xrpl/basics/Log.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/core/Job.h>
#include <xrpl/server/NetworkOPs.h>
#include <xrpl/shamap/SHAMap.h>
#include <xrpl/shamap/SHAMapAddNode.h>
#include <xrpl/shamap/SHAMapMissingNode.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <xrpl.pb.h>
@@ -173,7 +173,7 @@ TransactionAcquire::trigger(std::shared_ptr<Peer> const& peer)
SHAMapAddNode
TransactionAcquire::takeNodes(
std::vector<std::pair<SHAMapNodeID, SHAMapTreeNodePtr>> data,
std::vector<std::pair<SHAMapNodeID, Slice>> const& data,
std::shared_ptr<Peer> const& peer)
{
ScopedLockType const sl(mtx_);

View File

@@ -21,8 +21,8 @@ public:
SHAMapAddNode
takeNodes(
std::vector<std::pair<SHAMapNodeID, SHAMapTreeNodePtr>> data,
std::shared_ptr<Peer> const& peer);
std::vector<std::pair<SHAMapNodeID, Slice>> const& data,
std::shared_ptr<Peer> const&);
void
init(int startPeers);

View File

@@ -17,7 +17,6 @@ enum class ProtocolFeature {
ValidatorListPropagation,
ValidatorList2Propagation,
LedgerReplay,
LedgerNodeDepth,
};
/** Represents a peer connection in the overlay. */

View File

@@ -61,7 +61,6 @@
#include <xrpl/server/Handoff.h>
#include <xrpl/server/LoadFeeTrack.h>
#include <xrpl/server/NetworkOPs.h>
#include <xrpl/shamap/SHAMap.h>
#include <xrpl/shamap/SHAMapNodeID.h>
#include <xrpl/tx/apply.h>
@@ -566,8 +565,6 @@ PeerImp::supportsFeature(ProtocolFeature f) const
return protocol_ >= make_protocol(2, 1);
case ProtocolFeature::ValidatorList2Propagation:
return protocol_ >= make_protocol(2, 2);
case ProtocolFeature::LedgerNodeDepth:
return protocol_ >= make_protocol(2, 3);
case ProtocolFeature::LedgerReplay:
return ledgerReplayEnabled_;
}
@@ -1614,8 +1611,7 @@ PeerImp::onMessage(std::shared_ptr<protocol::TMGetLedger> const& m)
}
}
// Verify and parse ledger node IDs
std::vector<SHAMapNodeID> nodeIDs;
// Verify ledger node IDs
if (itype != protocol::liBASE)
{
if (m->nodeids_size() <= 0)
@@ -1624,16 +1620,13 @@ PeerImp::onMessage(std::shared_ptr<protocol::TMGetLedger> const& m)
return;
}
nodeIDs.reserve(m->nodeids_size());
for (auto const& nodeId : m->nodeids())
{
auto parsed = deserializeSHAMapNodeID(nodeId);
if (!parsed)
if (deserializeSHAMapNodeID(nodeId) == std::nullopt)
{
badData("Invalid SHAMap node ID");
return;
}
nodeIDs.push_back(std::move(*parsed));
}
}
@@ -1656,11 +1649,10 @@ PeerImp::onMessage(std::shared_ptr<protocol::TMGetLedger> const& m)
// Queue a job to process the request
std::weak_ptr<PeerImp> const weak = shared_from_this();
app_.getJobQueue().addJob(
jtLEDGER_REQ, "RcvGetLedger", [weak, m, nodeIDs = std::move(nodeIDs)]() mutable {
if (auto peer = weak.lock())
peer->processLedgerRequest(m, std::move(nodeIDs));
});
app_.getJobQueue().addJob(jtLEDGER_REQ, "RcvGetLedger", [weak, m]() {
if (auto peer = weak.lock())
peer->processLedgerRequest(m);
});
}
void
@@ -3369,9 +3361,7 @@ PeerImp::getTxSet(std::shared_ptr<protocol::TMGetLedger> const& m) const
}
void
PeerImp::processLedgerRequest(
std::shared_ptr<protocol::TMGetLedger> const& m,
std::vector<SHAMapNodeID> nodeIDs)
PeerImp::processLedgerRequest(std::shared_ptr<protocol::TMGetLedger> const& m)
{
// Do not resource charge a peer responding to a relay
if (!m->has_requestcookie())
@@ -3456,25 +3446,26 @@ PeerImp::processLedgerRequest(
}
// Add requested node data to reply
if (!nodeIDs.empty())
if (m->nodeids_size() > 0)
{
std::uint32_t const defaultDepth = isHighLatency() ? 2 : 1;
auto const queryDepth{m->has_querydepth() ? m->querydepth() : defaultDepth};
std::vector<SHAMapNodeData> data;
auto const useLedgerNodeDepth = supportsFeature(ProtocolFeature::LedgerNodeDepth);
std::vector<std::pair<SHAMapNodeID, Blob>> data;
for (auto const& nodeID : nodeIDs)
for (int i = 0;
i < m->nodeids_size() && ledgerData.nodes_size() < Tuning::softMaxReplyNodes;
++i)
{
if (ledgerData.nodes_size() >= Tuning::softMaxReplyNodes)
break;
auto const shaMapNodeId{deserializeSHAMapNodeID(m->nodeids(i))};
data.clear();
data.reserve(Tuning::softMaxReplyNodes);
try
{
if (map->getNodeFat(nodeID, data, fatLeaves, queryDepth))
// NOLINTNEXTLINE(bugprone-unchecked-optional-access) nodeids checked in onGetLedger
if (map->getNodeFat(*shaMapNodeId, data, fatLeaves, queryDepth))
{
JLOG(p_journal_.trace())
<< "processLedgerRequest: getNodeFat got " << data.size() << " nodes";
@@ -3483,26 +3474,9 @@ PeerImp::processLedgerRequest(
{
if (ledgerData.nodes_size() >= Tuning::hardMaxReplyNodes)
break;
protocol::TMLedgerNode* node{ledgerData.add_nodes()};
node->set_nodedata(d.data.data(), d.data.size());
// When the LedgerNodeDepth protocol feature is not supported by the peer,
// we always set the `nodeid` field. However, when it is supported then we
// set the `id` field for inner nodes and the `depth` field for leaf nodes.
if (!useLedgerNodeDepth)
{
node->set_nodeid(d.nodeID.getRawString());
}
else if (d.isLeaf)
{
node->set_depth(d.nodeID.getDepth());
}
else
{
node->set_id(d.nodeID.getRawString());
}
node->set_nodeid(d.first.getRawString());
node->set_nodedata(d.second.data(), d.second.size());
}
}
else
@@ -3541,7 +3515,7 @@ PeerImp::processLedgerRequest(
info += ", no hash specified";
JLOG(p_journal_.warn())
<< "processLedgerRequest: getNodeFat with nodeId " << nodeID
<< "processLedgerRequest: getNodeFat with nodeId " << *shaMapNodeId
<< " and ledger info type " << info << " throws exception: " << e.what();
}
}

View File

@@ -14,7 +14,6 @@
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/STValidation.h>
#include <xrpl/resource/Fees.h>
#include <xrpl/shamap/SHAMapNodeID.h>
#include <boost/circular_buffer.hpp>
#include <boost/endian/conversion.hpp>
@@ -793,9 +792,7 @@ private:
getTxSet(std::shared_ptr<protocol::TMGetLedger> const& m) const;
void
processLedgerRequest(
std::shared_ptr<protocol::TMGetLedger> const& m,
std::vector<SHAMapNodeID> nodeIDs);
processLedgerRequest(std::shared_ptr<protocol::TMGetLedger> const& m);
};
//------------------------------------------------------------------------------

View File

@@ -28,7 +28,6 @@ namespace xrpl {
constexpr ProtocolVersion const supportedProtocolList[]{
{2, 1},
{2, 2},
{2, 3},
};
// This ugly construct ensures that supportedProtocolList is sorted in strictly