ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2026-06-08 11:16:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: Commits for 2.5.0-b3 (#2197)

Browse Source
This commit is contained in:
Sergey Kuznetsov
2025-06-09 11:48:12 +01:00
committed by GitHub
parent 764601e7fc a68229e9d7
commit 7ec5f5e1bd
232 changed files with 7643 additions and 4141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
.github/actions/build_docker_image/action.yml vendored
View File

@@ -5,9 +5,6 @@ inputs:
images:
description: Name of the images to use as a base name
required: true
dockerhub_repo:
description: DockerHub repository name
required: true
push_image:
description: Whether to push the image to the registry (true/false)
required: true
@@ -20,15 +17,19 @@ inputs:
platforms:
description: Platforms to build the image for (e.g. linux/amd64,linux/arm64)
required: true
description:
dockerhub_repo:
description: DockerHub repository name
required: false
dockerhub_description:
description: Short description of the image
required: true
required: false
runs:
using: composite
steps:
- name: Login to DockerHub
if: ${{ inputs.push_image == 'true' }}
if: ${{ inputs.push_image == 'true' && inputs.dockerhub_repo != '' }}
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ env.DOCKERHUB_USER }}
@@ -54,7 +55,7 @@ runs:
tags: ${{ inputs.tags }}
- name: Build and push
uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
with:
context: ${{ inputs.directory }}
platforms: ${{ inputs.platforms }}
@@ -62,11 +63,11 @@ runs:
tags: ${{ steps.meta.outputs.tags }}
- name: Update DockerHub description
if: ${{ inputs.push_image == 'true' }}
if: ${{ inputs.push_image == 'true' && inputs.dockerhub_repo != '' }}
uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_PW }}
repository: ${{ inputs.dockerhub_repo }}
short-description: ${{ inputs.description }}
short-description: ${{ inputs.dockerhub_description }}
readme-filepath: ${{ inputs.directory }}/README.md

20
.github/actions/generate/action.yml vendored
View File

@@ -49,7 +49,7 @@ runs:
- name: Run conan
shell: bash
env:
BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '' }}"
BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '*' }}"
CODE_COVERAGE: "${{ inputs.code_coverage == 'true' && 'True' || 'False' }}"
STATIC_OPTION: "${{ inputs.static == 'true' && 'True' || 'False' }}"
INTEGRATION_TESTS_OPTION: "${{ inputs.build_integration_tests == 'true' && 'True' || 'False' }}"
@@ -59,15 +59,15 @@ runs:
conan \
install .. \
-of . \
-b $BUILD_OPTION \
-s build_type="${{ inputs.build_type }}" \
-o clio:static="${STATIC_OPTION}" \
-o clio:tests=True \
-o clio:integration_tests="${INTEGRATION_TESTS_OPTION}" \
-o clio:lint=False \
-o clio:coverage="${CODE_COVERAGE}" \
-o clio:time_trace="${TIME_TRACE}" \
--profile "${{ inputs.conan_profile }}"
-b "$BUILD_OPTION" \
-s "build_type=${{ inputs.build_type }}" \
-o "&:static=${STATIC_OPTION}" \
-o "&:tests=True" \
-o "&:integration_tests=${INTEGRATION_TESTS_OPTION}" \
-o "&:lint=False" \
-o "&:coverage=${CODE_COVERAGE}" \
-o "&:time_trace=${TIME_TRACE}" \
--profile:all "${{ inputs.conan_profile }}"
- name: Run cmake
shell: bash

10
.github/actions/prepare_runner/action.yml vendored
View File

@@ -18,13 +18,13 @@ runs:
ca-certificates \
ccache \
clang-build-analyzer \
conan@1 \
conan \
gh \
jq \
llvm@14 \
ninja \
pkg-config
echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
echo "/opt/homebrew/opt/conan@2/bin" >> $GITHUB_PATH
- name: Install CMake 3.31.6 on mac
if: ${{ runner.os == 'macOS' }}
@@ -55,14 +55,14 @@ runs:
shell: bash
run: |
echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
echo "CONAN_USER_HOME=${{ github.workspace }}" >> $GITHUB_ENV
echo "CONAN_HOME=${{ github.workspace }}/.conan2" >> $GITHUB_ENV
- name: Set env variables for Linux
if: ${{ runner.os == 'Linux' }}
shell: bash
run: |
echo "CCACHE_DIR=/root/.ccache" >> $GITHUB_ENV
echo "CONAN_USER_HOME=/root/" >> $GITHUB_ENV
echo "CONAN_HOME=/root/.conan2" >> $GITHUB_ENV
- name: Set CCACHE_DISABLE=1
if: ${{ inputs.disable_ccache == 'true' }}
@@ -74,4 +74,4 @@ runs:
shell: bash
run: |
mkdir -p "$CCACHE_DIR"
mkdir -p "$CONAN_USER_HOME/.conan"
mkdir -p "$CONAN_HOME"

8
.github/actions/restore_cache/action.yml vendored
View File

@@ -3,7 +3,7 @@ description: Find and restores conan and ccache cache
inputs:
conan_dir:
description: Path to .conan directory
description: Path to Conan directory
required: true
conan_profile:
description: Conan profile name
@@ -41,8 +41,8 @@ runs:
id: conan_hash
shell: bash
run: |
conan info . -j info.json -o clio:tests=True
packages_info="$(cat info.json | jq '.[] | "\(.display_name): \(.id)"' | grep -v 'clio')"
conan graph info . --format json --out-file info.json -o '&:tests=True' --profile:all ${{ inputs.conan_profile }}
packages_info="$(cat info.json | jq -r '.graph.nodes[]?.ref' | grep -v 'clio')"
echo "$packages_info"
hash="$(echo "$packages_info" | shasum -a 256 | cut -d ' ' -f 1)"
rm info.json
@@ -52,7 +52,7 @@ runs:
uses: actions/cache/restore@v4
id: conan_cache
with:
path: ${{ inputs.conan_dir }}/data
path: ${{ inputs.conan_dir }}/p
key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ steps.conan_hash.outputs.hash }}
- name: Restore ccache cache

4
.github/actions/save_cache/action.yml vendored
View File

@@ -42,13 +42,13 @@ runs:
if: ${{ inputs.conan_cache_hit != 'true' }}
shell: bash
run: |
conan remove "*" -s -b -f
conan cache clean --source --build --temp
- name: Save conan cache
if: ${{ inputs.conan_cache_hit != 'true' }}
uses: actions/cache/save@v4
with:
path: ${{ inputs.conan_dir }}/data
path: ${{ inputs.conan_dir }}/p
key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ inputs.conan_hash }}
- name: Save ccache cache

19
.github/actions/setup_conan/action.yml vendored
View File

@@ -12,22 +12,11 @@ runs:
- name: Create conan profile on macOS
if: ${{ runner.os == 'macOS' }}
shell: bash
env:
CONAN_PROFILE: ${{ inputs.conan_profile }}
run: |
echo "Creating \"$CONAN_PROFILE\" conan profile"
conan profile new "$CONAN_PROFILE" --detect --force
conan profile update settings.compiler.libcxx=libc++ "$CONAN_PROFILE"
conan profile update settings.compiler.cppstd=20 "$CONAN_PROFILE"
conan profile update env.CXXFLAGS=-DBOOST_ASIO_DISABLE_CONCEPTS "$CONAN_PROFILE"
conan profile update "conf.tools.build:cxxflags+=[\"-DBOOST_ASIO_DISABLE_CONCEPTS\"]" "$CONAN_PROFILE"
conan profile detect --name "${{ inputs.conan_profile }}" --force
sed -i '' 's/compiler.cppstd=[^ ]*/compiler.cppstd=20/' "${{ env.CONAN_HOME }}/profiles/${{ inputs.conan_profile }}"
- name: Add conan-non-prod artifactory
- name: Add artifactory remote
shell: bash
run: |
if [[ -z "$(conan remote list | grep conan-non-prod)" ]]; then
echo "Adding conan-non-prod"
conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
else
echo "Conan-non-prod is available"
fi
conan remote add --index 0 --force ripple http://18.143.149.228:8081/artifactory/api/conan/dev

24
.github/scripts/prepare-release-artifacts.sh vendored Executable file
View File

@@ -0,0 +1,24 @@
#!/bin/bash
set -ex -o pipefail
BINARY_NAME="clio_server"
ARTIFACTS_DIR="$1"
if [ -z "${ARTIFACTS_DIR}" ]; then
echo "Usage: $0 <artifacts_directory>"
exit 1
fi
cd "${ARTIFACTS_DIR}" || exit 1
for artifact_name in $(ls); do
pushd "${artifact_name}" || exit 1
zip -r "../${artifact_name}.zip" ./${BINARY_NAME}
popd || exit 1
rm "${artifact_name}/${BINARY_NAME}"
rm -r "${artifact_name}"
sha256sum "./${artifact_name}.zip" > "./${artifact_name}.zip.sha256sum"
done

1
.github/workflows/build.yml vendored
View File

@@ -18,6 +18,7 @@ on:
- "!.github/actions/create_issue/**"
- CMakeLists.txt
- conanfile.py
- "cmake/**"
- "src/**"
- "tests/**"

8
.github/workflows/build_clio_docker_image.yml vendored
View File

@@ -85,16 +85,16 @@ jobs:
- name: Build Docker image
uses: ./.github/actions/build_docker_image
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
images: |
rippleci/clio
ghcr.io/xrplf/clio
dockerhub_repo: rippleci/clio
rippleci/clio
push_image: ${{ inputs.publish_image }}
directory: docker/clio
tags: ${{ inputs.tags }}
platforms: linux/amd64
description: Clio is an XRP Ledger API server.
dockerhub_repo: rippleci/clio
dockerhub_description: Clio is an XRP Ledger API server.

4
.github/workflows/build_impl.yml vendored
View File

@@ -92,7 +92,7 @@ jobs:
uses: ./.github/actions/restore_cache
id: restore_cache
with:
conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
conan_dir: ${{ env.CONAN_HOME }}
conan_profile: ${{ inputs.conan_profile }}
ccache_dir: ${{ env.CCACHE_DIR }}
build_type: ${{ inputs.build_type }}
@@ -172,7 +172,7 @@ jobs:
if: ${{ !inputs.disable_cache && github.ref == 'refs/heads/develop' }}
uses: ./.github/actions/save_cache
with:
conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
conan_dir: ${{ env.CONAN_HOME }}
conan_hash: ${{ steps.restore_cache.outputs.conan_hash }}
conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
ccache_dir: ${{ env.CCACHE_DIR }}

2
.github/workflows/clang-tidy.yml vendored
View File

@@ -49,7 +49,7 @@ jobs:
uses: ./.github/actions/restore_cache
id: restore_cache
with:
conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
conan_dir: ${{ env.CONAN_HOME }}
ccache_dir: ${{ env.CCACHE_DIR }}
conan_profile: ${{ env.CONAN_PROFILE }}

38
.github/workflows/docs.yml vendored
View File

@@ -5,23 +5,14 @@ on:
branches: [develop]
workflow_dispatch:
permissions:
contents: read
pages: write
id-token: write
concurrency:
# Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
deploy:
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
build:
runs-on: ubuntu-latest
continue-on-error: true
container:
image: ghcr.io/xrplf/clio-ci:latest
@@ -31,10 +22,16 @@ jobs:
with:
lfs: true
- name: Build docs
run: |
mkdir -p build_docs && cd build_docs
cmake ../docs && cmake --build . --target docs
- name: Create build directory
run: mkdir build_docs
- name: Configure CMake
working-directory: build_docs
run: cmake ../docs
- name: Build
working-directory: build_docs
run: cmake --build . --target docs
- name: Setup Pages
uses: actions/configure-pages@v5
@@ -45,6 +42,19 @@ jobs:
path: build_docs/html
name: docs-develop
deploy:
needs: build
permissions:
pages: write
id-token: write
environment:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
runs-on: ubuntu-latest
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4

32
.github/workflows/nightly.yml vendored
View File

@@ -11,10 +11,12 @@ on:
- .github/workflows/release_impl.yml
- .github/workflows/build_and_test.yml
- .github/workflows/build_impl.yml
- .github/workflows/test_impl.yml
- .github/workflows/build_clio_docker_image.yml
- ".github/actions/**"
- "!.github/actions/code_coverage/**"
- .github/scripts/prepare-release-artifacts.sh
concurrency:
# Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
@@ -33,16 +35,25 @@ jobs:
conan_profile: default_apple_clang
build_type: Release
static: false
sanitizer: "false"
- os: heavy
conan_profile: gcc
build_type: Release
static: true
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
sanitizer: "false"
- os: heavy
conan_profile: gcc
build_type: Debug
static: true
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
sanitizer: "false"
- os: heavy
conan_profile: gcc.ubsan
build_type: Release
static: false
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
sanitizer: "ubsan"
uses: ./.github/workflows/build_and_test.yml
with:
@@ -55,6 +66,7 @@ jobs:
run_integration_tests: true
upload_clio_server: true
disable_cache: true
sanitizer: ${{ matrix.sanitizer }}
analyze_build_time:
name: Analyze Build Time
@@ -63,13 +75,10 @@ jobs:
fail-fast: false
matrix:
include:
# TODO: Enable when we have at least ubuntu 22.04
# as ClangBuildAnalyzer requires relatively modern glibc
#
# - os: heavy
# conan_profile: clang
# container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
# static: true
- os: heavy
conan_profile: clang
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
static: true
- os: macos15
conan_profile: default_apple_clang
container: ""
@@ -95,7 +104,14 @@ jobs:
overwrite_release: true
title: "Clio development (nightly) build"
version: nightly
notes_header_file: nightly_notes.md
header: >
# Release notes
> **Note:** Please remember that this is a development release and it is not recommended for production use.
Changelog (including previous releases): <https://github.com/XRPLF/clio/commits/nightly>
generate_changelog: false
draft: false
build_and_publish_docker_image:
uses: ./.github/workflows/build_clio_docker_image.yml

7
.github/workflows/nightly_notes.md vendored
View File

@@ -1,7 +0,0 @@
# Release notes
> **Note:** Please remember that this is a development release and it is not recommended for production use.
Changelog (including previous releases): <https://github.com/XRPLF/clio/commits/nightly>
## SHA256 checksums

4
.github/workflows/pre-commit-autoupdate.yml vendored
View File

@@ -33,7 +33,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
with:
branch: update/pre-commit-hooks
title: Update pre-commit hooks
commit-message: "style: update pre-commit hooks"
title: "style: Update pre-commit hooks"
commit-message: "style: Update pre-commit hooks"
body: Update versions of pre-commit hooks to latest version.
reviewers: "godexsoft,kuznetsss,PeterChen13579,mathbunnyru"

56
.github/workflows/release.yml vendored Normal file
View File

@@ -0,0 +1,56 @@
name: Create release
on:
push:
tags:
- "*.*.*"
pull_request:
paths:
- .github/workflows/release.yml
concurrency:
# Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build-and-test:
name: Build and Test
strategy:
fail-fast: false
matrix:
include:
- os: macos15
conan_profile: default_apple_clang
build_type: Release
static: false
- os: heavy
conan_profile: gcc
build_type: Release
static: true
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
uses: ./.github/workflows/build_and_test.yml
with:
runs_on: ${{ matrix.os }}
container: ${{ matrix.container }}
conan_profile: ${{ matrix.conan_profile }}
build_type: ${{ matrix.build_type }}
static: ${{ matrix.static }}
run_unit_tests: true
run_integration_tests: true
upload_clio_server: true
disable_cache: true
release:
needs: build-and-test
uses: ./.github/workflows/release_impl.yml
with:
overwrite_release: false
title: "${{ github.ref_name}}"
version: "${{ github.ref_name }}"
header: >
# Introducing Clio version ${{ github.ref_name }}
generate_changelog: true
draft: true

71
.github/workflows/release_impl.yml vendored
View File

@@ -18,14 +18,26 @@ on:
required: true
type: string
notes_header_file:
description: "Release notes header file"
header:
description: "Release notes header"
required: true
type: string
generate_changelog:
description: "Generate changelog"
required: false
type: boolean
draft:
description: "Create a draft release"
required: false
type: boolean
jobs:
release:
runs-on: ubuntu-latest
runs-on: heavy
container:
image: ghcr.io/xrplf/clio-ci:latest
env:
GH_REPO: ${{ github.repository }}
GH_TOKEN: ${{ github.token }}
@@ -35,29 +47,55 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Prepare runner
uses: ./.github/actions/prepare_runner
with:
disable_ccache: true
- uses: actions/download-artifact@v4
with:
path: release_artifacts
pattern: clio_server_*
- name: Prepare files
- name: Create release notes
shell: bash
run: |
printf '%s\n' "${{ inputs.header }}" > "${RUNNER_TEMP}/release_notes.md"
- name: Generate changelog
shell: bash
if: ${{ inputs.generate_changelog }}
run: |
LAST_TAG=$(gh release view --json tagName -q .tagName)
LAST_TAG_COMMIT=$(git rev-parse $LAST_TAG)
BASE_COMMIT=$(git merge-base HEAD $LAST_TAG_COMMIT)
git-cliff "${BASE_COMMIT}..HEAD" --ignore-tags "nightly|-b"
cat CHANGELOG.md >> "${RUNNER_TEMP}/release_notes.md"
- name: Prepare release artifacts
shell: bash
run: .github/scripts/prepare-release-artifacts.sh release_artifacts
- name: Append sha256 checksums
shell: bash
working-directory: release_artifacts
run: |
cp ${{ github.workspace }}/.github/workflows/${{ inputs.notes_header_file }} "${RUNNER_TEMP}/release_notes.md"
echo '' >> "${RUNNER_TEMP}/release_notes.md"
echo '```' >> "${RUNNER_TEMP}/release_notes.md"
{
echo '## SHA256 checksums'
echo
echo '```'
cat *.sha256sum
echo '```'
} >> "${RUNNER_TEMP}/release_notes.md"
for d in $(ls); do
archive_name=$(ls $d)
mv ${d}/${archive_name} ./
rm -r $d
sha256sum ./$archive_name > ./${archive_name}.sha256sum
cat ./$archive_name.sha256sum >> "${RUNNER_TEMP}/release_notes.md"
done
echo '```' >> "${RUNNER_TEMP}/release_notes.md"
- name: Upload release notes
uses: actions/upload-artifact@v4
with:
name: release_notes_${{ inputs.version }}
path: "${RUNNER_TEMP}/release_notes.md"
- name: Remove current release and tag
if: ${{ github.event_name != 'pull_request' && inputs.overwrite_release }}
@@ -74,5 +112,6 @@ jobs:
${{ inputs.overwrite_release && '--prerelease' || '' }} \
--title "${{ inputs.title }}" \
--target $GITHUB_SHA \
${{ inputs.draft && '--draft' || '' }} \
--notes-file "${RUNNER_TEMP}/release_notes.md" \
./release_artifacts/clio_server*

1
.github/workflows/sanitizers.yml vendored
View File

@@ -18,6 +18,7 @@ on:
- .github/scripts/execute-tests-under-sanitizer
- CMakeLists.txt
- conanfile.py
- "cmake/**"
# We don't run sanitizer on code change, because it takes too long
# - "src/**"

110
.github/workflows/update_docker_ci.yml vendored
View File

@@ -9,6 +9,7 @@ on:
- "docker/ci/**"
- "docker/compilers/**"
- "docker/tools/**"
push:
branches: [develop]
paths:
@@ -16,10 +17,9 @@ on:
- ".github/actions/build_docker_image/**"
# CI image must update when either its Dockerfile changes
# or any compilers changed and were pushed by hand
- "docker/ci/**"
- "docker/compilers/**"
- "docker/tools/**"
workflow_dispatch:
concurrency:
@@ -28,22 +28,117 @@ concurrency:
cancel-in-progress: true
jobs:
build_and_push:
name: Build and push docker image
gcc:
name: Build and push GCC docker image
runs-on: [self-hosted, heavy]
steps:
- uses: actions/checkout@v4
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
with:
files: "docker/compilers/gcc/**"
- uses: ./.github/actions/build_docker_image
# Skipping this build for now, because CI environment is not stable
if: false && steps.changed-files.outputs.any_changed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
with:
images: |
ghcr.io/xrplf/clio-gcc
rippleci/clio_gcc
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/compilers/gcc
tags: |
type=raw,value=latest
type=raw,value=12
type=raw,value=12.3.0
type=raw,value=${{ github.sha }}
platforms: linux/amd64,linux/arm64
dockerhub_repo: rippleci/clio_gcc
dockerhub_description: GCC compiler for XRPLF/clio.
clang:
name: Build and push Clang docker image
runs-on: [self-hosted, heavy]
steps:
- uses: actions/checkout@v4
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
with:
files: "docker/compilers/clang/**"
- uses: ./.github/actions/build_docker_image
if: steps.changed-files.outputs.any_changed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
with:
images: |
ghcr.io/xrplf/clio-clang
rippleci/clio_clang
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/compilers/clang
tags: |
type=raw,value=latest
type=raw,value=16
type=raw,value=${{ github.sha }}
platforms: linux/amd64,linux/arm64
dockerhub_repo: rippleci/clio_clang
dockerhub_description: Clang compiler for XRPLF/clio.
tools:
name: Build and push tools docker image
runs-on: [self-hosted, heavy]
steps:
- uses: actions/checkout@v4
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
with:
files: "docker/tools/**"
- uses: ./.github/actions/build_docker_image
if: steps.changed-files.outputs.any_changed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
images: |
ghcr.io/xrplf/clio-tools
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/tools
tags: |
type=raw,value=latest
type=raw,value=${{ github.sha }}
platforms: linux/amd64,linux/arm64
ci:
name: Build and push CI docker image
runs-on: [self-hosted, heavy]
needs: [gcc, clang, tools]
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/build_docker_image
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
images: |
rippleci/clio_ci
ghcr.io/xrplf/clio-ci
dockerhub_repo: rippleci/clio_ci
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/ci
tags: |
@@ -51,4 +146,5 @@ jobs:
type=raw,value=gcc_12_clang_16
type=raw,value=${{ github.sha }}
platforms: linux/amd64,linux/arm64
description: CI image for XRPLF/clio.
dockerhub_repo: rippleci/clio_ci
dockerhub_description: CI image for XRPLF/clio.

4
.hadolint.yml
View File

@@ -1,8 +1,6 @@
---
ignored:
- DL3003
- DL3007
- DL3008
- DL3013
- DL3015
- DL3027
- DL3047

5
.pre-commit-config.yaml
View File

@@ -11,6 +11,8 @@
#
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
exclude: ^docs/doxygen-awesome-theme/
repos:
# `pre-commit sample-config` default hooks
- repo: https://github.com/pre-commit/pre-commit-hooks
@@ -20,16 +22,13 @@ repos:
- id: check-executables-have-shebangs
- id: check-shebang-scripts-are-executable
- id: end-of-file-fixer
exclude: ^docs/doxygen-awesome-theme/
- id: trailing-whitespace
exclude: ^docs/doxygen-awesome-theme/
# Autoformat: YAML, JSON, Markdown, etc.
- repo: https://github.com/rbubley/mirrors-prettier
rev: 787fb9f542b140ba0b2aced38e6a3e68021647a3 # frozen: v3.5.3
hooks:
- id: prettier
exclude: ^docs/doxygen-awesome-theme/
- repo: https://github.com/igorshubovych/markdownlint-cli
rev: 586c3ea3f51230da42bab657c6a32e9e66c364f0 # frozen: v0.44.0

3
benchmarks/util/async/ExecutionContextBenchmarks.cpp
View File

@@ -191,8 +191,9 @@ generateData()
constexpr auto kTOTAL = 10'000;
std::vector<uint64_t> data;
data.reserve(kTOTAL);
util::MTRandomGenerator randomGenerator;
for (auto i = 0; i < kTOTAL; ++i)
data.push_back(util::Random::uniform(1, 100'000'000));
data.push_back(randomGenerator.uniform(1, 100'000'000));
return data;
}

17
cliff.toml
View File

@@ -8,15 +8,24 @@
[changelog]
# template for the changelog header
header = """
# Changelog\n
All notable changes to this project will be documented in this file.\n
"""
# template for the changelog body
# https://keats.github.io/tera/docs/#introduction
body = """
{% if version %}\
## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
Version {{ version | trim_start_matches(pat="v") }} of Clio, an XRP Ledger API server optimized for HTTP and WebSocket API calls, is now available.
{% else %}\
Clio, an XRP Ledger API server optimized for HTTP and WebSocket API calls, is under active development.
{% endif %}\
<!-- Please, remove one of the 2 following lines -->
This release adds new features and bug fixes.
This release adds bug fixes.
\
{% if version %}
## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
{% else %}
## [unreleased]
{% endif %}\
{% for group, commits in commits | filter(attribute="merge_commit", value=false) | group_by(attribute="group") %}
@@ -24,7 +33,7 @@ body = """
{% for commit in commits %}
- {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
{% if commit.breaking %}[**breaking**] {% endif %}\
{{ commit.message | upper_first }} {% if commit.remote.username %}by @{{ commit.remote.username }}{% endif %}\
{{ commit.message | upper_first }}{% if commit.remote.username %} by @{{ commit.remote.username }}{% endif %}\
{% endfor %}
{% endfor %}\n
"""

12
conanfile.py
View File

@@ -2,10 +2,10 @@ from conan import ConanFile
from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
class Clio(ConanFile):
class ClioConan(ConanFile):
name = 'clio'
license = 'ISC'
author = 'Alex Kremer <akremer@ripple.com>, John Freeman <jfreeman@ripple.com>'
author = 'Alex Kremer <akremer@ripple.com>, John Freeman <jfreeman@ripple.com>, Ayaz Salikhov <asalikhov@ripple.com>'
url = 'https://github.com/xrplf/clio'
description = 'Clio RPC server'
settings = 'os', 'compiler', 'build_type', 'arch'
@@ -28,10 +28,10 @@ class Clio(ConanFile):
'boost/1.83.0',
'cassandra-cpp-driver/2.17.0',
'fmt/10.1.1',
'protobuf/3.21.9',
'protobuf/3.21.12',
'grpc/1.50.1',
'openssl/1.1.1v',
'xrpl/2.5.0-b1',
'xrpl/2.5.0-rc1',
'zlib/1.3.1',
'libbacktrace/cci.20210118'
]
@@ -100,6 +100,10 @@ class Clio(ConanFile):
tc.variables['benchmark'] = self.options.benchmark
tc.variables['snapshot'] = self.options.snapshot
tc.variables['time_trace'] = self.options.time_trace
if self.settings.compiler == 'clang' and self.settings.compiler.version == 16:
tc.extra_cxxflags = ["-DBOOST_ASIO_DISABLE_CONCEPTS"]
tc.generate()
def build(self):

145
docker/ci/Dockerfile
View File

@@ -1,6 +1,10 @@
FROM rippleci/clio_clang:16
# TODO: change this when we are able to push gcc image to ghcr.io
FROM rippleci/clio_gcc:12.3.0 AS clio-gcc
FROM ghcr.io/xrplf/clio-tools:latest AS clio-tools
FROM ghcr.io/xrplf/clio-clang:16
ARG DEBIAN_FRONTEND=noninteractive
ARG TARGETARCH
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -13,31 +17,53 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
USER root
WORKDIR /root
ENV CCACHE_VERSION=4.10.2 \
LLVM_TOOLS_VERSION=19 \
GH_VERSION=2.40.0 \
DOXYGEN_VERSION=1.12.0 \
CLANG_BUILD_ANALYZER_VERSION=1.6.0 \
GIT_CLIFF_VERSION=2.8.0
ARG LLVM_TOOLS_VERSION=19
# Add repositories
RUN apt-get -qq update \
&& apt-get -qq install -y --no-install-recommends --no-install-suggests gnupg wget curl software-properties-common \
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
curl \
gnupg \
wget \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM_TOOLS_VERSION} main" >> /etc/apt/sources.list \
&& wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
apt-get clean && rm -rf /var/lib/apt/lists/*
&& wget --progress=dot:giga -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
# Install packages
RUN apt update -qq \
&& apt install -y --no-install-recommends --no-install-suggests python3 python3-pip git git-lfs make ninja-build flex bison jq graphviz \
clang-tidy-${LLVM_TOOLS_VERSION} clang-tools-${LLVM_TOOLS_VERSION} \
&& pip3 install -q --upgrade --no-cache-dir pip && pip3 install -q --no-cache-dir conan==1.62 gcovr cmake==3.31.6 pre-commit \
&& apt-get clean && apt remove -y software-properties-common
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
bison \
clang-tidy-${LLVM_TOOLS_VERSION} \
clang-tools-${LLVM_TOOLS_VERSION} \
flex \
git \
git-lfs \
graphviz \
jq \
make \
ninja-build \
python3 \
python3-pip \
zip \
&& pip3 install -q --upgrade --no-cache-dir pip \
&& pip3 install -q --no-cache-dir \
cmake==3.31.6 \
conan==2.17.0 \
gcovr \
pre-commit \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
# Install gcc-12 and make ldconfig aware of the new libstdc++ location (for gcc)
# Note: Clang is using libc++ instead
COPY --from=rippleci/clio_gcc:12.3.0 /gcc12.deb /
RUN apt update && apt-get install -y binutils libc6-dev \
COPY --from=clio-gcc /gcc12.deb /
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
binutils \
libc6-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& dpkg -i /gcc12.deb \
&& rm -rf /gcc12.deb \
&& ldconfig
@@ -51,74 +77,29 @@ RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
&& update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
&& update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100
WORKDIR /tmp
# Install ccache from source
RUN wget "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz" \
&& tar xf "ccache-${CCACHE_VERSION}.tar.gz" \
&& cd "ccache-${CCACHE_VERSION}" \
&& mkdir build && cd build \
&& cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
&& cmake --build . --target install \
&& rm -rf /tmp/* /var/tmp/*
# Install doxygen from source
RUN wget "https://github.com/doxygen/doxygen/releases/download/Release_${DOXYGEN_VERSION//./_}/doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
&& tar xf "doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
&& cd "doxygen-${DOXYGEN_VERSION}" \
&& mkdir build && cd build \
&& cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
&& cmake --build . --target install \
&& rm -rf /tmp/* /var/tmp/*
# Install ClangBuildAnalyzer
RUN wget "https://github.com/aras-p/ClangBuildAnalyzer/releases/download/v${CLANG_BUILD_ANALYZER_VERSION}/ClangBuildAnalyzer-linux" \
&& chmod +x ClangBuildAnalyzer-linux \
&& mv ClangBuildAnalyzer-linux /usr/bin/ClangBuildAnalyzer \
&& rm -rf /tmp/* /var/tmp/*
# Install git-cliff
RUN wget "https://github.com/orhun/git-cliff/releases/download/v${GIT_CLIFF_VERSION}/git-cliff-${GIT_CLIFF_VERSION}-x86_64-unknown-linux-musl.tar.gz" \
&& tar xf git-cliff-${GIT_CLIFF_VERSION}-x86_64-unknown-linux-musl.tar.gz \
&& mv git-cliff-${GIT_CLIFF_VERSION}/git-cliff /usr/bin/git-cliff \
&& rm -rf /tmp/* /var/tmp/*
# Install gh
RUN wget "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz" \
&& tar xf gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
&& mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh \
&& rm -rf /tmp/* /var/tmp/*
COPY --from=clio-tools \
/usr/local/bin/ccache \
/usr/local/bin/doxygen \
/usr/local/bin/ClangBuildAnalyzer \
/usr/local/bin/git-cliff \
/usr/local/bin/gh \
/usr/local/bin/
WORKDIR /root
# Setup conan
RUN conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
RUN conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev
# Note: intentionally leaving cppstd=20
RUN conan profile new gcc --detect \
&& conan profile update settings.compiler=gcc gcc \
&& conan profile update settings.compiler.version=12 gcc \
&& conan profile update settings.compiler.cppstd=20 gcc \
&& conan profile update settings.compiler.libcxx=libstdc++11 gcc \
&& conan profile update env.CC=/usr/bin/gcc-12 gcc \
&& conan profile update env.CXX=/usr/bin/g++-12 gcc \
&& conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/gcc-12\", \"cpp\": \"/usr/bin/g++-12\"}" gcc
WORKDIR /root/.conan2/profiles
RUN conan profile new clang --detect \
&& conan profile update settings.compiler=clang clang \
&& conan profile update settings.compiler.version=16 clang \
&& conan profile update settings.compiler.cppstd=20 clang \
&& conan profile update settings.compiler.libcxx=libc++ clang \
&& conan profile update env.CC=/usr/bin/clang-16 clang \
&& conan profile update env.CXX=/usr/bin/clang++-16 clang \
&& conan profile update env.CXXFLAGS="-DBOOST_ASIO_DISABLE_CONCEPTS" clang \
&& conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/clang-16\", \"cpp\": \"/usr/bin/clang++-16\"}" clang
COPY conan/clang.profile ./clang
COPY conan/sanitizer_template.profile ./clang.asan
COPY conan/sanitizer_template.profile ./clang.tsan
COPY conan/sanitizer_template.profile ./clang.ubsan
RUN echo "include(gcc)" >> .conan/profiles/default
COPY conan/gcc.profile ./gcc
COPY conan/sanitizer_template.profile ./gcc.asan
COPY conan/sanitizer_template.profile ./gcc.tsan
COPY conan/sanitizer_template.profile ./gcc.ubsan
COPY conan/gcc.asan /root/.conan/profiles
COPY conan/gcc.tsan /root/.conan/profiles
COPY conan/gcc.ubsan /root/.conan/profiles
COPY conan/clang.asan /root/.conan/profiles
COPY conan/clang.tsan /root/.conan/profiles
COPY conan/clang.ubsan /root/.conan/profiles
WORKDIR /root

16
docker/ci/README.md
View File

@@ -5,13 +5,17 @@ It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but c
The image is based on Ubuntu 20.04 and contains:
- ccache 4.11.3
- clang 16.0.6
- gcc 12.3
- ClangBuildAnalyzer 1.6.0
- conan 2.17.0
- doxygen 1.12
- gh 2.40
- ccache 4.10.2
- conan 1.62
- gcc 12.3.0
- gh 2.74
- git-cliff 2.9.1
- and some other useful tools
Conan is set up to build Clio without any additional steps. There are two preset conan profiles: `clang` and `gcc` to use corresponding compiler. By default conan is setup to use `gcc`.
Sanitizer builds for `ASAN`, `TSAN` and `UBSAN` are enabled via conan profiles for each of the supported compilers. These can be selected using the following pattern (all lowercase): `[compiler].[sanitizer]` (e.g. `--profile gcc.tsan`).
Conan is set up to build Clio without any additional steps.
There are two preset conan profiles: `clang` and `gcc` to use corresponding compiler.
`ASan`, `TSan` and `UBSan` sanitizer builds are enabled via conan profiles for each of the supported compilers.
These can be selected using the following pattern (all lowercase): `[compiler].[sanitizer]` (e.g. `--profile:all gcc.tsan`).

9
docker/ci/conan/clang.asan
View File

@@ -1,9 +0,0 @@
include(clang)
[options]
boost:extra_b2_flags="cxxflags=\"-fsanitize=address\" linkflags=\"-fsanitize=address\""
boost:without_stacktrace=True
[env]
CFLAGS="-fsanitize=address"
CXXFLAGS="-fsanitize=address"
LDFLAGS="-fsanitize=address"

11
docker/ci/conan/clang.profile Normal file
View File

@@ -0,0 +1,11 @@
[settings]
arch=x86_64
build_type=Release
compiler=clang
compiler.cppstd=20
compiler.libcxx=libc++
compiler.version=16
os=Linux
[conf]
tools.build:compiler_executables={'c': '/usr/bin/clang-16', 'cpp': '/usr/bin/clang++-16'}

9
docker/ci/conan/clang.tsan
View File

@@ -1,9 +0,0 @@
include(clang)
[options]
boost:extra_b2_flags="cxxflags=\"-fsanitize=thread\" linkflags=\"-fsanitize=thread\""
boost:without_stacktrace=True
[env]
CFLAGS="-fsanitize=thread"
CXXFLAGS="-fsanitize=thread"
LDFLAGS="-fsanitize=thread"

9
docker/ci/conan/clang.ubsan
View File

@@ -1,9 +0,0 @@
include(clang)
[options]
boost:extra_b2_flags="cxxflags=\"-fsanitize=undefined\" linkflags=\"-fsanitize=undefined\""
boost:without_stacktrace=True
[env]
CFLAGS="-fsanitize=undefined"
CXXFLAGS="-fsanitize=undefined"
LDFLAGS="-fsanitize=undefined"

9
docker/ci/conan/gcc.asan
View File

@@ -1,9 +0,0 @@
include(gcc)
[options]
boost:extra_b2_flags="cxxflags=\"-fsanitize=address\" linkflags=\"-fsanitize=address\""
boost:without_stacktrace=True
[env]
CFLAGS="-fsanitize=address"
CXXFLAGS="-fsanitize=address"
LDFLAGS="-fsanitize=address"

11
docker/ci/conan/gcc.profile Normal file
View File

@@ -0,0 +1,11 @@
[settings]
arch=x86_64
build_type=Release
compiler=gcc
compiler.cppstd=20
compiler.libcxx=libstdc++11
compiler.version=12
os=Linux
[conf]
tools.build:compiler_executables={'c': '/usr/bin/gcc-12', 'cpp': '/usr/bin/g++-12'}

9
docker/ci/conan/gcc.tsan
View File

@@ -1,9 +0,0 @@
include(gcc)
[options]
boost:extra_b2_flags="cxxflags=\"-fsanitize=thread\" linkflags=\"-fsanitize=thread\""
boost:without_stacktrace=True
[env]
CFLAGS="-fsanitize=thread"
CXXFLAGS="-fsanitize=thread"
LDFLAGS="-fsanitize=thread"

9
docker/ci/conan/gcc.ubsan
View File

@@ -1,9 +0,0 @@
include(gcc)
[options]
boost:extra_b2_flags="cxxflags=\"-fsanitize=undefined\" linkflags=\"-fsanitize=undefined\""
boost:without_stacktrace=True
[env]
CFLAGS="-fsanitize=undefined"
CXXFLAGS="-fsanitize=undefined"
LDFLAGS="-fsanitize=undefined"

15
docker/ci/conan/sanitizer_template.profile Normal file
View File

@@ -0,0 +1,15 @@
{% set compiler, sani = profile_name.split('.') %}
{% set sanitizer_opt_map = {'asan': 'address', 'tsan': 'thread', 'ubsan': 'undefined'} %}
{% set sanitizer = sanitizer_opt_map[sani] %}
include({{ compiler }})
[options]
boost/*:extra_b2_flags="cxxflags=\"-fsanitize={{ sanitizer }}\" linkflags=\"-fsanitize={{ sanitizer }}\""
boost/*:without_stacktrace=True
[conf]
tools.build:cflags+=["-fsanitize={{ sanitizer }}"]
tools.build:cxxflags+=["-fsanitize={{ sanitizer }}"]
tools.build:exelinkflags+=["-fsanitize={{ sanitizer }}"]

19
docker/clio/Dockerfile
View File

@@ -2,14 +2,17 @@ FROM ubuntu:22.04
COPY ./clio_server /opt/clio/bin/clio_server
RUN ln -s /opt/clio/bin/clio_server /usr/local/bin/clio_server && \
mkdir -p /opt/clio/etc/ && \
mkdir -p /opt/clio/log/ && \
groupadd -g 10001 clio && \
useradd -u 10000 -g 10001 -s /bin/bash clio && \
chown clio:clio /opt/clio/log && \
apt update && \
apt install -y libatomic1
RUN ln -s /opt/clio/bin/clio_server /usr/local/bin/clio_server \
&& mkdir -p /opt/clio/etc/ \
&& mkdir -p /opt/clio/log/ \
&& groupadd -g 10001 clio \
&& useradd -u 10000 -g 10001 -s /bin/bash clio \
&& chown clio:clio /opt/clio/log \
&& apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
libatomic1 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
USER clio
ENTRYPOINT ["/opt/clio/bin/clio_server"]

21
docker/compilers/clang-16/Dockerfile
View File

@@ -1,21 +0,0 @@
FROM ubuntu:focal
ARG DEBIAN_FRONTEND=noninteractive
ARG TARGETARCH
SHELL ["/bin/bash", "-c"]
# hadolint ignore=DL3002
USER root
WORKDIR /root
ENV CLANG_VERSION=16
RUN apt update -qq \
&& apt install -qq -y --no-install-recommends --no-install-suggests \
wget software-properties-common gnupg
RUN wget https://apt.llvm.org/llvm.sh \
&& chmod +x llvm.sh \
&& ./llvm.sh ${CLANG_VERSION} \
&& rm -rf llvm.sh \
&& apt-get install -y libc++-16-dev libc++abi-16-dev

30
docker/compilers/clang/Dockerfile Normal file
View File

@@ -0,0 +1,30 @@
FROM ubuntu:20.04
ARG DEBIAN_FRONTEND=noninteractive
SHELL ["/bin/bash", "-c"]
# hadolint ignore=DL3002
USER root
WORKDIR /root
ARG CLANG_VERSION=16
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
wget \
software-properties-common \
gnupg \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN wget --progress=dot:giga https://apt.llvm.org/llvm.sh \
&& chmod +x llvm.sh \
&& ./llvm.sh ${CLANG_VERSION} \
&& rm -rf llvm.sh \
&& apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
libc++-${CLANG_VERSION}-dev \
libc++abi-${CLANG_VERSION}-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

3
docker/compilers/clang/README.md Normal file
View File

@@ -0,0 +1,3 @@
# Clang compiler
This image contains clang compiler to build <https://github.com/XRPLF/clio>.

86
docker/compilers/gcc-12/Dockerfile → docker/compilers/gcc/Dockerfile
View File

@@ -1,48 +1,59 @@
FROM ubuntu:focal as build
FROM ubuntu:20.04 AS build
ARG DEBIAN_FRONTEND=noninteractive
ARG TARGETARCH
ARG UBUNTU_VERSION=20.04
ARG GCC_VERSION=12.3.0
ARG BUILD_VERSION=1
ARG BUILD_VERSION=2
RUN apt update && apt install -y wget build-essential file flex libz-dev libzstd-dev
RUN wget https://gcc.gnu.org/pub/gcc/releases/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz \
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
build-essential \
file \
flex \
libz-dev \
libzstd-dev \
software-properties-common \
wget \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN wget --progress=dot:giga https://gcc.gnu.org/pub/gcc/releases/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz \
&& tar xf gcc-$GCC_VERSION.tar.gz \
&& cd /gcc-$GCC_VERSION && ./contrib/download_prerequisites
RUN mkdir /${TARGETARCH}-gcc-12
WORKDIR /${TARGETARCH}-gcc-12
RUN /gcc-$GCC_VERSION/configure \
--with-pkgversion="clio-build-$BUILD_VERSION https://github.com/XRPLF/clio" \
--enable-languages=c,c++ \
--prefix=/usr \
--with-gcc-major-version-only \
--program-suffix=-12 \
--enable-shared \
--enable-linker-build-id \
--libexecdir=/usr/lib \
--without-included-gettext \
--enable-threads=posix \
--libdir=/usr/lib \
--disable-nls \
--enable-clocale=gnu \
--enable-libstdcxx-backtrace=yes \
--enable-libstdcxx-debug \
--enable-libstdcxx-time=yes \
--with-default-libstdcxx-abi=new \
--enable-gnu-unique-object \
--disable-vtable-verify \
--enable-plugin \
--enable-default-pie \
--with-system-zlib \
--enable-libphobos-checking=release \
--with-target-system-zlib=auto \
--disable-werror \
--enable-cet \
--disable-multilib \
--without-cuda-driver \
--enable-checking=release \
--with-pkgversion="clio-build-$BUILD_VERSION https://github.com/XRPLF/clio" \
--enable-languages=c,c++ \
--prefix=/usr \
--with-gcc-major-version-only \
--program-suffix=-12 \
--enable-shared \
--enable-linker-build-id \
--libexecdir=/usr/lib \
--without-included-gettext \
--enable-threads=posix \
--libdir=/usr/lib \
--disable-nls \
--enable-clocale=gnu \
--enable-libstdcxx-backtrace=yes \
--enable-libstdcxx-debug \
--enable-libstdcxx-time=yes \
--with-default-libstdcxx-abi=new \
--enable-gnu-unique-object \
--disable-vtable-verify \
--enable-plugin \
--enable-default-pie \
--with-system-zlib \
--enable-libphobos-checking=release \
--with-target-system-zlib=auto \
--disable-werror \
--enable-cet \
--disable-multilib \
--without-cuda-driver \
--enable-checking=release \
&& make -j "$(nproc)" \
&& make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION \
&& mkdir -p /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64 \
@@ -58,11 +69,16 @@ RUN mkdir /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/DEBIAN \
&& dpkg-deb --build --root-owner-group /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION /gcc12.deb
# Create final image
FROM ubuntu:focal as gcc
FROM ubuntu:20.04
COPY --from=build /gcc12.deb /
# Make gcc-12 available but also leave gcc12.deb for others to copy if needed
RUN apt update && apt-get install -y binutils libc6-dev \
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
binutils \
libc6-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& dpkg -i /gcc12.deb
RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \

3
docker/compilers/gcc/README.md Normal file
View File

@@ -0,0 +1,3 @@
# gcc compiler
This image contains gcc compiler to build <https://github.com/XRPLF/clio>.

0
docker/compilers/gcc-12/control.m4 → docker/compilers/gcc/control.m4
View File

0
docker/compilers/gcc-12/ld.so.conf → docker/compilers/gcc/ld.so.conf
View File

64
docker/tools/Dockerfile Normal file
View File

@@ -0,0 +1,64 @@
FROM ubuntu:20.04
ARG DEBIAN_FRONTEND=noninteractive
ARG TARGETARCH
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
bison \
build-essential \
cmake \
flex \
ninja-build \
software-properties-common \
wget \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /tmp
ARG CCACHE_VERSION=4.11.3
RUN wget --progress=dot:giga "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz" \
&& tar xf "ccache-${CCACHE_VERSION}.tar.gz" \
&& cd "ccache-${CCACHE_VERSION}" \
&& mkdir build \
&& cd build \
&& cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=False .. \
&& cmake --build . --target install \
&& rm -rf /tmp/* /var/tmp/*
ARG DOXYGEN_VERSION=1.12.0
RUN wget --progress=dot:giga "https://github.com/doxygen/doxygen/releases/download/Release_${DOXYGEN_VERSION//./_}/doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
&& tar xf "doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
&& cd "doxygen-${DOXYGEN_VERSION}" \
&& mkdir build \
&& cd build \
&& cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
&& cmake --build . --target install \
&& rm -rf /tmp/* /var/tmp/*
ARG CLANG_BUILD_ANALYZER_VERSION=1.6.0
RUN wget --progress=dot:giga "https://github.com/aras-p/ClangBuildAnalyzer/archive/refs/tags/v${CLANG_BUILD_ANALYZER_VERSION}.tar.gz" \
&& tar xf "v${CLANG_BUILD_ANALYZER_VERSION}.tar.gz" \
&& cd "ClangBuildAnalyzer-${CLANG_BUILD_ANALYZER_VERSION}" \
&& mkdir build \
&& cd build \
&& cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
&& cmake --build . --target install \
&& rm -rf /tmp/* /var/tmp/*
ARG GIT_CLIFF_VERSION=2.9.1
RUN wget --progress=dot:giga "https://github.com/orhun/git-cliff/releases/download/v${GIT_CLIFF_VERSION}/git-cliff-${GIT_CLIFF_VERSION}-x86_64-unknown-linux-musl.tar.gz" \
&& tar xf git-cliff-${GIT_CLIFF_VERSION}-x86_64-unknown-linux-musl.tar.gz \
&& mv git-cliff-${GIT_CLIFF_VERSION}/git-cliff /usr/local/bin/git-cliff \
&& rm -rf /tmp/* /var/tmp/*
ARG GH_VERSION=2.74.0
RUN wget --progress=dot:giga "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz" \
&& tar xf gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
&& mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/local/bin/gh \
&& rm -rf /tmp/* /var/tmp/*
WORKDIR /root

1
docs/Doxyfile
View File

@@ -22,6 +22,7 @@ WARNINGS = ${LINT}
WARN_NO_PARAMDOC = ${LINT}
WARN_IF_INCOMPLETE_DOC = ${LINT}
WARN_IF_UNDOCUMENTED = ${LINT}
WARN_AS_ERROR = ${WARN_AS_ERROR}
GENERATE_LATEX = NO
GENERATE_HTML = YES

51
docs/build-clio.md
View File

@@ -24,36 +24,36 @@ Clio requires `compiler.cppstd=20` in your Conan profile (`~/.conan/profiles/def
> [!NOTE]
> Although Clio is built using C++23, it's required to set `compiler.cppstd=20` for the time being as some of Clio's dependencies are not yet capable of building under C++23.
**Mac example**:
**Mac apple-clang 16 example**:
```text
[settings]
os=Macos
os_build=Macos
arch=armv8
arch_build=armv8
compiler=apple-clang
compiler.version=15
compiler.libcxx=libc++
build_type=Release
compiler=apple-clang
compiler.cppstd=20
compiler.libcxx=libc++
compiler.version=16
os=Macos
[conf]
tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]
tools.build:cxxflags+=["-Wno-missing-template-arg-list-after-template-kw"]
```
**Linux example**:
**Linux gcc-12 example**:
```text
[settings]
os=Linux
os_build=Linux
arch=x86_64
arch_build=x86_64
compiler=gcc
compiler.version=12
compiler.libcxx=libstdc++11
build_type=Release
compiler=gcc
compiler.cppstd=20
compiler.libcxx=libstdc++11
compiler.version=12
os=Linux
[conf]
tools.build:compiler_executables={'c': '/usr/bin/gcc-12', 'cpp': '/usr/bin/g++-12'}
```
#### Artifactory
@@ -61,7 +61,7 @@ compiler.cppstd=20
Make sure artifactory is setup with Conan.
```sh
conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev
```
Now you should be able to download the prebuilt `xrpl` package on some platforms.
@@ -81,19 +81,21 @@ Navigate to Clio's root directory and run:
```sh
mkdir build && cd build
conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True
conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True'
# You can also add -GNinja to use Ninja build system instead of Make
cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
cmake --build . --parallel 8 # or without the number if you feel extra adventurous
```
> [!TIP]
> You can omit the `-o tests=True` if you don't want to build `clio_tests`.
> You can omit the `-o '&:tests=True'` if you don't want to build `clio_tests`.
If successful, `conan install` will find the required packages and `cmake` will do the rest. You should see `clio_server` and `clio_tests` in the `build` directory (the current directory).
> [!TIP]
> To generate a Code Coverage report, include `-o coverage=True` in the `conan install` command above, along with `-o tests=True` to enable tests. After running the `cmake` commands, execute `make clio_tests-ccov`. The coverage report will be found at `clio_tests-llvm-cov/index.html`.
> To generate a Code Coverage report, include `-o '&:coverage=True'` in the `conan install` command above, along with `-o '&:tests=True'` to enable tests.
> After running the `cmake` commands, execute `make clio_tests-ccov`.
> The coverage report will be found at `clio_tests-llvm-cov/index.html`.
<!-- markdownlint-disable-line MD028 -->
@@ -106,11 +108,11 @@ The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/
To generate the API docs:
1. First, include `-o docs=True` in the conan install command. For example:
1. First, include `-o '&:docs=True'` in the conan install command. For example:
```sh
mkdir build && cd build
conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o docs=True
conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True' -o '&:docs=True'
```
2. Once that has completed successfully, run the `cmake` command and add the `--target docs` option:
@@ -134,7 +136,7 @@ It is also possible to build Clio using [Docker](https://www.docker.com/) if you
docker run -it ghcr.io/xrplf/clio-ci:latest
git clone https://github.com/XRPLF/clio
mkdir build && cd build
conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True
conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True'
cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
cmake --build . --parallel 8 # or without the number if you feel extra adventurous
```
@@ -192,10 +194,11 @@ Sometimes, during development, you need to build against a custom version of `li
The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 19.0.
Clang-tidy can be run by CMake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command:
Clang-tidy can be run by CMake when building the project.
To achieve this, you just need to provide the option `-o '&:lint=True'` for the `conan install` command:
```sh
conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=True
conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True' -o '&:lint=True'
```
By default CMake will try to find `clang-tidy` automatically in your system.

1
pre-commit-hooks/check-doxygen-docs.sh
View File

@@ -61,6 +61,7 @@ pushd ${DOCDIR} > /dev/null 2>&1
cat ${ROOT}/docs/Doxyfile | \
sed \
-e "s/\${LINT}/YES/" \
-e "s/\${WARN_AS_ERROR}/NO/" \
-e "s!\${SOURCE}!${ROOT}!" \
-e "s/\${USE_DOT}/NO/" \
-e "s/\${EXCLUDES}/impl/" \

19
src/app/ClioApplication.cpp
View File

@@ -36,6 +36,8 @@
#include "rpc/RPCEngine.hpp"
#include "rpc/WorkQueue.hpp"
#include "rpc/common/impl/HandlerProvider.hpp"
#include "util/Random.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include "util/build/Build.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
@@ -103,6 +105,10 @@ ClioApplication::run(bool const useNgWebServer)
// This is not the only io context in the application.
boost::asio::io_context ioc{threads};
// Similarly we need a context to run ETLng on
// In the future we can remove the raw ioc and use ctx instead
util::async::CoroExecutionContext ctx{threads};
// Rate limiter, to prevent abuse
auto whitelistHandler = web::dosguard::WhitelistHandler{config_};
auto const dosguardWeights = web::dosguard::Weights::make(config_);
@@ -139,14 +145,19 @@ ClioApplication::run(bool const useNgWebServer)
// The server uses the balancer to forward RPCs to a rippled node.
// The balancer itself publishes to streams (transactions_proposed and accounts_proposed)
auto balancer = [&] -> std::shared_ptr<etlng::LoadBalancerInterface> {
if (config_.get<bool>("__ng_etl"))
return etlng::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
if (config_.get<bool>("__ng_etl")) {
return etlng::LoadBalancer::makeLoadBalancer(
config_, ioc, backend, subscriptions, std::make_unique<util::MTRandomGenerator>(), ledgers
);
}
return etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
return etl::LoadBalancer::makeLoadBalancer(
config_, ioc, backend, subscriptions, std::make_unique<util::MTRandomGenerator>(), ledgers
);
}();
// ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes
auto etl = etl::ETLService::makeETLService(config_, ioc, backend, subscriptions, balancer, ledgers);
auto etl = etl::ETLService::makeETLService(config_, ioc, ctx, backend, subscriptions, balancer, ledgers);
auto workQueue = rpc::WorkQueue::makeWorkQueue(config_);
auto counters = rpc::Counters::makeCounters(workQueue);

6
src/data/AmendmentCenter.hpp
View File

@@ -139,6 +139,12 @@ struct Amendments {
REGISTER(DeepFreeze);
REGISTER(PermissionDelegation);
REGISTER(fixPayChanCancelAfter);
REGISTER(Batch);
REGISTER(PermissionedDEX);
REGISTER(SingleAssetVault);
REGISTER(TokenEscrow);
REGISTER(fixAMMv1_3);
REGISTER(fixEnforceNFTokenTrustlineV2);
// Obsolete but supported by libxrpl
REGISTER(CryptoConditionsSuite);

17
src/etl/CacheLoader.hpp
View File

@@ -26,6 +26,7 @@
#include "etl/impl/CursorFromAccountProvider.hpp"
#include "etl/impl/CursorFromDiffProvider.hpp"
#include "etl/impl/CursorFromFixDiffNumProvider.hpp"
#include "etlng/CacheLoaderInterface.hpp"
#include "util/Assert.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include "util/log/Logger.hpp"
@@ -33,6 +34,7 @@
#include <cstdint>
#include <functional>
#include <memory>
#include <utility>
namespace etl {
@@ -46,7 +48,7 @@ namespace etl {
* @tparam ExecutionContextType The type of the execution context to use
*/
template <typename ExecutionContextType = util::async::CoroExecutionContext>
class CacheLoader {
class CacheLoader : public etlng::CacheLoaderInterface {
using CacheLoaderType = impl::CacheLoaderImpl<data::LedgerCacheInterface>;
util::Logger log_{"ETL"};
@@ -67,10 +69,13 @@ public:
*/
CacheLoader(
util::config::ClioConfigDefinition const& config,
std::shared_ptr<BackendInterface> const& backend,
std::shared_ptr<BackendInterface> backend,
data::LedgerCacheInterface& cache
)
: backend_{backend}, cache_{cache}, settings_{makeCacheLoaderSettings(config)}, ctx_{settings_.numThreads}
: backend_{std::move(backend)}
, cache_{cache}
, settings_{makeCacheLoaderSettings(config)}
, ctx_{settings_.numThreads}
{
}
@@ -83,7 +88,7 @@ public:
* @param seq The sequence number to load cache for
*/
void
load(uint32_t const seq)
load(uint32_t const seq) override
{
ASSERT(not cache_.get().isFull(), "Cache must not be full. seq = {}", seq);
@@ -129,7 +134,7 @@ public:
* @brief Requests the loader to stop asap
*/
void
stop() noexcept
stop() noexcept override
{
if (loader_ != nullptr)
loader_->stop();
@@ -139,7 +144,7 @@ public:
* @brief Waits for the loader to finish background work
*/
void
wait() noexcept
wait() noexcept override
{
if (loader_ != nullptr)
loader_->wait();

92
src/etl/ETLService.cpp
View File

@@ -20,12 +20,34 @@
#include "etl/ETLService.hpp"
#include "data/BackendInterface.hpp"
#include "etl/CacheLoader.hpp"
#include "etl/CorruptionDetector.hpp"
#include "etl/ETLState.hpp"
#include "etl/LoadBalancer.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etl/SystemState.hpp"
#include "etl/impl/AmendmentBlockHandler.hpp"
#include "etl/impl/ExtractionDataPipe.hpp"
#include "etl/impl/Extractor.hpp"
#include "etl/impl/LedgerFetcher.hpp"
#include "etl/impl/LedgerLoader.hpp"
#include "etl/impl/LedgerPublisher.hpp"
#include "etl/impl/Transformer.hpp"
#include "etlng/ETLService.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "etlng/LoadBalancer.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/impl/LedgerPublisher.hpp"
#include "etlng/impl/TaskManagerProvider.hpp"
#include "etlng/impl/ext/Cache.hpp"
#include "etlng/impl/ext/Core.hpp"
#include "etlng/impl/ext/MPT.hpp"
#include "etlng/impl/ext/NFT.hpp"
#include "etlng/impl/ext/Successor.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/Assert.hpp"
#include "util/Constants.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
@@ -45,6 +67,76 @@
namespace etl {
std::shared_ptr<etlng::ETLServiceInterface>
ETLService::makeETLService(
util::config::ClioConfigDefinition const& config,
boost::asio::io_context& ioc,
util::async::AnyExecutionContext ctx,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etlng::LoadBalancerInterface> balancer,
std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
)
{
std::shared_ptr<etlng::ETLServiceInterface> ret;
if (config.get<bool>("__ng_etl")) {
ASSERT(
std::dynamic_pointer_cast<etlng::LoadBalancer>(balancer), "LoadBalancer type must be etlng::LoadBalancer"
);
auto state = std::make_shared<etl::SystemState>();
auto fetcher = std::make_shared<etl::impl::LedgerFetcher>(backend, balancer);
auto extractor = std::make_shared<etlng::impl::Extractor>(fetcher);
auto publisher = std::make_shared<etlng::impl::LedgerPublisher>(ioc, backend, subscriptions, *state);
auto cacheLoader = std::make_shared<etl::CacheLoader<>>(config, backend, backend->cache());
auto cacheUpdater = std::make_shared<etlng::impl::CacheUpdater>(backend->cache());
auto amendmentBlockHandler = std::make_shared<etlng::impl::AmendmentBlockHandler>(ctx, *state);
auto loader = std::make_shared<etlng::impl::Loader>(
backend,
etlng::impl::makeRegistry(
*state,
etlng::impl::CacheExt{cacheUpdater},
etlng::impl::CoreExt{backend},
etlng::impl::SuccessorExt{backend, backend->cache()},
etlng::impl::NFTExt{backend},
etlng::impl::MPTExt{backend}
),
amendmentBlockHandler
);
auto taskManagerProvider = std::make_shared<etlng::impl::TaskManagerProvider>(*ledgers, extractor, loader);
ret = std::make_shared<etlng::ETLService>(
ctx,
config,
backend,
balancer,
ledgers,
publisher,
cacheLoader,
cacheUpdater,
extractor,
loader, // loader itself
loader, // initial load observer
taskManagerProvider,
state
);
} else {
ASSERT(std::dynamic_pointer_cast<etl::LoadBalancer>(balancer), "LoadBalancer type must be etl::LoadBalancer");
ret = std::make_shared<etl::ETLService>(config, ioc, backend, subscriptions, balancer, ledgers);
}
// inject networkID into subscriptions, as transaction feed require it to inject CTID in response
if (auto const state = ret->getETLState(); state)
subscriptions->setNetworkID(state->networkID);
ret->run();
return ret;
}
// Database must be populated when this starts
std::optional<uint32_t>
ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)

34
src/etl/ETLService.hpp
View File

@@ -22,7 +22,6 @@
#include "data/BackendInterface.hpp"
#include "etl/CacheLoader.hpp"
#include "etl/ETLState.hpp"
#include "etl/LoadBalancer.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etl/SystemState.hpp"
#include "etl/impl/AmendmentBlockHandler.hpp"
@@ -32,12 +31,12 @@
#include "etl/impl/LedgerLoader.hpp"
#include "etl/impl/LedgerPublisher.hpp"
#include "etl/impl/Transformer.hpp"
#include "etlng/ETLService.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "etlng/LoadBalancer.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/impl/LedgerPublisher.hpp"
#include "etlng/impl/TaskManagerProvider.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/Assert.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include "util/log/Logger.hpp"
#include <boost/asio/io_context.hpp>
@@ -150,6 +149,7 @@ public:
*
* @param config The configuration to use
* @param ioc io context to run on
* @param ctx Execution context for asynchronous operations
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param balancer Load balancer to use
@@ -160,34 +160,12 @@ public:
makeETLService(
util::config::ClioConfigDefinition const& config,
boost::asio::io_context& ioc,
util::async::AnyExecutionContext ctx,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etlng::LoadBalancerInterface> balancer,
std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
)
{
std::shared_ptr<etlng::ETLServiceInterface> ret;
if (config.get<bool>("__ng_etl")) {
ASSERT(
std::dynamic_pointer_cast<etlng::LoadBalancer>(balancer),
"LoadBalancer type must be etlng::LoadBalancer"
);
ret = std::make_shared<etlng::ETLService>(config, backend, subscriptions, balancer, ledgers);
} else {
ASSERT(
std::dynamic_pointer_cast<etl::LoadBalancer>(balancer), "LoadBalancer type must be etl::LoadBalancer"
);
ret = std::make_shared<etl::ETLService>(config, ioc, backend, subscriptions, balancer, ledgers);
}
// inject networkID into subscriptions, as transaction feed require it to inject CTID in response
if (auto const state = ret->getETLState(); state)
subscriptions->setNetworkID(state->networkID);
ret->run();
return ret;
}
);
/**
* @brief Stops components and joins worker thread.

17
src/etl/LoadBalancer.cpp
View File

@@ -71,12 +71,19 @@ LoadBalancer::makeLoadBalancer(
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory
)
{
return std::make_shared<LoadBalancer>(
config, ioc, std::move(backend), std::move(subscriptions), std::move(validatedLedgers), std::move(sourceFactory)
config,
ioc,
std::move(backend),
std::move(subscriptions),
std::move(randomGenerator),
std::move(validatedLedgers),
std::move(sourceFactory)
);
}
@@ -85,10 +92,12 @@ LoadBalancer::LoadBalancer(
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory
)
: forwardingCounters_{
: randomGenerator_(std::move(randomGenerator))
, forwardingCounters_{
.successDuration = PrometheusService::counterInt(
"forwarding_duration_milliseconds_counter",
Labels({util::prometheus::Label{"status", "success"}}),
@@ -319,7 +328,7 @@ void
LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter)
{
ASSERT(not sources_.empty(), "ETL sources must be configured to execute functions.");
size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
size_t numAttempts = 0;
@@ -403,7 +412,7 @@ LoadBalancer::forwardToRippledImpl(
++forwardingCounters_.cacheMiss.get();
ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
auto numAttempts = 0u;

7
src/etl/LoadBalancer.hpp
View File

@@ -29,6 +29,7 @@
#include "rpc/Errors.hpp"
#include "util/Assert.hpp"
#include "util/Mutex.hpp"
#include "util/Random.hpp"
#include "util/ResponseExpirationCache.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
@@ -88,6 +89,8 @@ private:
std::optional<util::ResponseExpirationCache> forwardingCache_;
std::optional<std::string> forwardingXUserValue_;
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator_;
std::vector<SourcePtr> sources_;
std::optional<ETLState> etlState_;
std::uint32_t downloadRanges_ =
@@ -123,6 +126,7 @@ public:
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param randomGenerator A random generator to use for selecting sources
* @param validatedLedgers The network validated ledgers datastructure
* @param sourceFactory A factory function to create a source
*/
@@ -131,6 +135,7 @@ public:
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory = makeSource
);
@@ -142,6 +147,7 @@ public:
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param randomGenerator A random generator to use for selecting sources
* @param validatedLedgers The network validated ledgers data structure
* @param sourceFactory A factory function to create a source
* @return A shared pointer to a new instance of LoadBalancer
@@ -152,6 +158,7 @@ public:
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory = makeSource
);

12
src/etl/impl/LedgerPublisher.hpp
View File

@@ -24,6 +24,7 @@
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etl/SystemState.hpp"
#include "etlng/LedgerPublisherInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/Assert.hpp"
#include "util/log/Logger.hpp"
@@ -31,6 +32,7 @@
#include "util/prometheus/Prometheus.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/asio/post.hpp>
#include <boost/asio/strand.hpp>
#include <xrpl/basics/chrono.h>
#include <xrpl/protocol/Fees.h>
@@ -66,7 +68,7 @@ namespace etl::impl {
* includes reading all of the transactions from the database) is done from the application wide asio io_service, and a
* strand is used to ensure ledgers are published in order.
*/
class LedgerPublisher {
class LedgerPublisher : public etlng::LedgerPublisherInterface {
util::Logger log_{"ETL"};
boost::asio::strand<boost::asio::io_context::executor_type> publishStrand_;
@@ -121,7 +123,7 @@ public:
uint32_t ledgerSequence,
std::optional<uint32_t> maxAttempts,
std::chrono::steady_clock::duration attemptsDelay = std::chrono::seconds{1}
)
) override
{
LOG(log_.info()) << "Attempting to publish ledger = " << ledgerSequence;
size_t numAttempts = 0;
@@ -235,7 +237,7 @@ public:
* @brief Get time passed since last publish, in seconds
*/
std::uint32_t
lastPublishAgeSeconds() const
lastPublishAgeSeconds() const override
{
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - getLastPublish())
.count();
@@ -245,7 +247,7 @@ public:
* @brief Get last publish time as a time point
*/
std::chrono::time_point<std::chrono::system_clock>
getLastPublish() const
getLastPublish() const override
{
return std::chrono::time_point<std::chrono::system_clock>{std::chrono::seconds{lastPublishSeconds_.get().value()
}};
@@ -255,7 +257,7 @@ public:
* @brief Get time passed since last ledger close, in seconds
*/
std::uint32_t
lastCloseAgeSeconds() const
lastCloseAgeSeconds() const override
{
std::shared_lock const lck(closeTimeMtx_);
auto now = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch())

1
src/etl/impl/SourceImpl.hpp
View File

@@ -198,7 +198,6 @@ public:
*
* @param sequence Sequence of the ledger to download
* @param numMarkers Number of markers to generate for async calls
* @param cacheOnly Only insert into cache, not the DB; defaults to false
* @return A std::pair of the data and a bool indicating whether the download was successful
*/
std::pair<std::vector<std::string>, bool>

4
src/etlng/CMakeLists.txt
View File

@@ -2,7 +2,8 @@ add_library(clio_etlng)
target_sources(
clio_etlng
PRIVATE LoadBalancer.cpp
PRIVATE ETLService.cpp
LoadBalancer.cpp
Source.cpp
impl/AmendmentBlockHandler.cpp
impl/AsyncGrpcCall.cpp
@@ -14,6 +15,7 @@ target_sources(
impl/TaskManager.cpp
impl/ext/Cache.cpp
impl/ext/Core.cpp
impl/ext/MPT.cpp
impl/ext/NFT.cpp
impl/ext/Successor.cpp
)

53
src/etlng/CacheLoaderInterface.hpp Normal file
View File

@@ -0,0 +1,53 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include <cstdint>
namespace etlng {
/**
* @brief An interface for the Cache Loader
*/
struct CacheLoaderInterface {
virtual ~CacheLoaderInterface() = default;
/**
* @brief Load the cache with the most recent ledger data
*
* @param seq The sequence number of the ledger to load
*/
virtual void
load(uint32_t const seq) = 0;
/**
* @brief Stop the cache loading process
*/
virtual void
stop() noexcept = 0;
/**
* @brief Wait for all cache loading tasks to complete
*/
virtual void
wait() noexcept = 0;
};
} // namespace etlng

66
src/etlng/CacheUpdaterInterface.hpp Normal file
View File

@@ -0,0 +1,66 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include <cstdint>
#include <vector>
namespace etlng {
/**
* @brief An interface for the Cache Updater
*/
struct CacheUpdaterInterface {
virtual ~CacheUpdaterInterface() = default;
/**
* @brief Update the cache with ledger data
* @param data The ledger data to update with
*/
virtual void
update(model::LedgerData const& data) = 0;
/**
* @brief Update the cache with ledger objects at a specific sequence
* @param seq The ledger sequence number
* @param objs The ledger objects to update with
*/
virtual void
update(uint32_t seq, std::vector<data::LedgerObject> const& objs) = 0;
/**
* @brief Update the cache with model objects at a specific sequence
* @param seq The ledger sequence number
* @param objs The model objects to update with
*/
virtual void
update(uint32_t seq, std::vector<model::Object> const& objs) = 0;
/**
* @brief Mark the cache as fully loaded
*/
virtual void
setFull() = 0;
};
} // namespace etlng

265
src/etlng/ETLService.cpp Normal file
View File

@@ -0,0 +1,265 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/ETLService.hpp"
#include "data/BackendInterface.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etl/ETLState.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etl/SystemState.hpp"
#include "etl/impl/AmendmentBlockHandler.hpp"
#include "etl/impl/LedgerFetcher.hpp"
#include "etlng/CacheLoaderInterface.hpp"
#include "etlng/CacheUpdaterInterface.hpp"
#include "etlng/ExtractorInterface.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LedgerPublisherInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/LoaderInterface.hpp"
#include "etlng/MonitorInterface.hpp"
#include "etlng/TaskManagerProviderInterface.hpp"
#include "etlng/impl/AmendmentBlockHandler.hpp"
#include "etlng/impl/CacheUpdater.hpp"
#include "etlng/impl/Extraction.hpp"
#include "etlng/impl/LedgerPublisher.hpp"
#include "etlng/impl/Loading.hpp"
#include "etlng/impl/Monitor.hpp"
#include "etlng/impl/Registry.hpp"
#include "etlng/impl/Scheduling.hpp"
#include "etlng/impl/TaskManager.hpp"
#include "etlng/impl/ext/Cache.hpp"
#include "etlng/impl/ext/Core.hpp"
#include "etlng/impl/ext/NFT.hpp"
#include "etlng/impl/ext/Successor.hpp"
#include "util/Assert.hpp"
#include "util/Profiler.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include "util/log/Logger.hpp"
#include <boost/json/object.hpp>
#include <boost/signals2/connection.hpp>
#include <fmt/core.h>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <utility>
namespace etlng {
ETLService::ETLService(
util::async::AnyExecutionContext ctx,
std::reference_wrapper<util::config::ClioConfigDefinition const> config,
std::shared_ptr<data::BackendInterface> backend,
std::shared_ptr<LoadBalancerInterface> balancer,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers,
std::shared_ptr<LedgerPublisherInterface> publisher,
std::shared_ptr<CacheLoaderInterface> cacheLoader,
std::shared_ptr<CacheUpdaterInterface> cacheUpdater,
std::shared_ptr<ExtractorInterface> extractor,
std::shared_ptr<LoaderInterface> loader,
std::shared_ptr<InitialLoadObserverInterface> initialLoadObserver,
std::shared_ptr<etlng::TaskManagerProviderInterface> taskManagerProvider,
std::shared_ptr<etl::SystemState> state
)
: ctx_(std::move(ctx))
, config_(config)
, backend_(std::move(backend))
, balancer_(std::move(balancer))
, ledgers_(std::move(ledgers))
, publisher_(std::move(publisher))
, cacheLoader_(std::move(cacheLoader))
, cacheUpdater_(std::move(cacheUpdater))
, extractor_(std::move(extractor))
, loader_(std::move(loader))
, initialLoadObserver_(std::move(initialLoadObserver))
, taskManagerProvider_(std::move(taskManagerProvider))
, state_(std::move(state))
{
LOG(log_.info()) << "Creating ETLng...";
}
ETLService::~ETLService()
{
stop();
LOG(log_.debug()) << "Destroying ETLng";
}
void
ETLService::run()
{
LOG(log_.info()) << "Running ETLng...";
// TODO: write-enabled node should start in readonly and do the 10 second dance to become a writer
mainLoop_.emplace(ctx_.execute([this] {
state_->isWriting =
not state_->isReadOnly; // TODO: this is now needed because we don't have a mechanism for readonly or
// ETL writer node. remove later in favor of real mechanism
auto const rng = loadInitialLedgerIfNeeded();
LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
std::optional<uint32_t> const mostRecentValidated = ledgers_->getMostRecent();
if (not mostRecentValidated) {
LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
"Exiting monitor loop";
return;
}
ASSERT(rng.has_value(), "Ledger range can't be null");
auto const nextSequence = rng->maxSequence + 1;
LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence;
startMonitor(nextSequence);
// TODO: we only want to run the full ETL task man if we are POSSIBLY a write node
// but definitely not in strict readonly
if (not state_->isReadOnly)
startLoading(nextSequence);
}));
}
void
ETLService::stop()
{
LOG(log_.info()) << "Stop called";
if (taskMan_)
taskMan_->stop();
if (monitor_)
monitor_->stop();
}
boost::json::object
ETLService::getInfo() const
{
boost::json::object result;
result["etl_sources"] = balancer_->toJson();
result["is_writer"] = static_cast<int>(state_->isWriting);
result["read_only"] = static_cast<int>(state_->isReadOnly);
auto last = publisher_->getLastPublish();
if (last.time_since_epoch().count() != 0)
result["last_publish_age_seconds"] = std::to_string(publisher_->lastPublishAgeSeconds());
return result;
}
bool
ETLService::isAmendmentBlocked() const
{
return state_->isAmendmentBlocked;
}
bool
ETLService::isCorruptionDetected() const
{
return state_->isCorruptionDetected;
}
std::optional<etl::ETLState>
ETLService::getETLState() const
{
return balancer_->getETLState();
}
std::uint32_t
ETLService::lastCloseAgeSeconds() const
{
return publisher_->lastCloseAgeSeconds();
}
std::optional<data::LedgerRange>
ETLService::loadInitialLedgerIfNeeded()
{
auto rng = backend_->hardFetchLedgerRangeNoThrow();
if (not rng.has_value()) {
LOG(log_.info()) << "Database is empty. Will download a ledger from the network.";
LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
if (auto const mostRecentValidated = ledgers_->getMostRecent(); mostRecentValidated.has_value()) {
auto const seq = *mostRecentValidated;
LOG(log_.info()) << "Ledger " << seq << " has been validated. Downloading... ";
auto [ledger, timeDiff] = ::util::timed<std::chrono::duration<double>>([this, seq]() {
return extractor_->extractLedgerOnly(seq).and_then([this, seq](auto&& data) {
// TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar
data.edgeKeys = balancer_->loadInitialLedger(seq, *initialLoadObserver_);
// TODO: this should be interruptible for graceful shutdown
return loader_->loadInitialLedger(data);
});
});
if (not ledger.has_value()) {
LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop";
return std::nullopt;
}
LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff;
LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
return backend_->hardFetchLedgerRangeNoThrow();
}
LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
"Exiting monitor loop";
return std::nullopt;
}
LOG(log_.info()) << "Database already populated. Picking up from the tip of history";
cacheLoader_->load(rng->maxSequence);
return rng;
}
void
ETLService::startMonitor(uint32_t seq)
{
monitor_ = std::make_unique<impl::Monitor>(ctx_, backend_, ledgers_, seq);
monitorSubscription_ = monitor_->subscribe([this](uint32_t seq) {
log_.info() << "MONITOR got new seq from db: " << seq;
// FIXME: is this the best way?
if (not state_->isWriting) {
auto const diff = data::synchronousAndRetryOnTimeout([this, seq](auto yield) {
return backend_->fetchLedgerDiff(seq, yield);
});
cacheUpdater_->update(seq, diff);
}
publisher_->publish(seq, {});
});
monitor_->run();
}
void
ETLService::startLoading(uint32_t seq)
{
taskMan_ = taskManagerProvider_->make(ctx_, *monitor_, seq);
taskMan_->run(config_.get().get<std::size_t>("extractor_threads"));
}
} // namespace etlng

237
src/etlng/ETLService.hpp
View File

@@ -20,22 +20,27 @@
#pragma once
#include "data/BackendInterface.hpp"
#include "data/LedgerCache.hpp"
#include "data/Types.hpp"
#include "etl/CacheLoader.hpp"
#include "etl/ETLState.hpp"
#include "etl/LedgerFetcherInterface.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etl/SystemState.hpp"
#include "etl/impl/AmendmentBlockHandler.hpp"
#include "etl/impl/LedgerFetcher.hpp"
#include "etl/impl/LedgerPublisher.hpp"
#include "etlng/AmendmentBlockHandlerInterface.hpp"
#include "etlng/CacheLoaderInterface.hpp"
#include "etlng/CacheUpdaterInterface.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "etlng/ExtractorInterface.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LedgerPublisherInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/LoaderInterface.hpp"
#include "etlng/MonitorInterface.hpp"
#include "etlng/TaskManagerInterface.hpp"
#include "etlng/TaskManagerProviderInterface.hpp"
#include "etlng/impl/AmendmentBlockHandler.hpp"
#include "etlng/impl/CacheUpdater.hpp"
#include "etlng/impl/Extraction.hpp"
#include "etlng/impl/LedgerPublisher.hpp"
#include "etlng/impl/Loading.hpp"
#include "etlng/impl/Monitor.hpp"
#include "etlng/impl/Registry.hpp"
@@ -45,14 +50,14 @@
#include "etlng/impl/ext/Core.hpp"
#include "etlng/impl/ext/NFT.hpp"
#include "etlng/impl/ext/Successor.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/Assert.hpp"
#include "util/Profiler.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include "util/async/AnyOperation.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/json/object.hpp>
#include <boost/signals2/connection.hpp>
#include <fmt/core.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/base_uint.h>
@@ -64,15 +69,12 @@
#include <xrpl/protocol/TxFormats.h>
#include <xrpl/protocol/TxMeta.h>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <optional>
#include <ranges>
#include <stdexcept>
#include <string>
#include <tuple>
#include <utility>
namespace etlng {
@@ -92,191 +94,94 @@ namespace etlng {
class ETLService : public ETLServiceInterface {
util::Logger log_{"ETL"};
util::async::AnyExecutionContext ctx_;
std::reference_wrapper<util::config::ClioConfigDefinition const> config_;
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions_;
std::shared_ptr<etlng::LoadBalancerInterface> balancer_;
std::shared_ptr<LoadBalancerInterface> balancer_;
std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers_;
std::shared_ptr<etl::CacheLoader<>> cacheLoader_;
std::shared_ptr<etl::LedgerFetcherInterface> fetcher_;
std::shared_ptr<LedgerPublisherInterface> publisher_;
std::shared_ptr<CacheLoaderInterface> cacheLoader_;
std::shared_ptr<CacheUpdaterInterface> cacheUpdater_;
std::shared_ptr<ExtractorInterface> extractor_;
std::shared_ptr<LoaderInterface> loader_;
std::shared_ptr<InitialLoadObserverInterface> initialLoadObserver_;
std::shared_ptr<etlng::TaskManagerProviderInterface> taskManagerProvider_;
std::shared_ptr<etl::SystemState> state_;
etl::SystemState state_;
util::async::CoroExecutionContext ctx_{8};
std::unique_ptr<MonitorInterface> monitor_;
std::unique_ptr<TaskManagerInterface> taskMan_;
std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler_;
std::shared_ptr<impl::Loader> loader_;
boost::signals2::scoped_connection monitorSubscription_;
std::optional<util::async::CoroExecutionContext::Operation<void>> mainLoop_;
std::optional<util::async::AnyOperation<void>> mainLoop_;
public:
/**
* @brief Create an instance of ETLService.
*
* @param config The configuration to use
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param balancer Load balancer to use
* @param ledgers The network validated ledgers datastructure
* @param ctx The execution context for asynchronous operations
* @param config The Clio configuration definition
* @param backend Interface to the backend database
* @param balancer Load balancer for distributing work
* @param ledgers Interface for accessing network validated ledgers
* @param publisher Interface for publishing ledger data
* @param cacheLoader Interface for loading cache data
* @param cacheUpdater Interface for updating cache data
* @param extractor The extractor to use
* @param loader Interface for loading data
* @param initialLoadObserver The observer for initial data loading
* @param taskManagerProvider The provider of the task manager instance
* @param state System state tracking object
*/
ETLService(
util::config::ClioConfigDefinition const& config,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etlng::LoadBalancerInterface> balancer,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers
)
: backend_(std::move(backend))
, subscriptions_(std::move(subscriptions))
, balancer_(std::move(balancer))
, ledgers_(std::move(ledgers))
, cacheLoader_(std::make_shared<etl::CacheLoader<>>(config, backend_, backend_->cache()))
, fetcher_(std::make_shared<etl::impl::LedgerFetcher>(backend_, balancer_))
, extractor_(std::make_shared<impl::Extractor>(fetcher_))
, amendmentBlockHandler_(std::make_shared<etlng::impl::AmendmentBlockHandler>(ctx_, state_))
, loader_(std::make_shared<impl::Loader>(
backend_,
fetcher_,
impl::makeRegistry(
impl::CacheExt{backend_->cache()},
impl::CoreExt{backend_},
impl::SuccessorExt{backend_, backend_->cache()},
impl::NFTExt{backend_}
),
amendmentBlockHandler_
))
{
LOG(log_.info()) << "Creating ETLng...";
}
util::async::AnyExecutionContext ctx,
std::reference_wrapper<util::config::ClioConfigDefinition const> config,
std::shared_ptr<data::BackendInterface> backend,
std::shared_ptr<LoadBalancerInterface> balancer,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers,
std::shared_ptr<LedgerPublisherInterface> publisher,
std::shared_ptr<CacheLoaderInterface> cacheLoader,
std::shared_ptr<CacheUpdaterInterface> cacheUpdater,
std::shared_ptr<ExtractorInterface> extractor,
std::shared_ptr<LoaderInterface> loader,
std::shared_ptr<InitialLoadObserverInterface> initialLoadObserver,
std::shared_ptr<etlng::TaskManagerProviderInterface> taskManagerProvider,
std::shared_ptr<etl::SystemState> state
);
~ETLService() override
{
LOG(log_.debug()) << "Stopping ETLng";
}
~ETLService() override;
void
run() override
{
LOG(log_.info()) << "run() in ETLng...";
mainLoop_.emplace(ctx_.execute([this] {
auto const rng = loadInitialLedgerIfNeeded();
LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
std::optional<uint32_t> const mostRecentValidated = ledgers_->getMostRecent();
if (not mostRecentValidated) {
LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
"Exiting monitor loop";
return;
}
ASSERT(rng.has_value(), "Ledger range can't be null");
auto const nextSequence = rng->maxSequence + 1;
LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence;
auto scheduler = impl::makeScheduler(impl::ForwardScheduler{*ledgers_, nextSequence}
// impl::BackfillScheduler{nextSequence - 1, nextSequence - 1000},
// TODO lift limit and start with rng.minSeq
);
auto man = impl::TaskManager(ctx_, *scheduler, *extractor_, *loader_);
// TODO: figure out this: std::make_shared<impl::Monitor>(backend_, ledgers_, nextSequence)
man.run({}); // TODO: needs to be interruptible and fill out settings
}));
}
run() override;
void
stop() override
{
LOG(log_.info()) << "Stop called";
// TODO: stop the service correctly
}
stop() override;
boost::json::object
getInfo() const override
{
// TODO
return {{"ok", true}};
}
getInfo() const override;
bool
isAmendmentBlocked() const override
{
// TODO
return false;
}
isAmendmentBlocked() const override;
bool
isCorruptionDetected() const override
{
// TODO
return false;
}
isCorruptionDetected() const override;
std::optional<etl::ETLState>
getETLState() const override
{
// TODO
return std::nullopt;
}
getETLState() const override;
std::uint32_t
lastCloseAgeSeconds() const override
{
// TODO
return 0;
}
lastCloseAgeSeconds() const override;
private:
// TODO: this better be std::expected
std::optional<data::LedgerRange>
loadInitialLedgerIfNeeded()
{
if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); not rng.has_value()) {
LOG(log_.info()) << "Database is empty. Will download a ledger from the network.";
loadInitialLedgerIfNeeded();
try {
LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
if (auto const mostRecentValidated = ledgers_->getMostRecent(); mostRecentValidated.has_value()) {
auto const seq = *mostRecentValidated;
LOG(log_.info()) << "Ledger " << seq << " has been validated. Downloading... ";
void
startMonitor(uint32_t seq);
auto [ledger, timeDiff] = ::util::timed<std::chrono::duration<double>>([this, seq]() {
return extractor_->extractLedgerOnly(seq).and_then([this, seq](auto&& data) {
// TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar
data.edgeKeys = balancer_->loadInitialLedger(seq, *loader_);
// TODO: this should be interruptible for graceful shutdown
return loader_->loadInitialLedger(data);
});
});
LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff;
LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
if (ledger.has_value())
return backend_->hardFetchLedgerRangeNoThrow();
LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop";
} else {
LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
"Exiting monitor loop";
}
} catch (std::runtime_error const& e) {
LOG(log_.fatal()) << "Failed to load initial ledger: " << e.what();
amendmentBlockHandler_->notifyAmendmentBlocked();
}
} else {
LOG(log_.info()) << "Database already populated. Picking up from the tip of history";
cacheLoader_->load(rng->maxSequence);
return rng;
}
return std::nullopt;
}
void
startLoading(uint32_t seq);
};
} // namespace etlng

27
src/etlng/LedgerPublisherInterface.hpp
View File

@@ -37,13 +37,38 @@ struct LedgerPublisherInterface {
* @param seq The sequence number of the ledger
* @param maxAttempts The maximum number of attempts to publish the ledger; no limit if nullopt
* @param attemptsDelay The delay between attempts
* @return Whether the ledger was found in the database and published
*/
virtual void
virtual bool
publish(
uint32_t seq,
std::optional<uint32_t> maxAttempts,
std::chrono::steady_clock::duration attemptsDelay = std::chrono::seconds{1}
) = 0;
/**
* @brief Get last publish time as a time point
*
* @return A std::chrono::time_point representing the time of the last publish
*/
virtual std::chrono::time_point<std::chrono::system_clock>
getLastPublish() const = 0;
/**
* @brief Get time passed since last ledger close, in seconds
*
* @return The number of seconds since the last ledger close as std::uint32_t
*/
virtual std::uint32_t
lastCloseAgeSeconds() const = 0;
/**
* @brief Get time passed since last publish, in seconds
*
* @return The number of seconds since the last publish as std::uint32_t
*/
virtual std::uint32_t
lastPublishAgeSeconds() const = 0;
};
} // namespace etlng

17
src/etlng/LoadBalancer.cpp
View File

@@ -72,12 +72,19 @@ LoadBalancer::makeLoadBalancer(
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory
)
{
return std::make_shared<LoadBalancer>(
config, ioc, std::move(backend), std::move(subscriptions), std::move(validatedLedgers), std::move(sourceFactory)
config,
ioc,
std::move(backend),
std::move(subscriptions),
std::move(randomGenerator),
std::move(validatedLedgers),
std::move(sourceFactory)
);
}
@@ -86,10 +93,12 @@ LoadBalancer::LoadBalancer(
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory
)
: forwardingCounters_{
: randomGenerator_(std::move(randomGenerator))
, forwardingCounters_{
.successDuration = PrometheusService::counterInt(
"forwarding_duration_milliseconds_counter",
Labels({util::prometheus::Label{"status", "success"}}),
@@ -323,7 +332,7 @@ void
LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter)
{
ASSERT(not sources_.empty(), "ETL sources must be configured to execute functions.");
size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
size_t numAttempts = 0;
@@ -407,7 +416,7 @@ LoadBalancer::forwardToRippledImpl(
++forwardingCounters_.cacheMiss.get();
ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
auto numAttempts = 0u;

7
src/etlng/LoadBalancer.hpp
View File

@@ -29,6 +29,7 @@
#include "rpc/Errors.hpp"
#include "util/Assert.hpp"
#include "util/Mutex.hpp"
#include "util/Random.hpp"
#include "util/ResponseExpirationCache.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
@@ -88,6 +89,8 @@ private:
std::optional<util::ResponseExpirationCache> forwardingCache_;
std::optional<std::string> forwardingXUserValue_;
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator_;
std::vector<SourcePtr> sources_;
std::optional<etl::ETLState> etlState_;
std::uint32_t downloadRanges_ =
@@ -123,6 +126,7 @@ public:
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param randomGenerator A random generator to use for selecting sources
* @param validatedLedgers The network validated ledgers datastructure
* @param sourceFactory A factory function to create a source
*/
@@ -131,6 +135,7 @@ public:
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory = makeSource
);
@@ -142,6 +147,7 @@ public:
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param randomGenerator A random generator to use for selecting sources
* @param validatedLedgers The network validated ledgers datastructure
* @param sourceFactory A factory function to create a source
* @return A shared pointer to a new instance of LoadBalancer
@@ -152,6 +158,7 @@ public:
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory = makeSource
);

7
src/etlng/MonitorInterface.hpp
View File

@@ -40,6 +40,13 @@ public:
virtual ~MonitorInterface() = default;
/**
* @brief Allows the loading process to notify of a freshly committed ledger
* @param seq The ledger sequence loaded
*/
virtual void
notifyLedgerLoaded(uint32_t seq) = 0;
/**
* @brief Allows clients to get notified when a new ledger becomes available in Clio's database
*

46
src/etlng/TaskManagerInterface.hpp Normal file
View File

@@ -0,0 +1,46 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include <cstddef>
namespace etlng {
/**
* @brief An interface for the Task Manager
*/
struct TaskManagerInterface {
virtual ~TaskManagerInterface() = default;
/**
* @brief Start the task manager with specified settings
* @param numExtractors The number of extraction tasks
*/
virtual void
run(size_t numExtractors) = 0;
/**
* @brief Stop the task manager
*/
virtual void
stop() = 0;
};
} // namespace etlng

51
src/etlng/TaskManagerProviderInterface.hpp Normal file
View File

@@ -0,0 +1,51 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "etlng/MonitorInterface.hpp"
#include "etlng/TaskManagerInterface.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
namespace etlng {
/**
* @brief An interface for providing the Task Manager
*/
struct TaskManagerProviderInterface {
virtual ~TaskManagerProviderInterface() = default;
/**
* @brief Make a task manager
*
* @param ctx The async context to associate the task manager instance with
* @param monitor The monitor to notify when ledger is loaded
* @param seq The sequence to start at
* @return A unique pointer to a TaskManager implementation
*/
virtual std::unique_ptr<TaskManagerInterface>
make(util::async::AnyExecutionContext ctx, std::reference_wrapper<MonitorInterface> monitor, uint32_t seq) = 0;
};
} // namespace etlng

2
src/etlng/impl/AmendmentBlockHandler.cpp
View File

@@ -36,7 +36,7 @@ AmendmentBlockHandler::ActionType const AmendmentBlockHandler::kDEFAULT_AMENDMEN
};
AmendmentBlockHandler::AmendmentBlockHandler(
util::async::AnyExecutionContext&& ctx,
util::async::AnyExecutionContext ctx,
etl::SystemState& state,
std::chrono::steady_clock::duration interval,
ActionType action

2
src/etlng/impl/AmendmentBlockHandler.hpp
View File

@@ -50,7 +50,7 @@ public:
static ActionType const kDEFAULT_AMENDMENT_BLOCK_ACTION;
AmendmentBlockHandler(
util::async::AnyExecutionContext&& ctx,
util::async::AnyExecutionContext ctx,
etl::SystemState& state,
std::chrono::steady_clock::duration interval = std::chrono::seconds{1},
ActionType action = kDEFAULT_AMENDMENT_BLOCK_ACTION

69
src/etlng/impl/CacheUpdater.hpp Normal file
View File

@@ -0,0 +1,69 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etlng/CacheUpdaterInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <functional>
#include <vector>
namespace etlng::impl {
class CacheUpdater : public CacheUpdaterInterface {
std::reference_wrapper<data::LedgerCacheInterface> cache_;
util::Logger log_{"ETL"};
public:
CacheUpdater(data::LedgerCacheInterface& cache) : cache_{cache}
{
}
void
update(model::LedgerData const& data) override
{
cache_.get().update(data.objects, data.seq);
}
void
update(uint32_t seq, std::vector<data::LedgerObject> const& objs) override
{
cache_.get().update(objs, seq);
}
void
update(uint32_t seq, std::vector<model::Object> const& objs) override
{
cache_.get().update(objs, seq);
}
void
setFull() override
{
cache_.get().setFull();
}
};
} // namespace etlng::impl

7
src/etlng/impl/Extraction.hpp
View File

@@ -90,6 +90,13 @@ public:
{
}
Extractor(Extractor const&) = delete;
Extractor(Extractor&&) = delete;
Extractor&
operator=(Extractor const&) = delete;
Extractor&
operator=(Extractor&&) = delete;
[[nodiscard]] std::optional<model::LedgerData>
extractLedgerWithDiff(uint32_t seq) override;

286
src/etlng/impl/LedgerPublisher.hpp Normal file
View File

@@ -0,0 +1,286 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "data/Types.hpp"
#include "etl/SystemState.hpp"
#include "etlng/LedgerPublisherInterface.hpp"
#include "etlng/impl/Loading.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/Assert.hpp"
#include "util/Mutex.hpp"
#include "util/log/Logger.hpp"
#include "util/prometheus/Counter.hpp"
#include "util/prometheus/Prometheus.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/asio/post.hpp>
#include <boost/asio/strand.hpp>
#include <xrpl/basics/chrono.h>
#include <xrpl/protocol/Fees.h>
#include <xrpl/protocol/LedgerHeader.h>
#include <xrpl/protocol/SField.h>
#include <xrpl/protocol/STObject.h>
#include <xrpl/protocol/Serializer.h>
#include <algorithm>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <mutex>
#include <optional>
#include <shared_mutex>
#include <string>
#include <thread>
#include <utility>
#include <vector>
namespace etlng::impl {
/**
* @brief Publishes ledgers in a synchronized fashion.
*
* If ETL is started far behind the network, ledgers will be written and published very rapidly. Monitoring processes
* will publish ledgers as they are written. However, to publish a ledger, the monitoring process needs to read all of
* the transactions for that ledger from the database. Reading the transactions from the database requires network
* calls, which can be slow. It is imperative however that the monitoring processes keep up with the writer, else the
* monitoring processes will not be able to detect if the writer failed. Therefore, publishing each ledger (which
* includes reading all of the transactions from the database) is done from the application wide asio io_service, and a
* strand is used to ensure ledgers are published in order.
*/
class LedgerPublisher : public etlng::LedgerPublisherInterface {
util::Logger log_{"ETL"};
boost::asio::strand<boost::asio::io_context::executor_type> publishStrand_;
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions_;
std::reference_wrapper<etl::SystemState const> state_; // shared state for ETL
util::Mutex<std::chrono::time_point<ripple::NetClock>, std::shared_mutex> lastCloseTime_;
std::reference_wrapper<util::prometheus::CounterInt> lastPublishSeconds_ = PrometheusService::counterInt(
"etl_last_publish_seconds",
{},
"Seconds since epoch of the last published ledger"
);
util::Mutex<std::optional<uint32_t>, std::shared_mutex> lastPublishedSequence_;
public:
/**
* @brief Create an instance of the publisher
*/
LedgerPublisher(
boost::asio::io_context& ioc, // TODO: replace with AsyncContext shared with ETLServiceNg
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
etl::SystemState const& state
)
: publishStrand_{boost::asio::make_strand(ioc)}
, backend_{std::move(backend)}
, subscriptions_{std::move(subscriptions)}
, state_{std::cref(state)}
{
}
/**
* @brief Attempt to read the specified ledger from the database, and then publish that ledger to the ledgers
* stream.
*
* @param ledgerSequence the sequence of the ledger to publish
* @param maxAttempts the number of times to attempt to read the ledger from the database
* @param attemptsDelay the delay between attempts to read the ledger from the database
* @return Whether the ledger was found in the database and published
*/
bool
publish(
uint32_t ledgerSequence,
std::optional<uint32_t> maxAttempts,
std::chrono::steady_clock::duration attemptsDelay = std::chrono::seconds{1}
) override
{
LOG(log_.info()) << "Attempting to publish ledger = " << ledgerSequence;
size_t numAttempts = 0;
while (not state_.get().isStopping) {
auto range = backend_->hardFetchLedgerRangeNoThrow();
if (!range || range->maxSequence < ledgerSequence) {
++numAttempts;
LOG(log_.debug()) << "Trying to publish. Could not find ledger with sequence = " << ledgerSequence;
// We try maxAttempts times to publish the ledger, waiting one second in between each attempt.
if (maxAttempts && numAttempts >= maxAttempts) {
LOG(log_.debug()) << "Failed to publish ledger after " << numAttempts << " attempts.";
return false;
}
std::this_thread::sleep_for(attemptsDelay);
continue;
}
auto lgr = data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchLedgerBySequence(ledgerSequence, yield);
});
ASSERT(lgr.has_value(), "Ledger must exist in database. Ledger sequence = {}", ledgerSequence);
publish(*lgr);
return true;
}
return false;
}
/**
* @brief Publish the passed ledger asynchronously.
*
* All ledgers are published thru publishStrand_ which ensures that all publishes are performed in a serial fashion.
*
* @param lgrInfo the ledger to publish
*/
void
publish(ripple::LedgerHeader const& lgrInfo)
{
boost::asio::post(publishStrand_, [this, lgrInfo = lgrInfo]() {
LOG(log_.info()) << "Publishing ledger " << std::to_string(lgrInfo.seq);
// TODO: This should probably not be part of publisher in the future
if (not state_.get().isWriting)
backend_->updateRange(lgrInfo.seq); // This can't be unit tested atm.
setLastClose(lgrInfo.closeTime);
auto age = lastCloseAgeSeconds();
// if the ledger closed over MAX_LEDGER_AGE_SECONDS ago, assume we are still catching up and don't publish
static constexpr std::uint32_t kMAX_LEDGER_AGE_SECONDS = 600;
if (age < kMAX_LEDGER_AGE_SECONDS) {
std::optional<ripple::Fees> fees = data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchFees(lgrInfo.seq, yield);
});
ASSERT(fees.has_value(), "Fees must exist for ledger {}", lgrInfo.seq);
auto transactions = data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchAllTransactionsInLedger(lgrInfo.seq, yield);
});
auto const ledgerRange = backend_->fetchLedgerRange();
ASSERT(ledgerRange.has_value(), "Ledger range must exist");
auto const range = fmt::format("{}-{}", ledgerRange->minSequence, ledgerRange->maxSequence);
subscriptions_->pubLedger(lgrInfo, *fees, range, transactions.size());
// order with transaction index
std::ranges::sort(transactions, [](auto const& t1, auto const& t2) {
ripple::SerialIter iter1{t1.metadata.data(), t1.metadata.size()};
ripple::STObject const object1(iter1, ripple::sfMetadata);
ripple::SerialIter iter2{t2.metadata.data(), t2.metadata.size()};
ripple::STObject const object2(iter2, ripple::sfMetadata);
return object1.getFieldU32(ripple::sfTransactionIndex) <
object2.getFieldU32(ripple::sfTransactionIndex);
});
for (auto const& txAndMeta : transactions)
subscriptions_->pubTransaction(txAndMeta, lgrInfo);
subscriptions_->pubBookChanges(lgrInfo, transactions);
setLastPublishTime();
LOG(log_.info()) << "Published ledger " << lgrInfo.seq;
} else {
LOG(log_.info()) << "Skipping publishing ledger " << lgrInfo.seq;
}
});
// we track latest publish-requested seq, not necessarily already published
setLastPublishedSequence(lgrInfo.seq);
}
/**
* @brief Get time passed since last publish, in seconds
*/
std::uint32_t
lastPublishAgeSeconds() const override
{
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - getLastPublish())
.count();
}
/**
* @brief Get last publish time as a time point
*/
std::chrono::time_point<std::chrono::system_clock>
getLastPublish() const override
{
return std::chrono::time_point<std::chrono::system_clock>{std::chrono::seconds{lastPublishSeconds_.get().value()
}};
}
/**
* @brief Get time passed since last ledger close, in seconds
*/
std::uint32_t
lastCloseAgeSeconds() const override
{
auto closeTime = lastCloseTime_.lock()->time_since_epoch().count();
auto now = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
if (now < (kRIPPLE_EPOCH_START + closeTime))
return 0;
return now - (kRIPPLE_EPOCH_START + closeTime);
}
/**
* @brief Get the sequence of the last schueduled ledger to publish, Be aware that the ledger may not have been
* published to network
*/
std::optional<uint32_t>
getLastPublishedSequence() const
{
return *lastPublishedSequence_.lock();
}
private:
void
setLastClose(std::chrono::time_point<ripple::NetClock> lastCloseTime)
{
auto closeTime = lastCloseTime_.lock<std::scoped_lock>();
*closeTime = lastCloseTime;
}
void
setLastPublishTime()
{
using namespace std::chrono;
auto const nowSeconds = duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
lastPublishSeconds_.get().set(nowSeconds);
}
void
setLastPublishedSequence(std::optional<uint32_t> lastPublishedSequence)
{
auto lastPublishSeq = lastPublishedSequence_.lock();
*lastPublishSeq = lastPublishedSequence;
}
};
} // namespace etlng::impl

52
src/etlng/impl/Loading.cpp
View File

@@ -20,7 +20,6 @@
#include "etlng/impl/Loading.hpp"
#include "data/BackendInterface.hpp"
#include "etl/LedgerFetcherInterface.hpp"
#include "etl/impl/LedgerLoader.hpp"
#include "etlng/AmendmentBlockHandlerInterface.hpp"
#include "etlng/Models.hpp"
@@ -46,12 +45,10 @@ namespace etlng::impl {
Loader::Loader(
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<etl::LedgerFetcherInterface> fetcher,
std::shared_ptr<RegistryInterface> registry,
std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler
)
: backend_(std::move(backend))
, fetcher_(std::move(fetcher))
, registry_(std::move(registry))
, amendmentBlockHandler_(std::move(amendmentBlockHandler))
{
@@ -81,31 +78,44 @@ Loader::onInitialLoadGotMoreObjects(
std::optional<std::string> lastKey
)
{
LOG(log_.debug()) << "On initial load: got more objects for seq " << seq << ". size = " << data.size();
registry_->dispatchInitialObjects(
seq, data, std::move(lastKey).value_or(std::string{}) // TODO: perhaps use optional all the way to extensions?
);
try {
LOG(log_.debug()) << "On initial load: got more objects for seq " << seq << ". size = " << data.size();
registry_->dispatchInitialObjects(
seq,
data,
std::move(lastKey).value_or(std::string{}) // TODO: perhaps use optional all the way to extensions?
);
} catch (std::runtime_error const& e) {
LOG(log_.fatal()) << "Failed to load initial objects for " << seq << ": " << e.what();
amendmentBlockHandler_->notifyAmendmentBlocked();
}
}
std::optional<ripple::LedgerHeader>
Loader::loadInitialLedger(model::LedgerData const& data)
{
// check that database is actually empty
auto rng = backend_->hardFetchLedgerRangeNoThrow();
if (rng) {
ASSERT(false, "Database is not empty");
try {
// check that database is actually empty
auto rng = backend_->hardFetchLedgerRangeNoThrow();
if (rng) {
ASSERT(false, "Database is not empty");
return std::nullopt;
}
LOG(log_.debug()) << "Deserialized ledger header. " << ::util::toString(data.header);
auto seconds = ::util::timed<std::chrono::seconds>([this, &data]() { registry_->dispatchInitialData(data); });
LOG(log_.info()) << "Dispatching initial data and submitting all writes took " << seconds << " seconds.";
backend_->finishWrites(data.seq);
LOG(log_.debug()) << "Loaded initial ledger";
return {data.header};
} catch (std::runtime_error const& e) {
LOG(log_.fatal()) << "Failed to load initial ledger " << data.seq << ": " << e.what();
amendmentBlockHandler_->notifyAmendmentBlocked();
return std::nullopt;
}
LOG(log_.debug()) << "Deserialized ledger header. " << ::util::toString(data.header);
auto seconds = ::util::timed<std::chrono::seconds>([this, &data]() { registry_->dispatchInitialData(data); });
LOG(log_.info()) << "Dispatching initial data and submitting all writes took " << seconds << " seconds.";
backend_->finishWrites(data.seq);
LOG(log_.debug()) << "Loaded initial ledger";
return {data.header};
}
} // namespace etlng::impl

9
src/etlng/impl/Loading.hpp
View File

@@ -49,7 +49,6 @@ namespace etlng::impl {
class Loader : public LoaderInterface, public InitialLoadObserverInterface {
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<etl::LedgerFetcherInterface> fetcher_;
std::shared_ptr<RegistryInterface> registry_;
std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler_;
@@ -62,11 +61,17 @@ public:
Loader(
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<etl::LedgerFetcherInterface> fetcher,
std::shared_ptr<RegistryInterface> registry,
std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler
);
Loader(Loader const&) = delete;
Loader(Loader&&) = delete;
Loader&
operator=(Loader const&) = delete;
Loader&
operator=(Loader&&) = delete;
void
load(model::LedgerData const& data) override;

9
src/etlng/impl/Monitor.cpp
View File

@@ -55,6 +55,15 @@ Monitor::~Monitor()
stop();
}
// TODO: think about using signals perhaps? maybe combining with onNextSequence?
// also, how do we not double invoke or does it not matter
void
Monitor::notifyLedgerLoaded(uint32_t seq)
{
LOG(log_.debug()) << "Loader notified about newly committed ledger " << seq;
repeatedTask_->invoke(); // force-invoke immediately
};
void
Monitor::run(std::chrono::steady_clock::duration repeatInterval)
{

3
src/etlng/impl/Monitor.hpp
View File

@@ -60,6 +60,9 @@ public:
);
~Monitor() override;
void
notifyLedgerLoaded(uint32_t seq) override;
void
run(std::chrono::steady_clock::duration repeatInterval) override;

54
src/etlng/impl/Registry.hpp
View File

@@ -19,12 +19,15 @@
#pragma once
#include "etl/SystemState.hpp"
#include "etlng/Models.hpp"
#include "etlng/RegistryInterface.hpp"
#include <xrpl/protocol/TxFormats.h>
#include <concepts>
#include <cstdint>
#include <functional>
#include <string>
#include <tuple>
#include <type_traits>
@@ -88,6 +91,7 @@ concept SomeExtension = NoTwoOfKind<T> and ContainsValidHook<T>;
template <SomeExtension... Ps>
class Registry : public RegistryInterface {
std::reference_wrapper<etl::SystemState const> state_;
std::tuple<Ps...> store_;
static_assert(
@@ -101,9 +105,9 @@ class Registry : public RegistryInterface {
);
public:
explicit constexpr Registry(SomeExtension auto&&... exts)
explicit constexpr Registry(etl::SystemState const& state, SomeExtension auto&&... exts)
requires(std::is_same_v<std::decay_t<decltype(exts)>, std::decay_t<Ps>> and ...)
: store_(std::forward<Ps>(exts)...)
: state_{state}, store_(std::forward<Ps>(exts)...)
{
}
@@ -121,9 +125,8 @@ public:
// send entire batch of data at once
{
auto const expand = [&](auto& p) {
if constexpr (requires { p.onLedgerData(data); }) {
p.onLedgerData(data);
}
if constexpr (requires { p.onLedgerData(data); })
executeIfAllowed(p, [&data](auto& p) { p.onLedgerData(data); });
};
std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
@@ -134,7 +137,7 @@ public:
auto const expand = [&]<typename P>(P& p, model::Transaction const& t) {
if constexpr (requires { p.onTransaction(data.seq, t); }) {
if (std::decay_t<P>::spec::wants(t.type))
p.onTransaction(data.seq, t);
executeIfAllowed(p, [&data, &t](auto& p) { p.onTransaction(data.seq, t); });
}
};
@@ -146,9 +149,8 @@ public:
// send per object path
{
auto const expand = [&]<typename P>(P&& p, model::Object const& o) {
if constexpr (requires { p.onObject(data.seq, o); }) {
p.onObject(data.seq, o);
}
if constexpr (requires { p.onObject(data.seq, o); })
executeIfAllowed(p, [&data, &o](auto& p) { p.onObject(data.seq, o); });
};
for (auto const& obj : data.objects) {
@@ -163,9 +165,8 @@ public:
// send entire vector path
{
auto const expand = [&](auto&& p) {
if constexpr (requires { p.onInitialObjects(seq, data, lastKey); }) {
p.onInitialObjects(seq, data, lastKey);
}
if constexpr (requires { p.onInitialObjects(seq, data, lastKey); })
executeIfAllowed(p, [seq, &data, &lastKey](auto& p) { p.onInitialObjects(seq, data, lastKey); });
};
std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
@@ -174,9 +175,8 @@ public:
// send per object path
{
auto const expand = [&]<typename P>(P&& p, model::Object const& o) {
if constexpr (requires { p.onInitialObject(seq, o); }) {
p.onInitialObject(seq, o);
}
if constexpr (requires { p.onInitialObject(seq, o); })
executeIfAllowed(p, [seq, &o](auto& p) { p.onInitialObject(seq, o); });
};
for (auto const& obj : data) {
@@ -191,9 +191,8 @@ public:
// send entire batch path
{
auto const expand = [&](auto&& p) {
if constexpr (requires { p.onInitialData(data); }) {
p.onInitialData(data);
}
if constexpr (requires { p.onInitialData(data); })
executeIfAllowed(p, [&data](auto& p) { p.onInitialData(data); });
};
std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
@@ -204,7 +203,7 @@ public:
auto const expand = [&]<typename P>(P&& p, model::Transaction const& tx) {
if constexpr (requires { p.onInitialTransaction(data.seq, tx); }) {
if (std::decay_t<P>::spec::wants(tx.type))
p.onInitialTransaction(data.seq, tx);
executeIfAllowed(p, [&data, &tx](auto& p) { p.onInitialTransaction(data.seq, tx); });
}
};
@@ -213,12 +212,25 @@ public:
}
}
}
private:
void
executeIfAllowed(auto& p, auto&& fn)
{
if constexpr (requires { p.allowInReadonly(); }) {
if (state_.get().isWriting or p.allowInReadonly())
fn(p);
} else {
if (state_.get().isWriting)
fn(p);
}
}
};
static auto
makeRegistry(auto&&... exts)
makeRegistry(etl::SystemState const& state, auto&&... exts)
{
return std::make_unique<Registry<std::decay_t<decltype(exts)>...>>(std::forward<decltype(exts)>(exts)...);
return std::make_unique<Registry<std::decay_t<decltype(exts)>...>>(state, std::forward<decltype(exts)>(exts)...);
}
} // namespace etlng::impl

73
src/etlng/impl/TaskManager.cpp
View File

@@ -22,15 +22,21 @@
#include "etlng/ExtractorInterface.hpp"
#include "etlng/LoaderInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/MonitorInterface.hpp"
#include "etlng/SchedulerInterface.hpp"
#include "etlng/impl/Monitor.hpp"
#include "etlng/impl/TaskQueue.hpp"
#include "util/LedgerUtils.hpp"
#include "util/Profiler.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include "util/async/AnyOperation.hpp"
#include "util/async/AnyStrand.hpp"
#include "util/log/Logger.hpp"
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <ranges>
#include <thread>
#include <utility>
@@ -39,12 +45,19 @@
namespace etlng::impl {
TaskManager::TaskManager(
util::async::AnyExecutionContext&& ctx,
std::reference_wrapper<SchedulerInterface> scheduler,
util::async::AnyExecutionContext ctx,
std::shared_ptr<SchedulerInterface> scheduler,
std::reference_wrapper<ExtractorInterface> extractor,
std::reference_wrapper<LoaderInterface> loader
std::reference_wrapper<LoaderInterface> loader,
std::reference_wrapper<MonitorInterface> monitor,
uint32_t startSeq
)
: ctx_(std::move(ctx)), schedulers_(scheduler), extractor_(extractor), loader_(loader)
: ctx_(std::move(ctx))
, schedulers_(std::move(scheduler))
, extractor_(extractor)
, loader_(loader)
, monitor_(monitor)
, queue_({.startSeq = startSeq, .increment = 1u, .limit = kQUEUE_SIZE_LIMIT})
{
}
@@ -54,37 +67,32 @@ TaskManager::~TaskManager()
}
void
TaskManager::run(Settings settings)
TaskManager::run(std::size_t numExtractors)
{
static constexpr auto kQUEUE_SIZE_LIMIT = 2048uz;
LOG(log_.debug()) << "Starting task manager with " << numExtractors << " extractors...\n";
auto schedulingStrand = ctx_.makeStrand();
PriorityQueue queue(ctx_.makeStrand(), kQUEUE_SIZE_LIMIT);
stop();
extractors_.clear();
loaders_.clear();
LOG(log_.debug()) << "Starting task manager...\n";
extractors_.reserve(numExtractors);
for ([[maybe_unused]] auto _ : std::views::iota(0uz, numExtractors))
extractors_.push_back(spawnExtractor(queue_));
extractors_.reserve(settings.numExtractors);
for ([[maybe_unused]] auto _ : std::views::iota(0uz, settings.numExtractors))
extractors_.push_back(spawnExtractor(schedulingStrand, queue));
loaders_.reserve(settings.numLoaders);
for ([[maybe_unused]] auto _ : std::views::iota(0uz, settings.numLoaders))
loaders_.push_back(spawnLoader(queue));
wait();
LOG(log_.debug()) << "All finished in task manager..\n";
// Only one forward loader for now. Backfill to be added here later
loaders_.push_back(spawnLoader(queue_));
}
util::async::AnyOperation<void>
TaskManager::spawnExtractor(util::async::AnyStrand& strand, PriorityQueue& queue)
TaskManager::spawnExtractor(TaskQueue& queue)
{
// TODO: these values may be extracted to config later and/or need to be fine-tuned on a realistic system
static constexpr auto kDELAY_BETWEEN_ATTEMPTS = std::chrono::milliseconds{100u};
static constexpr auto kDELAY_BETWEEN_ENQUEUE_ATTEMPTS = std::chrono::milliseconds{1u};
return strand.execute([this, &queue](auto stopRequested) {
return ctx_.execute([this, &queue](auto stopRequested) {
while (not stopRequested) {
if (auto task = schedulers_.get().next(); task.has_value()) {
if (auto task = schedulers_->next(); task.has_value()) {
if (auto maybeBatch = extractor_.get().extractLedgerWithDiff(task->seq); maybeBatch.has_value()) {
LOG(log_.debug()) << "Adding data after extracting diff";
while (not queue.enqueue(*maybeBatch)) {
@@ -107,13 +115,26 @@ TaskManager::spawnExtractor(util::async::AnyStrand& strand, PriorityQueue& queue
}
util::async::AnyOperation<void>
TaskManager::spawnLoader(PriorityQueue& queue)
TaskManager::spawnLoader(TaskQueue& queue)
{
static constexpr auto kNANO_TO_SECOND = 1.0e9;
return ctx_.execute([this, &queue](auto stopRequested) {
while (not stopRequested) {
// TODO (https://github.com/XRPLF/clio/issues/66): does not tell the loader whether it's out of order or not
if (auto data = queue.dequeue(); data.has_value())
loader_.get().load(*data);
if (auto data = queue.dequeue(); data.has_value()) {
auto nanos = util::timed<std::chrono::nanoseconds>([this, data = *data] { loader_.get().load(data); });
auto const seconds = nanos / kNANO_TO_SECOND;
auto const txnCount = data->transactions.size();
auto const objCount = data->objects.size();
LOG(log_.info()) << "Wrote ledger " << data->seq << " with header: " << util::toString(data->header)
<< ". txns[" << txnCount << "]; objs[" << objCount << "]; in " << seconds
<< " seconds;"
<< " tps[" << txnCount / seconds << "], ops[" << objCount / seconds << "]";
monitor_.get().notifyLedgerLoaded(data->seq);
}
}
});
}

54
src/etlng/impl/TaskManager.hpp
View File

@@ -21,74 +21,70 @@
#include "etlng/ExtractorInterface.hpp"
#include "etlng/LoaderInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/MonitorInterface.hpp"
#include "etlng/SchedulerInterface.hpp"
#include "util/StrandedPriorityQueue.hpp"
#include "etlng/TaskManagerInterface.hpp"
#include "etlng/impl/Monitor.hpp"
#include "etlng/impl/TaskQueue.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include "util/async/AnyOperation.hpp"
#include "util/async/AnyStrand.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/protocol/TxFormats.h>
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <vector>
namespace etlng::impl {
class TaskManager {
class TaskManager : public TaskManagerInterface {
static constexpr auto kQUEUE_SIZE_LIMIT = 2048uz;
util::async::AnyExecutionContext ctx_;
std::reference_wrapper<SchedulerInterface> schedulers_;
std::shared_ptr<SchedulerInterface> schedulers_;
std::reference_wrapper<ExtractorInterface> extractor_;
std::reference_wrapper<LoaderInterface> loader_;
std::reference_wrapper<MonitorInterface> monitor_;
impl::TaskQueue queue_;
std::atomic_uint32_t nextForwardSequence_;
std::vector<util::async::AnyOperation<void>> extractors_;
std::vector<util::async::AnyOperation<void>> loaders_;
util::Logger log_{"ETL"};
struct ReverseOrderComparator {
[[nodiscard]] bool
operator()(model::LedgerData const& lhs, model::LedgerData const& rhs) const noexcept
{
return lhs.seq > rhs.seq;
}
};
public:
struct Settings {
size_t numExtractors; /**< number of extraction tasks */
size_t numLoaders; /**< number of loading tasks */
};
// reverse order loading is needed (i.e. start with oldest seq in forward fill buffer)
using PriorityQueue = util::StrandedPriorityQueue<model::LedgerData, ReverseOrderComparator>;
TaskManager(
util::async::AnyExecutionContext&& ctx,
std::reference_wrapper<SchedulerInterface> scheduler,
util::async::AnyExecutionContext ctx,
std::shared_ptr<SchedulerInterface> scheduler,
std::reference_wrapper<ExtractorInterface> extractor,
std::reference_wrapper<LoaderInterface> loader
std::reference_wrapper<LoaderInterface> loader,
std::reference_wrapper<MonitorInterface> monitor,
uint32_t startSeq
);
~TaskManager();
~TaskManager() override;
void
run(Settings settings);
run(std::size_t numExtractors) override;
void
stop();
stop() override;
private:
void
wait();
[[nodiscard]] util::async::AnyOperation<void>
spawnExtractor(util::async::AnyStrand& strand, PriorityQueue& queue);
spawnExtractor(TaskQueue& queue);
[[nodiscard]] util::async::AnyOperation<void>
spawnLoader(PriorityQueue& queue);
spawnLoader(TaskQueue& queue);
};
} // namespace etlng::impl

74
src/etlng/impl/TaskManagerProvider.hpp Normal file
View File

@@ -0,0 +1,74 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etlng/ExtractorInterface.hpp"
#include "etlng/LoaderInterface.hpp"
#include "etlng/MonitorInterface.hpp"
#include "etlng/TaskManagerInterface.hpp"
#include "etlng/TaskManagerProviderInterface.hpp"
#include "etlng/impl/Scheduling.hpp"
#include "etlng/impl/TaskManager.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include <cstdint>
#include <functional>
#include <memory>
#include <utility>
namespace etlng::impl {
/**
* @brief Implementation of the TaskManagerProvider interface
*/
class TaskManagerProvider : public TaskManagerProviderInterface {
std::reference_wrapper<etl::NetworkValidatedLedgersInterface> ledgers_;
std::shared_ptr<ExtractorInterface> extractor_;
std::shared_ptr<LoaderInterface> loader_;
public:
/**
* @brief Constructor
*
* @param ledgers Reference to ledgers
* @param extractor The extractor
* @param loader The loader
*/
TaskManagerProvider(
std::reference_wrapper<etl::NetworkValidatedLedgersInterface> ledgers,
std::shared_ptr<ExtractorInterface> extractor,
std::shared_ptr<LoaderInterface> loader
)
: ledgers_(ledgers), extractor_(std::move(extractor)), loader_(std::move(loader))
{
}
std::unique_ptr<TaskManagerInterface>
make(util::async::AnyExecutionContext ctx, std::reference_wrapper<MonitorInterface> monitor, uint32_t seq) override
{
auto scheduler = impl::makeScheduler(impl::ForwardScheduler{ledgers_, seq});
// TODO: add impl::BackfillScheduler{seq - 1, seq - 1000},
return std::make_unique<TaskManager>(std::move(ctx), std::move(scheduler), *extractor_, *loader_, monitor, seq);
}
};
} // namespace etlng::impl

96
src/util/StrandedPriorityQueue.hpp → src/etlng/impl/TaskQueue.hpp
View File

@@ -19,36 +19,58 @@
#pragma once
#include "util/async/AnyStrand.hpp"
#include "etlng/Models.hpp"
#include "util/Mutex.hpp"
#include <cstddef>
#include <functional>
#include <cstdint>
#include <optional>
#include <queue>
#include <type_traits>
#include <utility>
#include <vector>
namespace util {
namespace etlng::impl {
struct ReverseOrderComparator {
[[nodiscard]] bool
operator()(model::LedgerData const& lhs, model::LedgerData const& rhs) const noexcept
{
return lhs.seq > rhs.seq;
}
};
/**
* @brief A wrapper for std::priority_queue that serialises operations using a strand
* @brief A wrapper for std::priority_queue that serialises operations using a mutex
* @note This may be a candidate for future improvements if performance proves to be poor (e.g. use a lock free queue)
*/
template <typename T, typename Compare = std::less<T>>
class StrandedPriorityQueue {
util::async::AnyStrand strand_;
class TaskQueue {
std::size_t limit_;
std::priority_queue<T, std::vector<T>, Compare> queue_;
std::uint32_t increment_;
struct Data {
std::uint32_t expectedSequence;
std::priority_queue<model::LedgerData, std::vector<model::LedgerData>, ReverseOrderComparator> forwardLoadQueue;
Data(std::uint32_t seq) : expectedSequence(seq)
{
}
};
util::Mutex<Data> data_;
public:
struct Settings {
std::uint32_t startSeq = 0u; // sequence to start from (for dequeue)
std::uint32_t increment = 1u; // increment sequence by this value once dequeue was successful
std::optional<std::size_t> limit = std::nullopt;
};
/**
* @brief Construct a new priority queue on a strand
* @param strand The strand to use
* @brief Construct a new priority queue
* @param limit The limit of items allowed simultaneously in the queue
*/
StrandedPriorityQueue(util::async::AnyStrand&& strand, std::optional<std::size_t> limit = std::nullopt)
: strand_(std::move(strand)), limit_(limit.value_or(0uz))
explicit TaskQueue(Settings settings)
: limit_(settings.limit.value_or(0uz)), increment_(settings.increment), data_(settings.startSeq)
{
}
@@ -56,25 +78,20 @@ public:
* @brief Enqueue a new item onto the queue if space is available
* @note This function blocks until the item is attempted to be added to the queue
*
* @tparam I Type of the item to add
* @param item The item to add
* @return true if item added to the queue; false otherwise
*/
template <typename I>
[[nodiscard]] bool
enqueue(I&& item)
requires std::is_same_v<std::decay_t<I>, T>
enqueue(model::LedgerData item)
{
return strand_
.execute([&item, this] {
if (limit_ == 0uz or queue_.size() < limit_) {
queue_.push(std::forward<I>(item));
return true;
}
return false;
})
.get()
.value_or(false); // if some exception happens - failed to add
auto lock = data_.lock();
if (limit_ == 0uz or lock->forwardLoadQueue.size() < limit_) {
lock->forwardLoadQueue.push(std::move(item));
return true;
}
return false;
}
/**
@@ -82,22 +99,19 @@ public:
* @note This function blocks until the item is taken off the queue
* @return An item if available; nullopt otherwise
*/
[[nodiscard]] std::optional<T>
[[nodiscard]] std::optional<model::LedgerData>
dequeue()
{
return strand_
.execute([this] -> std::optional<T> {
std::optional<T> out;
auto lock = data_.lock();
std::optional<model::LedgerData> out;
if (not queue_.empty()) {
out.emplace(queue_.top());
queue_.pop();
}
if (not lock->forwardLoadQueue.empty() && lock->forwardLoadQueue.top().seq == lock->expectedSequence) {
out.emplace(lock->forwardLoadQueue.top());
lock->forwardLoadQueue.pop();
lock->expectedSequence += increment_;
}
return out;
})
.get()
.value_or(std::nullopt);
return out;
}
/**
@@ -109,8 +123,8 @@ public:
[[nodiscard]] bool
empty()
{
return strand_.execute([this] { return queue_.empty(); }).get().value();
return data_.lock()->forwardLoadQueue.empty();
}
};
} // namespace util
} // namespace etlng::impl

19
src/etlng/impl/ext/Cache.cpp
View File

@@ -19,41 +19,42 @@
#include "etlng/impl/ext/Cache.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "etlng/CacheUpdaterInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <memory>
#include <string>
#include <utility>
#include <vector>
namespace etlng::impl {
CacheExt::CacheExt(data::LedgerCacheInterface& cache) : cache_(cache)
CacheExt::CacheExt(std::shared_ptr<CacheUpdaterInterface> cacheUpdater) : cacheUpdater_(std::move(cacheUpdater))
{
}
void
CacheExt::onLedgerData(model::LedgerData const& data) const
CacheExt::onLedgerData(model::LedgerData const& data)
{
cache_.get().update(data.objects, data.seq);
LOG(log_.trace()) << "got data. objects cnt = " << data.objects.size();
cacheUpdater_->update(data);
}
void
CacheExt::onInitialData(model::LedgerData const& data) const
CacheExt::onInitialData(model::LedgerData const& data)
{
LOG(log_.trace()) << "got initial data. objects cnt = " << data.objects.size();
cache_.get().update(data.objects, data.seq);
cache_.get().setFull();
cacheUpdater_->update(data);
cacheUpdater_->setFull();
}
void
CacheExt::onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey)
const
{
LOG(log_.trace()) << "got initial objects cnt = " << objs.size();
cache_.get().update(objs, seq);
cacheUpdater_->update(seq, objs);
}
} // namespace etlng::impl

22
src/etlng/impl/ext/Cache.hpp
View File

@@ -19,33 +19,41 @@
#pragma once
#include "data/LedgerCacheInterface.hpp"
#include "etlng/CacheUpdaterInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/CacheUpdater.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <vector>
namespace etlng::impl {
class CacheExt {
std::reference_wrapper<data::LedgerCacheInterface> cache_;
std::shared_ptr<CacheUpdaterInterface> cacheUpdater_;
util::Logger log_{"ETL"};
public:
CacheExt(data::LedgerCacheInterface& cache);
CacheExt(std::shared_ptr<CacheUpdaterInterface> cacheUpdater);
void
onLedgerData(model::LedgerData const& data) const;
onLedgerData(model::LedgerData const& data);
void
onInitialData(model::LedgerData const& data) const;
onInitialData(model::LedgerData const& data);
void
onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey) const;
onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey);
// We want cache updates through ETL if we are a potential writer but currently are not writing to DB
static bool
allowInReadonly()
{
return true;
}
};
} // namespace etlng::impl

10
src/etlng/impl/ext/Core.cpp
View File

@@ -34,7 +34,7 @@ CoreExt::CoreExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move
}
void
CoreExt::onLedgerData(model::LedgerData const& data) const
CoreExt::onLedgerData(model::LedgerData const& data)
{
LOG(log_.debug()) << "Loading ledger data for " << data.seq;
backend_->writeLedger(data.header, auto{data.rawHeader});
@@ -42,7 +42,7 @@ CoreExt::onLedgerData(model::LedgerData const& data) const
}
void
CoreExt::onInitialData(model::LedgerData const& data) const
CoreExt::onInitialData(model::LedgerData const& data)
{
LOG(log_.info()) << "Loading initial ledger data for " << data.seq;
backend_->writeLedger(data.header, auto{data.rawHeader});
@@ -50,21 +50,21 @@ CoreExt::onInitialData(model::LedgerData const& data) const
}
void
CoreExt::onInitialObject(uint32_t seq, model::Object const& obj) const
CoreExt::onInitialObject(uint32_t seq, model::Object const& obj)
{
LOG(log_.trace()) << "got initial OBJ = " << obj.key << " for seq " << seq;
backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
}
void
CoreExt::onObject(uint32_t seq, model::Object const& obj) const
CoreExt::onObject(uint32_t seq, model::Object const& obj)
{
LOG(log_.trace()) << "got OBJ = " << obj.key << " for seq " << seq;
backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
}
void
CoreExt::insertTransactions(model::LedgerData const& data) const
CoreExt::insertTransactions(model::LedgerData const& data)
{
for (auto const& txn : data.transactions) {
LOG(log_.trace()) << "Inserting transaction = " << txn.sttx.getTransactionID();

10
src/etlng/impl/ext/Core.hpp
View File

@@ -39,20 +39,20 @@ public:
CoreExt(std::shared_ptr<BackendInterface> backend);
void
onLedgerData(model::LedgerData const& data) const;
onLedgerData(model::LedgerData const& data);
void
onInitialData(model::LedgerData const& data) const;
onInitialData(model::LedgerData const& data);
void
onInitialObject(uint32_t seq, model::Object const& obj) const;
onInitialObject(uint32_t seq, model::Object const& obj);
void
onObject(uint32_t seq, model::Object const& obj) const;
onObject(uint32_t seq, model::Object const& obj);
private:
void
insertTransactions(model::LedgerData const& data) const;
insertTransactions(model::LedgerData const& data);
};
} // namespace etlng::impl

78
src/etlng/impl/ext/MPT.cpp Normal file
View File

@@ -0,0 +1,78 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ext/MPT.hpp"
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "etl/MPTHelpers.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/basics/strHex.h>
#include <cstdint>
#include <memory>
#include <optional>
#include <utility>
#include <vector>
namespace etlng::impl {
MPTExt::MPTExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(backend))
{
}
void
MPTExt::onLedgerData(model::LedgerData const& data)
{
LOG(log_.trace()) << "got TXS cnt = " << data.transactions.size() << "; OBJS size = " << data.objects.size();
writeMPTHoldersFromTransactions(data);
}
void
MPTExt::onInitialObject(uint32_t, model::Object const& obj)
{
LOG(log_.trace()) << "got initial object with key: " << ripple::strHex(obj.key);
if (auto const mptHolder = etl::getMPTHolderFromObj(obj.keyRaw, obj.dataRaw); mptHolder.has_value())
backend_->writeMPTHolders({*mptHolder});
}
void
MPTExt::onInitialData(model::LedgerData const& data)
{
LOG(log_.trace()) << "got initial TXS cnt = " << data.transactions.size();
writeMPTHoldersFromTransactions(data);
}
void
MPTExt::writeMPTHoldersFromTransactions(model::LedgerData const& data)
{
std::vector<MPTHolderData> holders;
for (auto const& tx : data.transactions) {
if (auto const mptHolder = etl::getMPTHolderFromTx(tx.meta, tx.sttx); mptHolder.has_value())
holders.push_back(*mptHolder);
}
if (not holders.empty())
backend_->writeMPTHolders(holders);
}
} // namespace etlng::impl

57
src/etlng/impl/ext/MPT.hpp Normal file
View File

@@ -0,0 +1,57 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/basics/strHex.h>
#include <xrpl/protocol/AccountID.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/TxMeta.h>
#include <cstdint>
#include <memory>
namespace etlng::impl {
class MPTExt {
std::shared_ptr<BackendInterface> backend_;
util::Logger log_{"ETL"};
public:
explicit MPTExt(std::shared_ptr<BackendInterface> backend);
void
onLedgerData(model::LedgerData const& data);
void
onInitialObject(uint32_t seq, model::Object const& obj);
void
onInitialData(model::LedgerData const& data);
private:
void
writeMPTHoldersFromTransactions(model::LedgerData const& data);
};
} // namespace etlng::impl

9
src/etlng/impl/ext/NFT.cpp
View File

@@ -37,27 +37,28 @@ NFTExt::NFTExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(b
}
void
NFTExt::onLedgerData(model::LedgerData const& data) const
NFTExt::onLedgerData(model::LedgerData const& data)
{
LOG(log_.trace()) << "got TXS cnt = " << data.transactions.size() << "; OBJS size = " << data.objects.size();
writeNFTs(data);
}
void
NFTExt::onInitialObject(uint32_t seq, model::Object const& obj) const
NFTExt::onInitialObject(uint32_t seq, model::Object const& obj)
{
LOG(log_.trace()) << "got initial object with key = " << obj.key;
backend_->writeNFTs(etl::getNFTDataFromObj(seq, obj.keyRaw, obj.dataRaw));
}
void
NFTExt::onInitialData(model::LedgerData const& data) const
NFTExt::onInitialData(model::LedgerData const& data)
{
LOG(log_.trace()) << "got initial TXS cnt = " << data.transactions.size();
writeNFTs(data);
}
void
NFTExt::writeNFTs(model::LedgerData const& data) const
NFTExt::writeNFTs(model::LedgerData const& data)
{
std::vector<NFTsData> nfts;
std::vector<NFTTransactionsData> nftTxs;

8
src/etlng/impl/ext/NFT.hpp
View File

@@ -36,17 +36,17 @@ public:
NFTExt(std::shared_ptr<BackendInterface> backend);
void
onLedgerData(model::LedgerData const& data) const;
onLedgerData(model::LedgerData const& data);
void
onInitialObject(uint32_t seq, model::Object const& obj) const;
onInitialObject(uint32_t seq, model::Object const& obj);
void
onInitialData(model::LedgerData const& data) const;
onInitialData(model::LedgerData const& data);
private:
void
writeNFTs(model::LedgerData const& data) const;
writeNFTs(model::LedgerData const& data);
};
} // namespace etlng::impl

3
src/feed/impl/TransactionFeed.cpp
View File

@@ -277,7 +277,8 @@ TransactionFeed::pub(
// determine the OrderBook
ripple::Book const book{
data->getFieldAmount(ripple::sfTakerGets).issue(),
data->getFieldAmount(ripple::sfTakerPays).issue()
data->getFieldAmount(ripple::sfTakerPays).issue(),
(*data)[~ripple::sfDomainID]
};
if (affectedBooks.find(book) == affectedBooks.end()) {
affectedBooks.insert(book);

11
src/rpc/BookChangesHelper.hpp
View File

@@ -62,6 +62,7 @@ struct BookChange {
ripple::STAmount lowRate;
ripple::STAmount openRate;
ripple::STAmount closeRate;
std::optional<ripple::uint256> domain;
};
/**
@@ -145,11 +146,15 @@ private:
auto const deltaPays =
finalFields.getFieldAmount(ripple::sfTakerPays) - previousFields.getFieldAmount(ripple::sfTakerPays);
transformAndStore(deltaGets, deltaPays);
transformAndStore(deltaGets, deltaPays, finalFields[~ripple::sfDomainID]);
}
void
transformAndStore(ripple::STAmount const& deltaGets, ripple::STAmount const& deltaPays)
transformAndStore(
ripple::STAmount const& deltaGets,
ripple::STAmount const& deltaPays,
std::optional<ripple::uint256> const& domain
)
{
auto const g = to_string(deltaGets.issue());
auto const p = to_string(deltaPays.issue());
@@ -189,6 +194,7 @@ private:
entry.lowRate = rate;
entry.closeRate = rate;
entry.domain = domain;
} else {
tally_[key] = {
.sideAVolume = first,
@@ -197,6 +203,7 @@ private:
.lowRate = rate,
.openRate = rate,
.closeRate = rate,
.domain = domain,
};
}
}

1
src/rpc/RPCCenter.cpp
View File

@@ -85,6 +85,7 @@ forwardedRpcs()
"channel_authorize",
"channel_verify",
"simulate",
"batch"
};
return kFORWARDED_RPCS;
}

24
src/rpc/RPCEngine.hpp
View File

@@ -163,19 +163,17 @@ public:
[this, &ctx](boost::asio::yield_context
) -> std::expected<util::ResponseExpirationCache::EntryData, util::ResponseExpirationCache::Error> {
auto result = buildResponseImpl(ctx);
auto extracted = std::visit(
util::OverloadSet{
[&result](Status status
) -> std::expected<boost::json::object, util::ResponseExpirationCache::Error> {
return std::unexpected{util::ResponseExpirationCache::Error{
.status = std::move(status), .warnings = std::move(result.warnings)
}};
},
[](boost::json::object obj
) -> std::expected<boost::json::object, util::ResponseExpirationCache::Error> { return obj; }
},
std::move(result.response)
);
auto const extracted =
[&result]() -> std::expected<boost::json::object, util::ResponseExpirationCache::Error> {
if (result.response.has_value()) {
return std::move(result.response).value();
}
return std::unexpected{util::ResponseExpirationCache::Error{
.status = std::move(result.response).error(), .warnings = std::move(result.warnings)
}};
}();
if (extracted.has_value()) {
return util::ResponseExpirationCache::EntryData{
.lastUpdated = std::chrono::steady_clock::now(), .response = std::move(extracted).value()

169
src/rpc/RPCHelpers.cpp
View File

@@ -99,7 +99,6 @@
#include <string>
#include <string_view>
#include <utility>
#include <variant>
#include <vector>
// local to compilation unit loggers
@@ -467,23 +466,23 @@ parseStringAsUInt(std::string const& value)
return index;
}
std::variant<Status, ripple::LedgerHeader>
std::expected<ripple::LedgerHeader, Status>
ledgerHeaderFromRequest(std::shared_ptr<data::BackendInterface const> const& backend, web::Context const& ctx)
{
auto hashValue = ctx.params.contains("ledger_hash") ? ctx.params.at("ledger_hash") : nullptr;
if (!hashValue.is_null()) {
if (!hashValue.is_string())
return Status{RippledError::rpcINVALID_PARAMS, "ledgerHashNotString"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "ledgerHashNotString"}};
ripple::uint256 ledgerHash;
if (!ledgerHash.parseHex(boost::json::value_to<std::string>(hashValue)))
return Status{RippledError::rpcINVALID_PARAMS, "ledgerHashMalformed"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "ledgerHashMalformed"}};
auto lgrInfo = backend->fetchLedgerByHash(ledgerHash, ctx.yield);
if (!lgrInfo || lgrInfo->seq > ctx.range.maxSequence)
return Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"};
return std::unexpected{Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"}};
return *lgrInfo;
}
@@ -507,18 +506,18 @@ ledgerHeaderFromRequest(std::shared_ptr<data::BackendInterface const> const& bac
}
if (!ledgerSequence)
return Status{RippledError::rpcINVALID_PARAMS, "ledgerIndexMalformed"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "ledgerIndexMalformed"}};
auto lgrInfo = backend->fetchLedgerBySequence(*ledgerSequence, ctx.yield);
if (!lgrInfo || lgrInfo->seq > ctx.range.maxSequence)
return Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"};
return std::unexpected{Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"}};
return *lgrInfo;
}
// extract ledgerHeaderFromRequest's parameter from context
std::variant<Status, ripple::LedgerHeader>
std::expected<ripple::LedgerHeader, Status>
getLedgerHeaderFromHashOrSeq(
BackendInterface const& backend,
boost::asio::yield_context yield,
@@ -528,7 +527,7 @@ getLedgerHeaderFromHashOrSeq(
)
{
std::optional<ripple::LedgerHeader> lgrInfo;
auto err = Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"};
auto const err = std::unexpected{Status{RippledError::rpcLGR_NOT_FOUND, "ledgerNotFound"}};
if (ledgerHash) {
// invoke uint256's constructor to parse the hex string , instead of
// copying buffer
@@ -589,7 +588,7 @@ getStartHint(ripple::SLE const& sle, ripple::AccountID const& accountID)
// traverse account's nfts
// return Status if error occurs
// return [nextpage, count of nft already found] if success
std::variant<Status, AccountCursor>
std::expected<AccountCursor, Status>
traverseNFTObjects(
BackendInterface const& backend,
std::uint32_t sequence,
@@ -605,7 +604,7 @@ traverseNFTObjects(
// check if nextPage is valid
if (nextPage != beast::zero and firstNFTPage.key != (nextPage & ~ripple::nft::pageMask))
return Status{RippledError::rpcINVALID_PARAMS, "Invalid marker."};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "Invalid marker."}};
// no marker, start from the last page
ripple::uint256 const currentPage = nextPage == beast::zero ? lastNFTPage.key : nextPage;
@@ -618,7 +617,7 @@ traverseNFTObjects(
return AccountCursor{.index = beast::zero, .hint = 0};
}
// marker is in the right range, but still invalid
return Status{RippledError::rpcINVALID_PARAMS, "Invalid marker."};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "Invalid marker."}};
}
// the object exists and the key is in right range, must be nft page
@@ -641,7 +640,7 @@ traverseNFTObjects(
return AccountCursor{.index = beast::zero, .hint = 0};
}
std::variant<Status, AccountCursor>
std::expected<AccountCursor, Status>
traverseOwnedNodes(
BackendInterface const& backend,
ripple::AccountID const& accountID,
@@ -656,7 +655,7 @@ traverseOwnedNodes(
auto const maybeCursor = parseAccountCursor(jsonCursor);
if (!maybeCursor)
return Status{RippledError::rpcINVALID_PARAMS, "Malformed cursor."};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "Malformed cursor."}};
// the format is checked in RPC framework level
auto [hexCursor, startHint] = *maybeCursor;
@@ -670,10 +669,10 @@ traverseOwnedNodes(
if (nftIncluded and (!jsonCursor or isNftMarkerNonZero)) {
auto const cursorMaybe = traverseNFTObjects(backend, sequence, accountID, hexCursor, limit, yield, atOwnedNode);
if (auto const status = std::get_if<Status>(&cursorMaybe))
return *status;
if (!cursorMaybe.has_value())
return cursorMaybe;
auto const [nextNFTPage, nftsCount] = std::get<AccountCursor>(cursorMaybe);
auto const [nextNFTPage, nftsCount] = cursorMaybe.value();
// if limit reach , we return the next page and max as marker
if (nftsCount >= limit)
@@ -694,7 +693,7 @@ traverseOwnedNodes(
);
}
std::variant<Status, AccountCursor>
std::expected<AccountCursor, Status>
traverseOwnedNodes(
BackendInterface const& backend,
ripple::Keylet const& owner,
@@ -728,7 +727,7 @@ traverseOwnedNodes(
auto hintDir = backend.fetchLedgerObject(hintIndex.key, sequence, yield);
if (!hintDir)
return Status(ripple::rpcINVALID_PARAMS, "Invalid marker.");
return std::unexpected{Status(ripple::rpcINVALID_PARAMS, "Invalid marker.")};
ripple::SerialIter hintDirIt{hintDir->data(), hintDir->size()};
ripple::SLE const hintDirSle{hintDirIt, hintIndex.key};
@@ -736,7 +735,7 @@ traverseOwnedNodes(
if (auto const& indexes = hintDirSle.getFieldV256(ripple::sfIndexes);
std::ranges::find(indexes, hexMarker) == std::end(indexes)) {
// the index specified by marker is not in the page specified by marker
return Status(ripple::rpcINVALID_PARAMS, "Invalid marker.");
return std::unexpected{Status(ripple::rpcINVALID_PARAMS, "Invalid marker.")};
}
currentIndex = hintIndex;
@@ -745,7 +744,7 @@ traverseOwnedNodes(
auto const ownerDir = backend.fetchLedgerObject(currentIndex.key, sequence, yield);
if (!ownerDir)
return Status(ripple::rpcINVALID_PARAMS, "Owner directory not found.");
return std::unexpected{Status(ripple::rpcINVALID_PARAMS, "Owner directory not found.")};
ripple::SerialIter ownedDirIt{ownerDir->data(), ownerDir->size()};
ripple::SLE const ownedDirSle{ownedDirIt, currentIndex.key};
@@ -1309,120 +1308,140 @@ postProcessOrderBook(
}
// get book via currency type
std::variant<Status, ripple::Book>
parseBook(ripple::Currency pays, ripple::AccountID payIssuer, ripple::Currency gets, ripple::AccountID getIssuer)
std::expected<ripple::Book, Status>
parseBook(
ripple::Currency pays,
ripple::AccountID payIssuer,
ripple::Currency gets,
ripple::AccountID getIssuer,
std::optional<std::string> const& domain
)
{
if (isXRP(pays) && !isXRP(payIssuer)) {
return Status{
return std::unexpected{Status{
RippledError::rpcSRC_ISR_MALFORMED, "Unneeded field 'taker_pays.issuer' for XRP currency specification."
};
}};
}
if (!isXRP(pays) && isXRP(payIssuer)) {
return Status{
RippledError::rpcSRC_ISR_MALFORMED, "Invalid field 'taker_pays.issuer', expected non-XRP issuer."
return std::unexpected{
Status{RippledError::rpcSRC_ISR_MALFORMED, "Invalid field 'taker_pays.issuer', expected non-XRP issuer."}
};
}
if (ripple::isXRP(gets) && !ripple::isXRP(getIssuer)) {
return Status{
return std::unexpected{Status{
RippledError::rpcDST_ISR_MALFORMED, "Unneeded field 'taker_gets.issuer' for XRP currency specification."
};
}};
}
if (!ripple::isXRP(gets) && ripple::isXRP(getIssuer)) {
return Status{
RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', expected non-XRP issuer."
return std::unexpected{
Status{RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', expected non-XRP issuer."}
};
}
if (pays == gets && payIssuer == getIssuer)
return Status{RippledError::rpcBAD_MARKET, "badMarket"};
return std::unexpected{Status{RippledError::rpcBAD_MARKET, "badMarket"}};
return ripple::Book{{pays, payIssuer}, {gets, getIssuer}};
std::optional<ripple::uint256> domainID = std::nullopt;
if (domain.has_value()) {
ripple::uint256 dom;
if (!dom.parseHex(*domain))
return std::unexpected{Status{RippledError::rpcDOMAIN_MALFORMED}};
domainID = dom;
}
return ripple::Book{{pays, payIssuer}, {gets, getIssuer}, domainID};
}
std::variant<Status, ripple::Book>
std::expected<ripple::Book, Status>
parseBook(boost::json::object const& request)
{
if (!request.contains("taker_pays"))
return Status{RippledError::rpcINVALID_PARAMS, "Missing field 'taker_pays'"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "Missing field 'taker_pays'"}};
if (!request.contains("taker_gets"))
return Status{RippledError::rpcINVALID_PARAMS, "Missing field 'taker_gets'"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "Missing field 'taker_gets'"}};
if (!request.at("taker_pays").is_object())
return Status{RippledError::rpcINVALID_PARAMS, "Field 'taker_pays' is not an object"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "Field 'taker_pays' is not an object"}};
if (!request.at("taker_gets").is_object())
return Status{RippledError::rpcINVALID_PARAMS, "Field 'taker_gets' is not an object"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "Field 'taker_gets' is not an object"}};
auto takerPays = request.at("taker_pays").as_object();
if (!takerPays.contains("currency"))
return Status{RippledError::rpcSRC_CUR_MALFORMED};
return std::unexpected{Status{RippledError::rpcSRC_CUR_MALFORMED}};
if (!takerPays.at("currency").is_string())
return Status{RippledError::rpcSRC_CUR_MALFORMED};
return std::unexpected{Status{RippledError::rpcSRC_CUR_MALFORMED}};
auto takerGets = request.at("taker_gets").as_object();
if (!takerGets.contains("currency"))
return Status{RippledError::rpcDST_AMT_MALFORMED};
return std::unexpected{Status{RippledError::rpcDST_AMT_MALFORMED}};
if (!takerGets.at("currency").is_string()) {
return Status{
return std::unexpected{Status{
RippledError::rpcDST_AMT_MALFORMED,
};
}};
}
if (request.contains("domain") && !request.at("domain").is_string())
return std::unexpected{Status{RippledError::rpcDOMAIN_MALFORMED}};
ripple::Currency payCurrency;
if (!ripple::to_currency(payCurrency, boost::json::value_to<std::string>(takerPays.at("currency"))))
return Status{RippledError::rpcSRC_CUR_MALFORMED};
return std::unexpected{Status{RippledError::rpcSRC_CUR_MALFORMED}};
ripple::Currency getCurrency;
if (!ripple::to_currency(getCurrency, boost::json::value_to<std::string>(takerGets["currency"])))
return Status{RippledError::rpcDST_AMT_MALFORMED};
return std::unexpected{Status{RippledError::rpcDST_AMT_MALFORMED}};
ripple::AccountID payIssuer;
if (takerPays.contains("issuer")) {
if (!takerPays.at("issuer").is_string())
return Status{RippledError::rpcINVALID_PARAMS, "takerPaysIssuerNotString"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "takerPaysIssuerNotString"}};
if (!ripple::to_issuer(payIssuer, boost::json::value_to<std::string>(takerPays.at("issuer"))))
return Status{RippledError::rpcSRC_ISR_MALFORMED};
return std::unexpected{Status{RippledError::rpcSRC_ISR_MALFORMED}};
if (payIssuer == ripple::noAccount())
return Status{RippledError::rpcSRC_ISR_MALFORMED};
return std::unexpected{Status{RippledError::rpcSRC_ISR_MALFORMED}};
} else {
payIssuer = ripple::xrpAccount();
}
if (isXRP(payCurrency) && !isXRP(payIssuer)) {
return Status{
return std::unexpected{Status{
RippledError::rpcSRC_ISR_MALFORMED, "Unneeded field 'taker_pays.issuer' for XRP currency specification."
};
}};
}
if (!isXRP(payCurrency) && isXRP(payIssuer)) {
return Status{
RippledError::rpcSRC_ISR_MALFORMED, "Invalid field 'taker_pays.issuer', expected non-XRP issuer."
return std::unexpected{
Status{RippledError::rpcSRC_ISR_MALFORMED, "Invalid field 'taker_pays.issuer', expected non-XRP issuer."}
};
}
if ((!isXRP(payCurrency)) && (!takerPays.contains("issuer")))
return Status{RippledError::rpcSRC_ISR_MALFORMED, "Missing non-XRP issuer."};
return std::unexpected{Status{RippledError::rpcSRC_ISR_MALFORMED, "Missing non-XRP issuer."}};
ripple::AccountID getIssuer;
if (takerGets.contains("issuer")) {
if (!takerGets["issuer"].is_string())
return Status{RippledError::rpcINVALID_PARAMS, "taker_gets.issuer should be string"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "taker_gets.issuer should be string"}};
if (!ripple::to_issuer(getIssuer, boost::json::value_to<std::string>(takerGets.at("issuer"))))
return Status{RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', bad issuer."};
if (!ripple::to_issuer(getIssuer, boost::json::value_to<std::string>(takerGets.at("issuer")))) {
return std::unexpected{
Status{RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', bad issuer."}
};
}
if (getIssuer == ripple::noAccount()) {
return Status{
RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', bad issuer account one."
return std::unexpected{
Status{RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', bad issuer account one."}
};
}
} else {
@@ -1430,34 +1449,42 @@ parseBook(boost::json::object const& request)
}
if (ripple::isXRP(getCurrency) && !ripple::isXRP(getIssuer)) {
return Status{
return std::unexpected{Status{
RippledError::rpcDST_ISR_MALFORMED, "Unneeded field 'taker_gets.issuer' for XRP currency specification."
};
}};
}
if (!ripple::isXRP(getCurrency) && ripple::isXRP(getIssuer)) {
return Status{
RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', expected non-XRP issuer."
return std::unexpected{
Status{RippledError::rpcDST_ISR_MALFORMED, "Invalid field 'taker_gets.issuer', expected non-XRP issuer."}
};
}
if (payCurrency == getCurrency && payIssuer == getIssuer)
return Status{RippledError::rpcBAD_MARKET, "badMarket"};
return std::unexpected{Status{RippledError::rpcBAD_MARKET, "badMarket"}};
return ripple::Book{{payCurrency, payIssuer}, {getCurrency, getIssuer}};
std::optional<ripple::uint256> domainID;
if (request.contains("domain")) {
ripple::uint256 dom;
if (!dom.parseHex(boost::json::value_to<std::string>(request.at("domain"))))
return std::unexpected{Status{RippledError::rpcDOMAIN_MALFORMED}};
domainID = dom;
}
return ripple::Book{{payCurrency, payIssuer}, {getCurrency, getIssuer}, domainID};
}
std::variant<Status, ripple::AccountID>
std::expected<ripple::AccountID, Status>
parseTaker(boost::json::value const& taker)
{
std::optional<ripple::AccountID> takerID = {};
if (!taker.is_string())
return {Status{RippledError::rpcINVALID_PARAMS, "takerNotString"}};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "takerNotString"}};
takerID = accountFromStringStrict(boost::json::value_to<std::string>(taker));
if (!takerID)
return Status{RippledError::rpcBAD_ISSUER, "invalidTakerAccount"};
return std::unexpected{Status{RippledError::rpcBAD_ISSUER, "invalidTakerAccount"}};
return *takerID;
}
@@ -1511,18 +1538,18 @@ isAdminCmd(std::string const& method, boost::json::object const& request)
return false;
}
std::variant<ripple::uint256, Status>
std::expected<ripple::uint256, Status>
getNFTID(boost::json::object const& request)
{
if (!request.contains(JS(nft_id)))
return Status{RippledError::rpcINVALID_PARAMS, "missingTokenID"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "missingTokenID"}};
if (!request.at(JS(nft_id)).is_string())
return Status{RippledError::rpcINVALID_PARAMS, "tokenIDNotString"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "tokenIDNotString"}};
ripple::uint256 tokenid;
if (!tokenid.parseHex(boost::json::value_to<std::string>(request.at(JS(nft_id)))))
return Status{RippledError::rpcINVALID_PARAMS, "malformedTokenID"};
return std::unexpected{Status{RippledError::rpcINVALID_PARAMS, "malformedTokenID"}};
return tokenid;
}

28
src/rpc/RPCHelpers.hpp
View File

@@ -73,7 +73,6 @@
#include <string>
#include <tuple>
#include <utility>
#include <variant>
#include <vector>
namespace rpc {
@@ -281,7 +280,7 @@ generatePubLedgerMessage(
* @param ctx The context of the request
* @return The ledger info or an error status
*/
std::variant<Status, ripple::LedgerHeader>
std::expected<ripple::LedgerHeader, Status>
ledgerHeaderFromRequest(std::shared_ptr<data::BackendInterface const> const& backend, web::Context const& ctx);
/**
@@ -294,7 +293,7 @@ ledgerHeaderFromRequest(std::shared_ptr<data::BackendInterface const> const& bac
* @param maxSeq The maximum sequence to search
* @return The ledger info or an error status
*/
std::variant<Status, ripple::LedgerHeader>
std::expected<ripple::LedgerHeader, Status>
getLedgerHeaderFromHashOrSeq(
BackendInterface const& backend,
boost::asio::yield_context yield,
@@ -316,7 +315,7 @@ getLedgerHeaderFromHashOrSeq(
* @param atOwnedNode The function to call for each owned node
* @return The status or the account cursor
*/
std::variant<Status, AccountCursor>
std::expected<AccountCursor, Status>
traverseOwnedNodes(
BackendInterface const& backend,
ripple::Keylet const& owner,
@@ -343,7 +342,7 @@ traverseOwnedNodes(
* @param nftIncluded Whether to include NFTs
* @return The status or the account cursor
*/
std::variant<Status, AccountCursor>
std::expected<AccountCursor, Status>
traverseOwnedNodes(
BackendInterface const& backend,
ripple::AccountID const& accountID,
@@ -626,10 +625,17 @@ postProcessOrderBook(
* @param payIssuer The issuer of the currency to pay
* @param gets The currency to get
* @param getIssuer The issuer of the currency to get
* @param domain The domain
* @return The book or an error status
*/
std::variant<Status, ripple::Book>
parseBook(ripple::Currency pays, ripple::AccountID payIssuer, ripple::Currency gets, ripple::AccountID getIssuer);
std::expected<ripple::Book, Status>
parseBook(
ripple::Currency pays,
ripple::AccountID payIssuer,
ripple::Currency gets,
ripple::AccountID getIssuer,
std::optional<std::string> const& domain
);
/**
* @brief Parse the book from the request
@@ -637,7 +643,7 @@ parseBook(ripple::Currency pays, ripple::AccountID payIssuer, ripple::Currency g
* @param request The request
* @return The book or an error status
*/
std::variant<Status, ripple::Book>
std::expected<ripple::Book, Status>
parseBook(boost::json::object const& request);
/**
@@ -646,7 +652,7 @@ parseBook(boost::json::object const& request);
* @param taker The taker as json
* @return The taker account or an error status
*/
std::variant<Status, ripple::AccountID>
std::expected<ripple::AccountID, Status>
parseTaker(boost::json::value const& taker);
/**
@@ -683,7 +689,7 @@ isAdminCmd(std::string const& method, boost::json::object const& request);
* @param request The request
* @return The NFTID or an error status
*/
std::variant<ripple::uint256, Status>
std::expected<ripple::uint256, Status>
getNFTID(boost::json::object const& request);
/**
@@ -793,7 +799,7 @@ parseRippleLibSeed(boost::json::value const& value);
* @param atOwnedNode The function to call for each owned node
* @return The account cursor or an error status
*/
std::variant<Status, AccountCursor>
std::expected<AccountCursor, Status>
traverseNFTObjects(
BackendInterface const& backend,
std::uint32_t sequence,

8
src/rpc/common/Types.hpp
View File

@@ -100,7 +100,7 @@ struct ReturnType {
*/
operator bool() const
{
return result.operator bool();
return result.has_value();
}
std::expected<boost::json::value, Status> result;
@@ -137,7 +137,7 @@ struct Result {
if (returnType) {
response = std::move(returnType.result).value().as_object();
} else {
response = std::move(returnType.result).error();
response = std::unexpected{std::move(returnType.result).error()};
}
warnings = std::move(returnType.warnings);
}
@@ -147,7 +147,7 @@ struct Result {
*
* @param status The status to construct the result from
*/
explicit Result(Status status) : response{std::move(status)}
explicit Result(Status status) : response{std::unexpected{std::move(status)}}
{
}
@@ -160,7 +160,7 @@ struct Result {
{
}
std::variant<Status, boost::json::object> response;
std::expected<boost::json::object, Status> response;
boost::json::array warnings;
};

9
src/rpc/handlers/AMMInfo.cpp
View File

@@ -57,7 +57,6 @@
#include <string>
#include <string_view>
#include <utility>
#include <variant>
namespace {
@@ -97,14 +96,14 @@ AMMInfoHandler::process(AMMInfoHandler::Input input, Context const& ctx) const
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AMMInfo's ledger range must be available");
auto const lgrInfoOrStatus = getLedgerHeaderFromHashOrSeq(
auto const expectedLgrInfo = getLedgerHeaderFromHashOrSeq(
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence
);
if (auto const status = std::get_if<Status>(&lgrInfoOrStatus))
return Error{*status};
if (!expectedLgrInfo.has_value())
return Error{expectedLgrInfo.error()};
auto const lgrInfo = std::get<LedgerHeader>(lgrInfoOrStatus);
auto const& lgrInfo = expectedLgrInfo.value();
if (input.accountID) {
auto keylet = keylet::account(*input.accountID);

17
src/rpc/handlers/AccountChannels.cpp
View File

@@ -44,7 +44,6 @@
#include <optional>
#include <string>
#include <utility>
#include <variant>
#include <vector>
namespace rpc {
@@ -86,14 +85,14 @@ AccountChannelsHandler::process(AccountChannelsHandler::Input input, Context con
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountChannel's ledger range must be available");
auto const lgrInfoOrStatus = getLedgerHeaderFromHashOrSeq(
auto const expectedLgrInfo = getLedgerHeaderFromHashOrSeq(
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence
);
if (auto status = std::get_if<Status>(&lgrInfoOrStatus))
return Error{*status};
if (!expectedLgrInfo.has_value())
return Error{expectedLgrInfo.error()};
auto const lgrInfo = std::get<ripple::LedgerHeader>(lgrInfoOrStatus);
auto const& lgrInfo = expectedLgrInfo.value();
auto const accountID = accountFromStringStrict(input.account);
auto const accountLedgerObject =
sharedPtrBackend_->fetchLedgerObject(ripple::keylet::account(*accountID).key, lgrInfo.seq, ctx.yield);
@@ -114,19 +113,19 @@ AccountChannelsHandler::process(AccountChannelsHandler::Input input, Context con
return true;
};
auto const next = traverseOwnedNodes(
auto const expectedNext = traverseOwnedNodes(
*sharedPtrBackend_, *accountID, lgrInfo.seq, input.limit, input.marker, ctx.yield, addToResponse
);
if (auto status = std::get_if<Status>(&next))
return Error{*status};
if (!expectedNext.has_value())
return Error{expectedNext.error()};
response.account = input.account;
response.limit = input.limit;
response.ledgerHash = ripple::strHex(lgrInfo.hash);
response.ledgerIndex = lgrInfo.seq;
auto const nextMarker = std::get<AccountCursor>(next);
auto const nextMarker = expectedNext.value();
if (nextMarker.isNonZero())
response.marker = nextMarker.toString();

9
src/rpc/handlers/AccountCurrencies.cpp
View File

@@ -40,7 +40,6 @@
#include <cstdint>
#include <limits>
#include <string>
#include <variant>
namespace rpc {
AccountCurrenciesHandler::Result
@@ -48,14 +47,14 @@ AccountCurrenciesHandler::process(AccountCurrenciesHandler::Input input, Context
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountCurrencies' ledger range must be available");
auto const lgrInfoOrStatus = getLedgerHeaderFromHashOrSeq(
auto const expectedLgrInfo = getLedgerHeaderFromHashOrSeq(
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence
);
if (auto const status = std::get_if<Status>(&lgrInfoOrStatus))
return Error{*status};
if (!expectedLgrInfo.has_value())
return Error{expectedLgrInfo.error()};
auto const lgrInfo = std::get<ripple::LedgerHeader>(lgrInfoOrStatus);
auto const& lgrInfo = expectedLgrInfo.value();
auto const accountID = accountFromStringStrict(input.account);
auto const accountLedgerObject =

9
src/rpc/handlers/AccountInfo.cpp
View File

@@ -46,7 +46,6 @@
#include <string>
#include <string_view>
#include <utility>
#include <variant>
#include <vector>
namespace rpc {
@@ -60,14 +59,14 @@ AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx)
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountInfo's ledger range must be available");
auto const lgrInfoOrStatus = getLedgerHeaderFromHashOrSeq(
auto const expectedLgrInfo = getLedgerHeaderFromHashOrSeq(
*sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence
);
if (auto const status = std::get_if<Status>(&lgrInfoOrStatus))
return Error{*status};
if (!expectedLgrInfo.has_value())
return Error{expectedLgrInfo.error()};
auto const lgrInfo = std::get<ripple::LedgerHeader>(lgrInfoOrStatus);
auto const& lgrInfo = expectedLgrInfo.value();
auto const accountStr = input.account.value_or(input.ident.value_or(""));
auto const accountID = accountFromStringStrict(accountStr);
auto const accountKeylet = ripple::keylet::account(*accountID);

Some files were not shown because too many files have changed in this diff Show More