mirror of
https://github.com/XRPLF/rippled.git
synced 2026-10-21 19:18:22 +00:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8c4bd3eef | ||
|
|
f41dfae183 | ||
|
|
9a0c1dd4cc | ||
|
|
70dd0d9520 | ||
|
|
20f1a7eaa4 | ||
|
|
4d2aaed51d | ||
|
|
7d90ade1f4 | ||
|
|
503927cfa4 | ||
|
|
a6130a9fae | ||
|
|
d6c119614d | ||
|
|
e554e03921 | ||
|
|
d3c04fe06d | ||
|
|
f80e72c6f7 | ||
|
|
c16f18f79b | ||
|
|
a40b88436d | ||
|
|
c65e4539f5 | ||
|
|
45ed9e4de7 |
No files matched your search
@@ -83,7 +83,6 @@ words:
|
||||
- coeffs
|
||||
- coldwallet
|
||||
- compr
|
||||
- cppcoro
|
||||
- conanfile
|
||||
- conanrun
|
||||
- confs
|
||||
@@ -130,7 +129,6 @@ words:
|
||||
- fsanitize
|
||||
- funclets
|
||||
- Gamal
|
||||
- gantt
|
||||
- gcov
|
||||
- gcovr
|
||||
- ghead
|
||||
@@ -180,7 +178,6 @@ words:
|
||||
- ltype
|
||||
- mathbunnyru
|
||||
- mcmodel
|
||||
- Mankawde
|
||||
- MEMORYSTATUSEX
|
||||
- MPTAMM
|
||||
- MPTDEX
|
||||
@@ -250,7 +247,6 @@ words:
|
||||
- protobuf
|
||||
- protos
|
||||
- ptrs
|
||||
- Pratik
|
||||
- pushd
|
||||
- pyenv
|
||||
- pyparsing
|
||||
@@ -262,8 +258,6 @@ words:
|
||||
- replayer
|
||||
- repodata
|
||||
- repomd
|
||||
- repost
|
||||
- reposts
|
||||
- rerandomize
|
||||
- rerandomization
|
||||
- rerandomized
|
||||
@@ -311,7 +305,6 @@ words:
|
||||
- sponsees
|
||||
- SRPMS
|
||||
- sslws
|
||||
- stackful
|
||||
- statsd
|
||||
- STATSDCOLLECTOR
|
||||
- stissue
|
||||
|
||||
@@ -15,21 +15,26 @@ outputs:
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
# A tag names its own version. Anything else is a development build named by
|
||||
# its commit hash, matching what cmake/XrplVersion.cmake derives: the head of
|
||||
# a pull request rather than the merge commit GitHub creates for it.
|
||||
# A tag names its own version. Anything else takes it from BuildInfo.cpp and
|
||||
# appends the commit hash as build metadata, joined with a plus sign because a
|
||||
# Conan version cannot contain two hyphens.
|
||||
- name: Determine version
|
||||
id: version
|
||||
shell: bash
|
||||
env:
|
||||
IS_TAG: ${{ startsWith(github.ref, 'refs/tags/') }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
SHA: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
SHA: ${{ github.sha }}
|
||||
run: |
|
||||
if [[ "${IS_TAG}" == "true" ]]; then
|
||||
version="${REF_NAME}"
|
||||
else
|
||||
version="0.0.0-dev+${SHA:0:7}"
|
||||
version="$(awk -F'"' '/versionString =/ { print $2 }' src/libxrpl/protocol/BuildInfo.cpp)"
|
||||
if [[ -z "${version}" ]]; then
|
||||
echo "Unable to read versionString from BuildInfo.cpp." >&2
|
||||
exit 1
|
||||
fi
|
||||
version="${version}+${SHA:0:7}"
|
||||
fi
|
||||
|
||||
echo "version=${version}" | tee -a "${GITHUB_OUTPUT}"
|
||||
|
||||
@@ -74,7 +74,7 @@
|
||||
"extra_cmake_args": "-Dvalidator_keys=ON",
|
||||
"package": {
|
||||
"type": "deb",
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-3a2d19f"
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-49cdc10"
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -86,7 +86,7 @@
|
||||
"extra_cmake_args": "-Dvalidator_keys=ON -Dassert=ON",
|
||||
"package": {
|
||||
"type": "deb",
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-3a2d19f",
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-49cdc10",
|
||||
"variant": "assert"
|
||||
}
|
||||
}
|
||||
@@ -101,7 +101,7 @@
|
||||
"extra_cmake_args": "-Dvalidator_keys=ON",
|
||||
"package": {
|
||||
"type": "rpm",
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-rhel:sha-3a2d19f"
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-rhel:sha-49cdc10"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -7,12 +7,12 @@ on:
|
||||
paths:
|
||||
- ".github/workflows/build-packaging-images.yml"
|
||||
- "bin/install-packaging-tools.sh"
|
||||
- "package/images/packaging/**"
|
||||
- "package/docker/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/build-packaging-images.yml"
|
||||
- "bin/install-packaging-tools.sh"
|
||||
- "package/images/packaging/**"
|
||||
- "package/docker/**"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
@@ -44,6 +44,6 @@ jobs:
|
||||
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@696384b292577293292daed06af0306d1b83bd7d
|
||||
with:
|
||||
image_name: xrpld/packaging-${{ matrix.distro.name }}
|
||||
dockerfile: package/images/packaging/Dockerfile
|
||||
dockerfile: package/docker/Dockerfile
|
||||
base_image: ${{ matrix.distro.base_image }}
|
||||
push: ${{ github.event_name == 'push' }}
|
||||
@@ -1,12 +1,9 @@
|
||||
# When a versioned tag is pushed, this workflow:
|
||||
#
|
||||
# - builds and tests the release binaries
|
||||
# - uploads the libxrpl recipe to the Conan remote
|
||||
# - builds and tests the release binaries
|
||||
# - builds the DEB and RPM packages
|
||||
# - publishes those packages to the XRPLF package repositories
|
||||
#
|
||||
# Nothing is published unless the build passes, which is also where CMake
|
||||
# rejects a tag that is not a valid version, e.g. 3.2.01.
|
||||
name: Tag
|
||||
|
||||
on:
|
||||
@@ -25,7 +22,6 @@ defaults:
|
||||
jobs:
|
||||
upload-recipe:
|
||||
if: ${{ github.repository == 'XRPLF/rippled' }}
|
||||
needs: build-test
|
||||
uses: ./.github/workflows/reusable-upload-recipe.yml
|
||||
secrets:
|
||||
remote_username: ${{ secrets.NEXUS_REMOTE_USERNAME }}
|
||||
@@ -55,6 +51,3 @@ jobs:
|
||||
remote_password: ${{ secrets.NEXUS_REMOTE_PASSWORD }}
|
||||
signing_key: ${{ secrets.NEXUS_PACKAGES_PRIVATE_KEY }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
antithesis_docker_host: ${{ secrets.ANTITHESIS_DOCKER_HOST }}
|
||||
antithesis_docker_path: ${{ secrets.ANTITHESIS_DOCKER_PATH }}
|
||||
antithesis_docker_credentials: ${{ secrets.ANTITHESIS_DOCKER_CREDENTIALS }}
|
||||
@@ -130,6 +130,3 @@ jobs:
|
||||
remote_password: ${{ secrets.NEXUS_REMOTE_PASSWORD }}
|
||||
signing_key: ${{ secrets.NEXUS_PACKAGES_PRIVATE_KEY }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
antithesis_docker_host: ${{ secrets.ANTITHESIS_DOCKER_HOST }}
|
||||
antithesis_docker_path: ${{ secrets.ANTITHESIS_DOCKER_PATH }}
|
||||
antithesis_docker_credentials: ${{ secrets.ANTITHESIS_DOCKER_CREDENTIALS }}
|
||||
@@ -196,19 +196,6 @@ jobs:
|
||||
env:
|
||||
BUILD_TYPE: ${{ inputs.build_type }}
|
||||
CMAKE_ARGS: ${{ inputs.cmake_args }}
|
||||
# GitHub creates a merge commit for a PR
|
||||
# https://www.kenmuse.com/blog/the-many-shas-of-a-github-pull-request/
|
||||
#
|
||||
# We:
|
||||
# - explicitly provide branch name
|
||||
# - use `github.event.pull_request.head.sha` to get the SHA of last commit in the PR branch
|
||||
#
|
||||
# This way it works both for PRs and pushes to branches.
|
||||
GITHUB_BRANCH_NAME: "${{ github.head_ref || github.ref_name }}"
|
||||
GITHUB_HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
|
||||
#
|
||||
# If tag is being pushed, we use that version.
|
||||
FORCE_XRPLD_VERSION: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || '' }}
|
||||
run: |
|
||||
cmake \
|
||||
-G '${{ runner.os == 'Windows' && 'Visual Studio 18 2026' || 'Ninja' }}' \
|
||||
|
||||
@@ -9,9 +9,8 @@
|
||||
# never reaches Nexus
|
||||
# - 'publish' uploads with the image's publish_pkg.py, doing a --dry-run
|
||||
# unless 'publish: true'
|
||||
# - 'docker' builds an Ubuntu image from the tested DEB, and one with the
|
||||
# voidstar binary for Antithesis, pushing them to Docker Hub and to the
|
||||
# Antithesis registry only with 'publish: true'
|
||||
# - 'docker' builds an Ubuntu image from the tested DEB, pushing it to Docker
|
||||
# Hub only with 'publish: true'
|
||||
#
|
||||
# Only linux/amd64 is supported; the runner is hardcoded in the jobs below.
|
||||
name: Package
|
||||
@@ -38,19 +37,10 @@ on:
|
||||
description: "The password or token for that Nexus account."
|
||||
required: false
|
||||
signing_key:
|
||||
description: "Armoured PGP private key used to sign the RPMs."
|
||||
description: "Armoured PGP private key used to sign the RPMs. Required when publishing."
|
||||
required: false
|
||||
dockerhub_token:
|
||||
description: "A Docker Hub organization access token for xrplf, with push access to xrplf/xrpld."
|
||||
required: false
|
||||
antithesis_docker_host:
|
||||
description: "The host of the Antithesis container registry, e.g. us-central1-docker.pkg.dev."
|
||||
required: false
|
||||
antithesis_docker_path:
|
||||
description: "The repository path in that registry, the image name excluded."
|
||||
required: false
|
||||
antithesis_docker_credentials:
|
||||
description: "The JSON key of a service account with push access to that repository."
|
||||
description: "A Docker Hub organization access token for xrplf, with push access to xrplf/xrpld. Required when publishing."
|
||||
required: false
|
||||
|
||||
defaults:
|
||||
@@ -257,21 +247,13 @@ jobs:
|
||||
|
||||
docker:
|
||||
needs: [test-install-deb, test-install-rpm]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target: [xrpld, voidstar]
|
||||
name: "docker ${{ matrix.target }}${{ !inputs.publish && ' (dry run)' || '' }}"
|
||||
name: "docker${{ !inputs.publish && ' (dry run)' || '' }}"
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
CONTEXT: image-context
|
||||
IMAGE: ${{ matrix.target == 'voidstar' && 'xrpld-voidstar' || 'xrplf/xrpld' }}:${{ github.ref_type == 'tag' && github.ref_name || 'develop' }}
|
||||
# Docker Hub is public, so a private build never reaches it;
|
||||
# the Antithesis registry is not.
|
||||
PUSH: ${{ inputs.publish && (matrix.target == 'voidstar' || github.event.repository.visibility == 'public') }}
|
||||
IMAGE: xrplf/xrpld:${{ github.ref_type == 'tag' && github.ref_name || 'develop' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
@@ -284,20 +266,13 @@ jobs:
|
||||
merge-multiple: true
|
||||
path: ${{ env.PACKAGE_DIR }}
|
||||
|
||||
- name: Download voidstar binary
|
||||
if: ${{ matrix.target == 'voidstar' }}
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: xrpld-ubuntu-clang-debug-amd64-voidstar
|
||||
path: ${{ env.CONTEXT }}
|
||||
|
||||
- name: Build image
|
||||
env:
|
||||
TARGET: ${{ matrix.target }}
|
||||
CONTEXT: image-context
|
||||
run: |
|
||||
mkdir -p "${CONTEXT}"
|
||||
find "${PACKAGE_DIR}" -type f -name 'xrpld_[0-9]*.deb' -exec cp {} "${CONTEXT}/" \;
|
||||
docker build --pull --file package/images/xrpld/Dockerfile --target "${TARGET}" --tag "${IMAGE}" "${CONTEXT}"
|
||||
docker build --pull --file package/image/Dockerfile --tag "${IMAGE}" "${CONTEXT}"
|
||||
|
||||
- name: Start the server
|
||||
run: |
|
||||
@@ -315,25 +290,14 @@ jobs:
|
||||
docker logs "${container}"
|
||||
exit 1
|
||||
|
||||
# Docker Hub is public, so a private build never reaches it.
|
||||
- name: Log in to Docker Hub
|
||||
if: ${{ env.PUSH == 'true' && matrix.target == 'xrpld' }}
|
||||
if: ${{ inputs.publish && github.event.repository.visibility == 'public' }}
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
username: xrplf
|
||||
password: ${{ secrets.dockerhub_token }}
|
||||
|
||||
- name: Log in to the Antithesis registry
|
||||
if: ${{ env.PUSH == 'true' && matrix.target == 'voidstar' }}
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ secrets.antithesis_docker_host }}
|
||||
username: _json_key
|
||||
password: ${{ secrets.antithesis_docker_credentials }}
|
||||
|
||||
- name: Push image
|
||||
if: ${{ env.PUSH == 'true' }}
|
||||
env:
|
||||
REGISTRY: ${{ matrix.target == 'voidstar' && format('{0}/{1}/', secrets.antithesis_docker_host, secrets.antithesis_docker_path) || '' }}
|
||||
run: |
|
||||
docker tag "${IMAGE}" "${REGISTRY}${IMAGE}"
|
||||
docker push "${REGISTRY}${IMAGE}"
|
||||
if: ${{ inputs.publish && github.event.repository.visibility == 'public' }}
|
||||
run: docker push "${IMAGE}"
|
||||
@@ -28,6 +28,7 @@ Version 3.4.0 is not yet released. These changes are available in the 3.4.0 beta
|
||||
|
||||
### Additions in 3.4.0
|
||||
|
||||
- `server_info` (admin): The `node_size` field has been removed; it reported the deprecated `[node_size]` config setting, which is still accepted as an alias for `[memory_limit]` and still emits a startup warning. Admin responses now include `memory_limit`, the cache memory budget in gigabytes rounded up (0 when enforcement is disabled).
|
||||
- `ledger`: `nftoken_id`, `nftoken_ids`, and `offer_id` are now included in transaction metadata when transactions are expanded (`expand`, or admin-only `full`), matching the `tx`, `account_tx`, and `subscribe` (`transactions` stream) responses. ([#5706](https://github.com/XRPLF/rippled/pull/5706))
|
||||
|
||||
### Bugfixes in 3.4.0
|
||||
|
||||
@@ -51,12 +51,6 @@ git checkout release/3.2.x
|
||||
For a stable release, choose one of the [tagged
|
||||
releases](https://github.com/XRPLF/rippled/releases).
|
||||
|
||||
A build reports `0.0.0-dev` with its short commit hash as build metadata, e.g.
|
||||
`0.0.0-dev+0123abc`. Only builds of a release, from a tag in CI or a versioned
|
||||
Conan reference, report a release version. To use another version, set the
|
||||
`FORCE_XRPLD_VERSION` environment variable when running CMake, e.g.
|
||||
`FORCE_XRPLD_VERSION=3.4.0`.
|
||||
|
||||
### Set Up Conan
|
||||
|
||||
Once your [development environment](./docs/build/environment.md) is ready, set
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,142 +0,0 @@
|
||||
# Boost.Coroutine to C++20 Migration — Task List
|
||||
|
||||
> Parent document: [BoostToStdCoroutineSwitchPlan.md](BoostToStdCoroutineSwitchPlan.md)
|
||||
|
||||
---
|
||||
|
||||
## Milestone 1: New Coroutine Primitives
|
||||
|
||||
- [ ] **1.1** Design `CoroTask<T>` class with `promise_type`
|
||||
- Define `promise_type` with `initial_suspend`, `final_suspend`, `unhandled_exception`, `return_value`/`return_void`
|
||||
- Implement `FinalAwaiter` for continuation support
|
||||
- Implement move-only RAII handle wrapper
|
||||
- Support both `CoroTask<T>` and `CoroTask<void>`
|
||||
|
||||
- [ ] **1.2** Design and implement `JobQueueAwaiter`
|
||||
- `await_suspend()` calls `jq_.addJob(type, name, [h]{ h.resume(); })`
|
||||
- Handle `addJob()` failure (shutdown) — resume with error flag or throw
|
||||
- Integrate `nSuspend_` counter increment/decrement
|
||||
|
||||
- [ ] **1.3** Implement `LocalValues` swap in new coroutine resume path
|
||||
- Before `handle.resume()`: save thread-local, install coroutine-local
|
||||
- After `handle.resume()` returns: restore thread-local
|
||||
- Ensure this works when coroutine migrates between threads
|
||||
|
||||
- [ ] **1.4** Add `postCoroTask()` template to `JobQueue`
|
||||
- Accept callable returning `CoroTask<void>`
|
||||
- Schedule initial execution on JobQueue (mirror `postCoro()` behavior)
|
||||
- Return a handle/shared_ptr for join/cancel
|
||||
|
||||
- [ ] **1.5** Write unit tests (`src/test/core/CoroTask_test.cpp`)
|
||||
- Test `CoroTask<void>` runs to completion
|
||||
- Test `CoroTask<int>` returns value
|
||||
- Test exception propagation across co_await
|
||||
- Test coroutine destruction before completion
|
||||
- Test `JobQueueAwaiter` schedules on correct thread
|
||||
- Test `LocalValue` isolation across 4+ coroutines
|
||||
- Test shutdown rejection (addJob returns false)
|
||||
- Test `correct_order` equivalent (yield → join → post → complete)
|
||||
- Test `incorrect_order` equivalent (post → yield → complete)
|
||||
- Test multiple sequential co_await points
|
||||
|
||||
- [ ] **1.6** Verify build on GCC 12+, Clang 16+
|
||||
- [ ] **1.7** Run ASAN + TSAN on new tests
|
||||
- [ ] **1.8** Run full `--unittest` suite (no regressions)
|
||||
- [ ] **1.9** Self-review and create PR #1
|
||||
|
||||
---
|
||||
|
||||
## Milestone 2: Entry Point Migration
|
||||
|
||||
- [ ] **2.1** Migrate `ServerHandler::onRequest()` (`ServerHandler.cpp:287`)
|
||||
- Replace `m_jobQueue.postCoro(jtCLIENT_RPC, ...)` with `postCoroTask()`
|
||||
- Update lambda to return `CoroTask<void>` (add `co_return`)
|
||||
- Update `processSession` to accept new coroutine type
|
||||
|
||||
- [ ] **2.2** Migrate `ServerHandler::onWSMessage()` (`ServerHandler.cpp:325`)
|
||||
- Replace `m_jobQueue.postCoro(jtCLIENT_WEBSOCKET, ...)` with `postCoroTask()`
|
||||
- Update lambda signature
|
||||
|
||||
- [ ] **2.3** Migrate `GRPCServer::CallData::process()` (`GRPCServer.cpp:102`)
|
||||
- Replace `app_.getJobQueue().postCoro(JobType::jtRPC, ...)` with `postCoroTask()`
|
||||
- Update `process(shared_ptr<Coro> coro)` overload signature
|
||||
|
||||
- [ ] **2.4** Update `RPC::Context` (`Context.h:27`)
|
||||
- Replace `std::shared_ptr<JobQueue::Coro> coro{}` with new coroutine wrapper type
|
||||
- Ensure all code that accesses `context.coro` compiles
|
||||
|
||||
- [ ] **2.5** Update `ServerHandler.h` signatures
|
||||
- `processSession()` and `processRequest()` parameter types
|
||||
|
||||
- [ ] **2.6** Update `GRPCServer.h` signatures
|
||||
- `process()` method parameter types
|
||||
|
||||
- [ ] **2.7** Run full `--unittest` suite
|
||||
- [ ] **2.8** Manual smoke test: HTTP + WS + gRPC RPC requests
|
||||
- [ ] **2.9** Run ASAN + TSAN
|
||||
- [ ] **2.10** Self-review and create PR #2
|
||||
|
||||
---
|
||||
|
||||
## Milestone 3: Handler Migration
|
||||
|
||||
- [ ] **3.1** Migrate `doRipplePathFind()` (`RipplePathFind.cpp`)
|
||||
- Replace `context.coro->yield()` with `co_await PathFindAwaiter{...}`
|
||||
- Replace continuation lambda's `coro->post()` / `coro->resume()` with awaiter scheduling
|
||||
- Handle shutdown case (post failure) in awaiter
|
||||
|
||||
- [ ] **3.2** Create `PathFindAwaiter` (or use generic `JobQueueAwaiter`)
|
||||
- Encapsulate the continuation + yield pattern from `RipplePathFind.cpp` lines 108-132
|
||||
|
||||
- [ ] **3.3** Update `Path_test.cpp`
|
||||
- Replace `postCoro` usage with `postCoroTask`
|
||||
- Ensure `context.coro` usage matches new type
|
||||
|
||||
- [ ] **3.4** Update `AMMTest.cpp`
|
||||
- Replace `postCoro` usage with `postCoroTask`
|
||||
|
||||
- [ ] **3.5** Rewrite `Coroutine_test.cpp` for new API
|
||||
- `correct_order`: postCoroTask → co_await → join → resume → complete
|
||||
- `incorrect_order`: post before yield equivalent
|
||||
- `thread_specific_storage`: 4 coroutines with LocalValue isolation
|
||||
|
||||
- [ ] **3.6** Update `JobQueue_test.cpp` `testPostCoro`
|
||||
- Migrate to `postCoroTask` API
|
||||
|
||||
- [ ] **3.7** Verify `ripple_path_find` works end-to-end with new coroutines
|
||||
- [ ] **3.8** Test shutdown-during-pathfind scenario
|
||||
- [ ] **3.9** Run full `--unittest` suite
|
||||
- [ ] **3.10** Run ASAN + TSAN
|
||||
- [ ] **3.11** Self-review and create PR #3
|
||||
|
||||
---
|
||||
|
||||
## Milestone 4: Cleanup & Validation
|
||||
|
||||
- [ ] **4.1** Delete `include/xrpl/core/Coro.ipp`
|
||||
- [ ] **4.2** Remove from `JobQueue.h`:
|
||||
- `#include <boost/coroutine2/all.hpp>`
|
||||
- `struct Coro_create_t`
|
||||
- `class Coro` (entire class)
|
||||
- `postCoro()` template
|
||||
- Comment block (lines 322-377) describing old race condition
|
||||
- [ ] **4.3** Update `cmake/deps/Boost.cmake`:
|
||||
- Remove `coroutine` from `find_package(Boost REQUIRED COMPONENTS ...)`
|
||||
- Remove `Boost::coroutine` from `target_link_libraries`
|
||||
- [ ] **4.4** Update `cmake/XrplInterface.cmake`:
|
||||
- Remove `BOOST_COROUTINES2_NO_DEPRECATION_WARNING`
|
||||
- [ ] **4.5** Run memory benchmark
|
||||
- Create N=1000 coroutines, compare RSS: before vs after
|
||||
- Document results
|
||||
- [ ] **4.6** Run context switch benchmark
|
||||
- 100K yield/resume cycles, compare latency: before vs after
|
||||
- Document results
|
||||
- [ ] **4.7** Run RPC throughput benchmark
|
||||
- Concurrent `ripple_path_find` requests, compare throughput
|
||||
- Document results
|
||||
- [ ] **4.8** Run full `--unittest` suite
|
||||
- [ ] **4.9** Run ASAN, TSAN, UBSan
|
||||
- Confirm `__asan_handle_no_return` warnings are gone
|
||||
- [ ] **4.10** Verify build on all supported compilers
|
||||
- [ ] **4.11** Self-review and create PR #4
|
||||
- [ ] **4.12** Document final benchmark results in PR description
|
||||
@@ -77,6 +77,7 @@ endif()
|
||||
include(PatchNixBinary)
|
||||
|
||||
include(XrplSanity)
|
||||
include(XrplVersion)
|
||||
include(XrplSettings)
|
||||
# this check has to remain in the top-level cmake because of the early return statement
|
||||
if(packages_only)
|
||||
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ $# -ne 3 || "$1" == "--help" || "$1" = "-h" ]]; then
|
||||
name=$(basename $0)
|
||||
cat <<-USAGE
|
||||
Usage: $name workbranch base/branch version
|
||||
|
||||
* workbranch will be created locally from base/branch. If it exists,
|
||||
it will be reused, so make sure you don't overwrite any work.
|
||||
* base/branch may be specified as user:branch to allow easy copying
|
||||
from Github PRs.
|
||||
USAGE
|
||||
exit 0
|
||||
fi
|
||||
|
||||
work="$1"
|
||||
shift
|
||||
|
||||
base=$(echo "$1" | sed "s/:/\//")
|
||||
shift
|
||||
|
||||
version=$1
|
||||
shift
|
||||
|
||||
set -e
|
||||
|
||||
git fetch upstreams
|
||||
|
||||
git checkout -B "${work}" --no-track "${base}"
|
||||
|
||||
push=$(git rev-parse --abbrev-ref --symbolic-full-name '@{push}' \
|
||||
2>/dev/null) || true
|
||||
if [[ "${push}" != "" ]]; then
|
||||
echo "Warning: ${push} may already exist."
|
||||
fi
|
||||
|
||||
build=$(find -name BuildInfo.cpp)
|
||||
sed 's/\(^.*versionString =\).*$/\1 "'${version}'"/' ${build} >version.cpp &&
|
||||
diff "${build}" version.cpp && exit 1 ||
|
||||
mv -vi version.cpp ${build}
|
||||
|
||||
git diff
|
||||
|
||||
git add ${build}
|
||||
|
||||
git commit -S -m "Set version to ${version}"
|
||||
|
||||
git log --oneline --first-parent ${base}^..
|
||||
|
||||
cat <<PUSH
|
||||
|
||||
-------------------------------------------------------------------
|
||||
This script will not push. Verify everything is correct, then push
|
||||
to your repo, and create a PR as described in CONTRIBUTING.md.
|
||||
-------------------------------------------------------------------
|
||||
PUSH
|
||||
+33
-15
@@ -1368,23 +1368,41 @@
|
||||
#
|
||||
# [node_size]
|
||||
#
|
||||
# Tunes the servers based on the expected load and available memory. Legal
|
||||
# sizes are "tiny", "small", "medium", "large", and "huge". We recommend
|
||||
# you start at the default and raise the setting if you have extra memory.
|
||||
# DEPRECATED. Each size is now an alias for a [memory_limit] value:
|
||||
# tiny = 4, small = 8, medium = 32, large = 64, huge = 128. Set
|
||||
# [memory_limit] instead; setting this logs a warning at startup.
|
||||
#
|
||||
# The code attempts to automatically determine the appropriate size for
|
||||
# this parameter based on the amount of RAM and the number of execution
|
||||
# cores available to the server. The current decision matrix is:
|
||||
# [memory_limit]
|
||||
#
|
||||
# | | Cores |
|
||||
# |---------|------------------------|
|
||||
# | RAM | 1 | 2 or 3 | ≥ 4 |
|
||||
# |---------|------|--------|--------|
|
||||
# | < ~8GB | tiny | tiny | tiny |
|
||||
# | < ~12GB | tiny | small | small |
|
||||
# | < ~16GB | tiny | small | medium |
|
||||
# | < ~24GB | tiny | small | large |
|
||||
# | < ~32GB | tiny | small | huge |
|
||||
# The memory budget, in gigabytes, that the server sizes its caches
|
||||
# within. Cache sizes scale with the budget; the SHAMap tree node cache
|
||||
# is capped to fit within half of it, enforced as it grows. Defaults to
|
||||
# detected physical RAM (capped by the container limit when one is set);
|
||||
# 0 selects minimal sizes with no enforcement. Values above 1024 are
|
||||
# rejected, and a value above detected RAM logs a warning.
|
||||
# Set this when xrpld shares the machine with other services or runs in
|
||||
# a container with a memory limit below the host's RAM. Thread counts
|
||||
# are unrelated: they come from the core count and the [workers] /
|
||||
# [io_workers] overrides.
|
||||
#
|
||||
# Example:
|
||||
# [memory_limit]
|
||||
# 16
|
||||
#
|
||||
# [tree_cache_age]
|
||||
#
|
||||
# Seconds a SHAMap tree node stays cached after its last use. The default
|
||||
# is 300. Accepted values are 10 to 3600.
|
||||
#
|
||||
# [ledger_cache_age]
|
||||
#
|
||||
# Seconds a full ledger stays in the ledger cache after its last use. The
|
||||
# default is 180. Accepted values are 10 to 3600.
|
||||
#
|
||||
# [ledger_fetch_size]
|
||||
#
|
||||
# How many historical ledgers to acquire per fetch pass while backfilling.
|
||||
# The default is 4. Accepted values are 1 to 16.
|
||||
#
|
||||
# [signing_support]
|
||||
#
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
include_guard()
|
||||
|
||||
set(GIT_BUILD_BRANCH "")
|
||||
set(GIT_COMMIT_HASH "")
|
||||
|
||||
find_package(Git)
|
||||
if(NOT Git_FOUND)
|
||||
message(WARNING "Git not found. Git branch and commit hash will be empty.")
|
||||
return()
|
||||
endif()
|
||||
|
||||
set(GIT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.git)
|
||||
|
||||
execute_process(
|
||||
COMMAND
|
||||
${GIT_EXECUTABLE} --git-dir=${GIT_DIRECTORY} rev-parse --abbrev-ref HEAD
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
OUTPUT_VARIABLE GIT_BUILD_BRANCH
|
||||
)
|
||||
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} --git-dir=${GIT_DIRECTORY} rev-parse HEAD
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
OUTPUT_VARIABLE GIT_COMMIT_HASH
|
||||
)
|
||||
|
||||
message(STATUS "Git branch: ${GIT_BUILD_BRANCH}")
|
||||
message(STATUS "Git commit hash: ${GIT_COMMIT_HASH}")
|
||||
@@ -81,7 +81,7 @@ include(target_link_modules)
|
||||
add_module(xrpl beast)
|
||||
target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main)
|
||||
|
||||
include(XrplVersion)
|
||||
include(GitInfo)
|
||||
add_module(xrpl git)
|
||||
target_compile_definitions(
|
||||
xrpl.libxrpl.git
|
||||
@@ -111,11 +111,6 @@ target_link_libraries(
|
||||
xrpl.libxrpl.protocol
|
||||
PUBLIC xrpl.libxrpl.crypto xrpl.libxrpl.git xrpl.libxrpl.json
|
||||
)
|
||||
# Only on BuildInfo.cpp, so a new version does not rebuild the whole module.
|
||||
set_source_files_properties(
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/libxrpl/protocol/BuildInfo.cpp
|
||||
PROPERTIES COMPILE_DEFINITIONS XRPLD_VERSION="${XRPLD_VERSION}"
|
||||
)
|
||||
|
||||
# Level 05
|
||||
add_module(xrpl protocol_autogen)
|
||||
|
||||
+11
-71
@@ -1,75 +1,15 @@
|
||||
find_package(Git)
|
||||
#[===================================================================[
|
||||
read version from source
|
||||
#]===================================================================]
|
||||
|
||||
set(GIT_BUILD_BRANCH "")
|
||||
set(GIT_COMMIT_HASH "")
|
||||
|
||||
if(DEFINED ENV{GITHUB_BRANCH_NAME})
|
||||
set(GIT_BUILD_BRANCH $ENV{GITHUB_BRANCH_NAME})
|
||||
set(GIT_COMMIT_HASH $ENV{GITHUB_HEAD_SHA})
|
||||
elseif(Git_FOUND AND EXISTS "${CMAKE_CURRENT_LIST_DIR}/../.git")
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/..
|
||||
OUTPUT_VARIABLE GIT_BUILD_BRANCH
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
COMMAND_ERROR_IS_FATAL ANY
|
||||
)
|
||||
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} rev-parse HEAD
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/..
|
||||
OUTPUT_VARIABLE GIT_COMMIT_HASH
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
COMMAND_ERROR_IS_FATAL ANY
|
||||
)
|
||||
endif()
|
||||
|
||||
message(STATUS "Git branch: ${GIT_BUILD_BRANCH}")
|
||||
message(STATUS "Git commit hash: ${GIT_COMMIT_HASH}")
|
||||
|
||||
if(
|
||||
DEFINED ENV{FORCE_XRPLD_VERSION}
|
||||
AND NOT "$ENV{FORCE_XRPLD_VERSION}" STREQUAL ""
|
||||
)
|
||||
message(
|
||||
STATUS
|
||||
"Using explicitly provided '$ENV{FORCE_XRPLD_VERSION}' as xrpld version"
|
||||
)
|
||||
|
||||
set(XRPLD_VERSION "$ENV{FORCE_XRPLD_VERSION}")
|
||||
|
||||
# The rules beast::SemanticVersion::parse applies, so that an invalid version
|
||||
# fails here rather than when xrpld starts. It reads each number as an int.
|
||||
set(SEMVER_NUMBER "(0|[1-9][0-9]*)")
|
||||
set(SEMVER_PRE_RELEASE "[A-Za-z1-9-][A-Za-z0-9-]*")
|
||||
set(SEMVER_METADATA "[A-Za-z0-9-]+")
|
||||
set(SEMVER_NUMBER_MAX 2147483647)
|
||||
if(
|
||||
NOT XRPLD_VERSION
|
||||
MATCHES
|
||||
"^${SEMVER_NUMBER}\\.${SEMVER_NUMBER}\\.${SEMVER_NUMBER}(-${SEMVER_PRE_RELEASE}(\\.${SEMVER_PRE_RELEASE})*)?(\\+${SEMVER_METADATA}(\\.${SEMVER_METADATA})*)?$"
|
||||
OR CMAKE_MATCH_1 GREATER SEMVER_NUMBER_MAX
|
||||
OR CMAKE_MATCH_2 GREATER SEMVER_NUMBER_MAX
|
||||
OR CMAKE_MATCH_3 GREATER SEMVER_NUMBER_MAX
|
||||
)
|
||||
message(
|
||||
FATAL_ERROR
|
||||
"FORCE_XRPLD_VERSION '${XRPLD_VERSION}' is not a semantic version xrpld accepts, see https://semver.org"
|
||||
)
|
||||
file(STRINGS src/libxrpl/protocol/BuildInfo.cpp BUILD_INFO)
|
||||
foreach(line_ ${BUILD_INFO})
|
||||
if(line_ MATCHES "versionString[ ]*=[ ]*\"(.+)\"")
|
||||
set(xrpld_version ${CMAKE_MATCH_1})
|
||||
endif()
|
||||
endforeach()
|
||||
if(xrpld_version)
|
||||
message(STATUS "xrpld version: ${xrpld_version}")
|
||||
else()
|
||||
message(STATUS "Using '0.0.0-dev+<git short rev>' as xrpld version")
|
||||
|
||||
if(GIT_COMMIT_HASH STREQUAL "")
|
||||
message(
|
||||
FATAL_ERROR
|
||||
"Unable to determine xrpld version without git, set FORCE_XRPLD_VERSION"
|
||||
)
|
||||
endif()
|
||||
|
||||
string(SUBSTRING ${GIT_COMMIT_HASH} 0 7 GIT_COMMIT_HASH_SHORT)
|
||||
|
||||
set(XRPLD_VERSION "0.0.0-dev+${GIT_COMMIT_HASH_SHORT}")
|
||||
message(FATAL_ERROR "unable to determine xrpld version")
|
||||
endif()
|
||||
|
||||
message(STATUS "Build version: ${XRPLD_VERSION}")
|
||||
+7
-16
@@ -1,13 +1,9 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
|
||||
from conan.tools.env import Environment
|
||||
|
||||
from conan import ConanFile
|
||||
|
||||
DEV_VERSION = "0.0.0-dev"
|
||||
|
||||
|
||||
class Xrpl(ConanFile):
|
||||
name = "xrpl"
|
||||
@@ -124,7 +120,13 @@ class Xrpl(ConanFile):
|
||||
}
|
||||
|
||||
def set_version(self):
|
||||
self.version = self.version or DEV_VERSION
|
||||
if self.version is None:
|
||||
path = f"{self.recipe_folder}/src/libxrpl/protocol/BuildInfo.cpp"
|
||||
regex = r"versionString\s?=\s?\"(.*)\""
|
||||
with open(path, encoding="utf-8") as file:
|
||||
matches = (re.search(regex, line) for line in file)
|
||||
match = next(m for m in matches if m)
|
||||
self.version = match.group(1)
|
||||
|
||||
def configure(self):
|
||||
if self.settings.compiler == "apple-clang":
|
||||
@@ -167,17 +169,6 @@ class Xrpl(ConanFile):
|
||||
generators = "CMakeDeps"
|
||||
|
||||
def generate(self):
|
||||
# The sources in the Conan cache have no git history, so the version
|
||||
# comes from the reference, unless it is not one, like 'develop'.
|
||||
if not os.path.exists(os.path.join(self.source_folder, ".git")):
|
||||
version = str(self.version)
|
||||
env = Environment()
|
||||
env.define(
|
||||
"FORCE_XRPLD_VERSION",
|
||||
version if re.match(r"\d+\.\d+\.\d+", version) else DEV_VERSION,
|
||||
)
|
||||
env.vars(self).save_script("xrpld_version")
|
||||
|
||||
tc = CMakeToolchain(self)
|
||||
tc.variables["tests"] = self.options.tests
|
||||
tc.variables["benchmark"] = self.options.benchmark
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@
|
||||
the `xrpld` DEB package installed on Ubuntu 26.04, running as the `xrpld` user.
|
||||
Each release is tagged with its version, `xrplf/xrpld:<version>`, and
|
||||
`xrplf/xrpld:develop` follows the `develop` branch.
|
||||
See [`package/README.md`](../package/README.md#docker-images) for how it is built
|
||||
See [`package/README.md`](../package/README.md#docker-image) for how it is built
|
||||
and tagged.
|
||||
|
||||
```bash
|
||||
|
||||
@@ -15,9 +15,11 @@
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <deque>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <type_traits>
|
||||
@@ -74,13 +76,24 @@ public:
|
||||
using SharedPointerType = SharedPointer;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @param cacheHardCap When positive, a hard upper bound on the number of
|
||||
* strongly-cached entries, enforced by demoting the approximately
|
||||
* oldest entry whenever growth would exceed it. 0 disables the cap
|
||||
* (the periodic sweep alone bounds the cache).
|
||||
* @param partitions Number of partitions the underlying map is split
|
||||
* into; defaults to the hardware concurrency. Exposed so tests can
|
||||
* pin a small, known partition count.
|
||||
*/
|
||||
TaggedCache(
|
||||
std::string const& name,
|
||||
int size,
|
||||
ClockType::duration expiration,
|
||||
ClockType& clock,
|
||||
beast::Journal journal,
|
||||
beast::insight::Collector::Ptr const& collector = beast::insight::NullCollector::make());
|
||||
beast::insight::Collector::Ptr const& collector = beast::insight::NullCollector::make(),
|
||||
int cacheHardCap = 0,
|
||||
std::optional<std::size_t> partitions = std::nullopt);
|
||||
|
||||
public:
|
||||
/**
|
||||
@@ -101,6 +114,13 @@ public:
|
||||
int
|
||||
getTrackSize() const;
|
||||
|
||||
/**
|
||||
* Returns the number of strong-key slots evictForHardCap has examined
|
||||
* since construction.
|
||||
*/
|
||||
std::uint64_t
|
||||
getEvictVisits() const;
|
||||
|
||||
float
|
||||
getHitRate();
|
||||
|
||||
@@ -314,6 +334,9 @@ private:
|
||||
public:
|
||||
SharedWeakComboPointerType ptr;
|
||||
ClockType::time_point lastAccess;
|
||||
// Stamped by queueStrong each time the entry becomes strong; the
|
||||
// strong-key slot carrying the same seq is the entry's live slot.
|
||||
std::uint64_t strongSeq{0};
|
||||
|
||||
ValueEntry(ClockType::time_point const& lastAccess, SharedPointerType const& ptr)
|
||||
: ptr(ptr), lastAccess(lastAccess)
|
||||
@@ -357,6 +380,26 @@ private:
|
||||
|
||||
using CacheType = HardenedPartitionedHashMap<key_type, Entry, Hash, KeyEqual>;
|
||||
|
||||
// Counts an entry that just became strong and, under a hard cap, queues
|
||||
// its key for eviction and evicts down to the cap. Caller holds mutex_.
|
||||
void
|
||||
addStrong(CacheType::Iterator const& it);
|
||||
|
||||
// Stamps the entry with a new strong seq and appends its key to
|
||||
// strongRing_, dropping stale slots once they outnumber the live ones.
|
||||
// Caller holds mutex_.
|
||||
void
|
||||
queueStrong(key_type const& key, Entry& entry);
|
||||
|
||||
// CLOCK eviction over strongRing_: pops slots from the front, discards
|
||||
// stale ones, gives `keep` and entries used since they became strong a
|
||||
// second chance at the back, and demotes the first other strong entry,
|
||||
// repeating until the count is back under cacheHardCap_ or the demotion
|
||||
// budget is spent. Amortized O(1) per insert. No-op for key caches;
|
||||
// caller holds mutex_.
|
||||
void
|
||||
evictForHardCap(CacheType::Iterator const& keep);
|
||||
|
||||
[[nodiscard]] std::thread
|
||||
sweepHelper(
|
||||
ClockType::time_point const& whenExpire,
|
||||
@@ -390,8 +433,38 @@ private:
|
||||
// Desired maximum cache age
|
||||
ClockType::duration const targetAge_;
|
||||
|
||||
// Hard upper bound on strongly-cached entries, enforced by
|
||||
// evictForHardCap whenever the strong count grows (fresh inserts and
|
||||
// weak-to-strong revivals). 0 disables it (sweep-only sizing).
|
||||
int const cacheHardCap_;
|
||||
|
||||
// Total hard-cap evictions (under mutex_); the first marks saturation
|
||||
// onset for logging.
|
||||
std::uint64_t hardCapEvictions_{0};
|
||||
|
||||
// Number of items cached
|
||||
int cacheCount_{0};
|
||||
|
||||
// A key, the strong seq its entry had when the slot was queued, and the
|
||||
// time it was queued.
|
||||
struct StrongSlot
|
||||
{
|
||||
key_type key;
|
||||
std::uint64_t seq;
|
||||
ClockType::time_point since;
|
||||
};
|
||||
|
||||
// Keys in the order their entries became strong, consumed front-first
|
||||
// by evictForHardCap. Filled only when cacheHardCap_ > 0. A slot is
|
||||
// stale once its entry is gone, weak, or strong again under a newer seq.
|
||||
std::deque<StrongSlot> strongRing_;
|
||||
|
||||
// Source of strong seqs; advanced under mutex_.
|
||||
std::uint64_t strongSeqNext_{0};
|
||||
|
||||
// Slots examined by evictForHardCap; read by getEvictVisits.
|
||||
std::uint64_t evictVisits_{0};
|
||||
|
||||
CacheType cache_; // Hold strong reference to recent objects
|
||||
std::uint64_t hits_{0};
|
||||
std::uint64_t misses_{0};
|
||||
|
||||
@@ -57,7 +57,9 @@ inline TaggedCache<
|
||||
ClockType::duration expiration,
|
||||
ClockType& clock,
|
||||
beast::Journal journal,
|
||||
beast::insight::Collector::Ptr const& collector)
|
||||
beast::insight::Collector::Ptr const& collector,
|
||||
int cacheHardCap,
|
||||
std::optional<std::size_t> partitions)
|
||||
: journal_(journal)
|
||||
, clock_(clock)
|
||||
, stats_(
|
||||
@@ -67,6 +69,8 @@ inline TaggedCache<
|
||||
, name_(name)
|
||||
, targetSize_(size)
|
||||
, targetAge_(expiration)
|
||||
, cacheHardCap_(cacheHardCap)
|
||||
, cache_(partitions)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -137,6 +141,23 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
|
||||
return cache_.size();
|
||||
}
|
||||
|
||||
template <
|
||||
class Key,
|
||||
class T,
|
||||
bool IsKeyCache,
|
||||
class SharedWeakUnionPointer,
|
||||
class SharedPointerType,
|
||||
class Hash,
|
||||
class KeyEqual,
|
||||
class Mutex>
|
||||
inline std::uint64_t
|
||||
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
|
||||
getEvictVisits() const
|
||||
{
|
||||
std::scoped_lock const lock(mutex_);
|
||||
return evictVisits_;
|
||||
}
|
||||
|
||||
template <
|
||||
class Key,
|
||||
class T,
|
||||
@@ -170,6 +191,7 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
|
||||
{
|
||||
std::scoped_lock const lock(mutex_);
|
||||
cache_.clear();
|
||||
strongRing_.clear();
|
||||
cacheCount_ = 0;
|
||||
}
|
||||
|
||||
@@ -188,6 +210,7 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
|
||||
{
|
||||
std::scoped_lock const lock(mutex_);
|
||||
cache_.clear();
|
||||
strongRing_.clear();
|
||||
cacheCount_ = 0;
|
||||
hits_ = 0;
|
||||
misses_ = 0;
|
||||
@@ -219,6 +242,141 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
|
||||
return true;
|
||||
}
|
||||
|
||||
template <
|
||||
class Key,
|
||||
class T,
|
||||
bool IsKeyCache,
|
||||
class SharedWeakUnionPointer,
|
||||
class SharedPointerType,
|
||||
class Hash,
|
||||
class KeyEqual,
|
||||
class Mutex>
|
||||
inline void
|
||||
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
|
||||
addStrong(CacheType::Iterator const& it)
|
||||
{
|
||||
++cacheCount_;
|
||||
if constexpr (!IsKeyCache)
|
||||
{
|
||||
if (cacheHardCap_ > 0)
|
||||
{
|
||||
queueStrong(it->first, it->second);
|
||||
if (cacheCount_ > cacheHardCap_)
|
||||
evictForHardCap(it);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <
|
||||
class Key,
|
||||
class T,
|
||||
bool IsKeyCache,
|
||||
class SharedWeakUnionPointer,
|
||||
class SharedPointerType,
|
||||
class Hash,
|
||||
class KeyEqual,
|
||||
class Mutex>
|
||||
inline void
|
||||
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
|
||||
queueStrong(key_type const& key, Entry& entry)
|
||||
{
|
||||
entry.strongSeq = ++strongSeqNext_;
|
||||
strongRing_.push_back({key, entry.strongSeq, clock_.now()});
|
||||
|
||||
// Stale slots accumulate from sweep and del; drop them once
|
||||
// they outnumber the live ones so the ring stays O(strong entries).
|
||||
constexpr std::size_t kRingSlack = 1024;
|
||||
if (strongRing_.size() > 2 * static_cast<std::size_t>(cacheCount_) + kRingSlack)
|
||||
{
|
||||
std::erase_if(strongRing_, [this](StrongSlot const& s) {
|
||||
auto const it = cache_.find(s.key);
|
||||
return it == cache_.end() || it->second.isWeak() || it->second.strongSeq != s.seq;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
template <
|
||||
class Key,
|
||||
class T,
|
||||
bool IsKeyCache,
|
||||
class SharedWeakUnionPointer,
|
||||
class SharedPointerType,
|
||||
class Hash,
|
||||
class KeyEqual,
|
||||
class Mutex>
|
||||
inline void
|
||||
TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash, KeyEqual, Mutex>::
|
||||
evictForHardCap(CacheType::Iterator const& keep)
|
||||
{
|
||||
if constexpr (!IsKeyCache)
|
||||
{
|
||||
key_type const keepKey = keep->first;
|
||||
|
||||
// Growth paths raise the count by one at a time, so a few demotions
|
||||
// per call let eviction catch up without stalling them.
|
||||
constexpr int kMaxDemotionsPerCall = 8;
|
||||
|
||||
for (int demotions = 0; cacheCount_ > cacheHardCap_ && demotions < kMaxDemotionsPerCall;
|
||||
++demotions)
|
||||
{
|
||||
// Every strong entry owns one live slot, and only `keep` can be
|
||||
// re-queued twice in one call, so two passes over the ring reach
|
||||
// a victim whenever one exists.
|
||||
bool demoted = false;
|
||||
for (std::size_t budget = 2 * strongRing_.size();
|
||||
!demoted && budget > 0 && !strongRing_.empty();
|
||||
--budget)
|
||||
{
|
||||
StrongSlot const slot = strongRing_.front();
|
||||
strongRing_.pop_front();
|
||||
++evictVisits_;
|
||||
|
||||
auto it = cache_.find(slot.key);
|
||||
if (it == cache_.end() || it->second.isWeak() || it->second.strongSeq != slot.seq)
|
||||
continue; // stale: the entry is gone, weak, or re-queued since
|
||||
|
||||
if (slot.key == keepKey || it->second.lastAccess > slot.since)
|
||||
{
|
||||
// The newest entry stays; one used since it became
|
||||
// strong gets a second chance at the back.
|
||||
queueStrong(slot.key, it->second);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (it->second.ptr.useCount() == 1)
|
||||
{
|
||||
// Sole owner: release entirely.
|
||||
cache_.erase(it);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Others hold it: keep it weakly tracked.
|
||||
it->second.ptr.convertToWeak();
|
||||
}
|
||||
--cacheCount_;
|
||||
demoted = true;
|
||||
|
||||
// First eviction marks saturation onset; then a heartbeat
|
||||
// every 100k to avoid flooding.
|
||||
++hardCapEvictions_;
|
||||
if (hardCapEvictions_ == 1 || hardCapEvictions_ % 100000 == 0)
|
||||
{
|
||||
JLOG(journal_.warn()) << name_ << ": hard-cap eviction #" << hardCapEvictions_
|
||||
<< " (cap " << cacheHardCap_ << ", strong " << cacheCount_
|
||||
<< ") - cache saturated, growth now evicts";
|
||||
}
|
||||
}
|
||||
|
||||
if (!demoted)
|
||||
{
|
||||
JLOG(journal_.debug()) << name_ << ": over hard cap " << cacheHardCap_
|
||||
<< " but no strong entry other than the newest to demote";
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <
|
||||
class Key,
|
||||
class T,
|
||||
@@ -360,11 +518,14 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
|
||||
|
||||
if (cit == cache_.end())
|
||||
{
|
||||
cache_.emplace(
|
||||
std::piecewise_construct,
|
||||
std::forward_as_tuple(key),
|
||||
std::forward_as_tuple(clock_.now(), data));
|
||||
++cacheCount_;
|
||||
auto const emplacedIt = cache_
|
||||
.emplace(
|
||||
std::piecewise_construct,
|
||||
std::forward_as_tuple(key),
|
||||
std::forward_as_tuple(clock_.now(), data))
|
||||
.first;
|
||||
// The just-inserted entry is the newest; evictForHardCap keeps it.
|
||||
addStrong(emplacedIt);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -414,12 +575,12 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
|
||||
data = cachedData;
|
||||
}
|
||||
|
||||
++cacheCount_;
|
||||
addStrong(cit);
|
||||
return true;
|
||||
}
|
||||
|
||||
entry.ptr = data;
|
||||
++cacheCount_;
|
||||
addStrong(cit);
|
||||
|
||||
return false;
|
||||
}
|
||||
@@ -695,7 +856,13 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
|
||||
++misses_;
|
||||
auto const [it, inserted] = cache_.emplace(digest, Entry(clock_.now(), std::move(sle)));
|
||||
if (!inserted)
|
||||
{
|
||||
it->second.touch(clock_.now());
|
||||
}
|
||||
else
|
||||
{
|
||||
addStrong(it);
|
||||
}
|
||||
return it->second.ptr.getStrong();
|
||||
}
|
||||
// End CachedSLEs functions.
|
||||
@@ -728,7 +895,7 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
|
||||
if (entry.isCached())
|
||||
{
|
||||
// independent of cache size, so not counted as a hit
|
||||
++cacheCount_;
|
||||
addStrong(cit);
|
||||
entry.touch(clock_.now());
|
||||
return entry.ptr.getStrong();
|
||||
}
|
||||
|
||||
@@ -22,11 +22,14 @@ struct Sections
|
||||
static constexpr auto kIoWorkers = "io_workers";
|
||||
static constexpr auto kIps = "ips";
|
||||
static constexpr auto kIpsFixed = "ips_fixed";
|
||||
static constexpr auto kLedgerCacheAge = "ledger_cache_age";
|
||||
static constexpr auto kLedgerFetchSize = "ledger_fetch_size";
|
||||
static constexpr auto kLedgerHistory = "ledger_history";
|
||||
static constexpr auto kLedgerReplay = "ledger_replay";
|
||||
static constexpr auto kLedgerTxTables = "ledger_tx_tables";
|
||||
static constexpr auto kMaxSubscriptionsPerConnection = "max_subscriptions_per_connection";
|
||||
static constexpr auto kMaxTransactions = "max_transactions";
|
||||
static constexpr auto kMemoryLimit = "memory_limit";
|
||||
static constexpr auto kNetworkId = "network_id";
|
||||
static constexpr auto kNetworkQuorum = "network_quorum";
|
||||
static constexpr auto kNodeDatabase = "node_db";
|
||||
@@ -64,6 +67,7 @@ struct Sections
|
||||
static constexpr auto kSslVerifyFile = "ssl_verify_file";
|
||||
static constexpr auto kSweepInterval = "sweep_interval";
|
||||
static constexpr auto kTransactionQueue = "transaction_queue";
|
||||
static constexpr auto kTreeCacheAge = "tree_cache_age";
|
||||
static constexpr auto kValidationSeed = "validation_seed";
|
||||
static constexpr auto kValidatorKeys = "validator_keys";
|
||||
static constexpr auto kValidatorKeyRevocation = "validator_key_revocation";
|
||||
|
||||
@@ -1,713 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <xrpl/beast/utility/instrumentation.h>
|
||||
|
||||
#include <coroutine>
|
||||
#include <exception>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <variant>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
template <typename T = void>
|
||||
class CoroTask;
|
||||
|
||||
/**
|
||||
* CoroTask<void> -- coroutine return type for void-returning coroutines.
|
||||
*
|
||||
* Class / Dependency Diagram
|
||||
* ==========================
|
||||
*
|
||||
* CoroTask<void>
|
||||
* +-----------------------------------------------+
|
||||
* | - handle_ : Handle (coroutine_handle<promise>) |
|
||||
* +-----------------------------------------------+
|
||||
* | + handle(), done() |
|
||||
* | + await_ready/suspend/resume (Awaiter iface) |
|
||||
* +-----------------------------------------------+
|
||||
* | owns
|
||||
* v
|
||||
* promise_type
|
||||
* +-----------------------------------------------+
|
||||
* | - exception_ : std::exception_ptr |
|
||||
* | - continuation_ : std::coroutine_handle<> |
|
||||
* +-----------------------------------------------+
|
||||
* | + get_return_object() -> CoroTask |
|
||||
* | + initial_suspend() -> suspend_always (lazy) |
|
||||
* | + final_suspend() -> FinalAwaiter |
|
||||
* | + return_void() |
|
||||
* | + unhandled_exception() |
|
||||
* +-----------------------------------------------+
|
||||
* | returns at final_suspend
|
||||
* v
|
||||
* FinalAwaiter
|
||||
* +-----------------------------------------------+
|
||||
* | await_suspend(h): |
|
||||
* | if continuation_ set -> symmetric transfer |
|
||||
* | else -> noop_coroutine |
|
||||
* +-----------------------------------------------+
|
||||
*
|
||||
* Design Notes
|
||||
* ------------
|
||||
* - Lazy start: initial_suspend returns suspend_always, so the coroutine
|
||||
* body does not execute until the handle is explicitly resumed.
|
||||
* - Symmetric transfer: await_suspend returns a coroutine_handle instead
|
||||
* of void/bool, allowing the scheduler to jump directly to the next
|
||||
* coroutine without growing the call stack.
|
||||
* - Continuation chaining: when one CoroTask is co_await-ed inside
|
||||
* another, the caller's handle is stored as continuation_ so
|
||||
* FinalAwaiter can resume it when this task finishes.
|
||||
* - Move-only: the handle is exclusively owned; copy is deleted.
|
||||
*
|
||||
* Usage Examples
|
||||
* ==============
|
||||
*
|
||||
* 1. Basic void coroutine (the most common case in xrpld):
|
||||
*
|
||||
* CoroTask<void> doWork(std::shared_ptr<CoroTaskRunner> runner) {
|
||||
* // do something
|
||||
* co_await runner->suspend(); // yield control
|
||||
* // resumed later via runner->post() or runner->resume()
|
||||
* co_return;
|
||||
* }
|
||||
*
|
||||
* 2. co_await-ing one CoroTask<void> from another (chaining):
|
||||
*
|
||||
* CoroTask<void> inner() {
|
||||
* // ...
|
||||
* co_return;
|
||||
* }
|
||||
* CoroTask<void> outer() {
|
||||
* co_await inner(); // continuation_ links outer -> inner
|
||||
* co_return; // FinalAwaiter resumes outer
|
||||
* }
|
||||
*
|
||||
* 3. Exceptions propagate through co_await:
|
||||
*
|
||||
* CoroTask<void> failing() {
|
||||
* throw std::runtime_error("oops");
|
||||
* co_return;
|
||||
* }
|
||||
* CoroTask<void> caller() {
|
||||
* try { co_await failing(); }
|
||||
* catch (std::runtime_error const&) { // caught here }
|
||||
* }
|
||||
*
|
||||
* Caveats / Pitfalls
|
||||
* ==================
|
||||
*
|
||||
* BUG-RISK: Dangling references in coroutine parameters.
|
||||
* Coroutine parameters are copied into the frame, but references
|
||||
* are NOT -- they are stored as-is. If the referent goes out of scope
|
||||
* before the coroutine finishes, you get use-after-free.
|
||||
*
|
||||
* // BROKEN -- local dies before coroutine runs:
|
||||
* CoroTask<void> bad(int& ref) { co_return; }
|
||||
* void launch() {
|
||||
* int local = 42;
|
||||
* auto task = bad(local); // frame stores &local
|
||||
* } // local destroyed; frame holds dangling ref
|
||||
*
|
||||
* // FIX -- pass by value, or ensure lifetime via shared_ptr.
|
||||
*
|
||||
* BUG-RISK: GCC 14 corrupts reference captures in coroutine lambdas.
|
||||
* When a lambda that returns CoroTask captures by reference ([&]),
|
||||
* GCC 14 may generate a corrupted coroutine frame. Always capture
|
||||
* by explicit pointer-to-value instead:
|
||||
*
|
||||
* // BROKEN on GCC 14:
|
||||
* jq.postCoroTask(t, n, [&](auto) -> CoroTask<void> { ... });
|
||||
*
|
||||
* // FIX -- capture pointers explicitly:
|
||||
* jq.postCoroTask(t, n, [ptr = &val](auto) -> CoroTask<void> { ... });
|
||||
*
|
||||
* BUG-RISK: Resuming a destroyed or completed CoroTask.
|
||||
* Calling handle().resume() after the coroutine has already run to
|
||||
* completion (done() == true) is undefined behavior. The CoroTaskRunner
|
||||
* guards against this with an XRPL_ASSERT, but standalone usage of
|
||||
* CoroTask must check done() before resuming.
|
||||
*
|
||||
* BUG-RISK: Moving a CoroTask that is being awaited.
|
||||
* If task A is co_await-ed by task B (so A.continuation_ == B), moving
|
||||
* or destroying A will invalidate the continuation link. Never move
|
||||
* or reassign a CoroTask while it is mid-execution or being awaited.
|
||||
*
|
||||
* LIMITATION: CoroTask is fire-and-forget for the top-level owner.
|
||||
* There is no built-in notification when the coroutine finishes.
|
||||
* The caller must use external synchronization (e.g. CoroTaskRunner::join
|
||||
* or a gate/condition_variable) to know when it is done.
|
||||
*
|
||||
* LIMITATION: No cancellation token.
|
||||
* There is no way to cancel a suspended CoroTask from outside. The
|
||||
* coroutine body must cooperatively check a flag (e.g. jq_.isStopping())
|
||||
* after each co_await and co_return early if needed.
|
||||
*
|
||||
* LIMITATION: Stackless -- cannot suspend from nested non-coroutine calls.
|
||||
* If a coroutine calls a regular function that wants to "yield", it
|
||||
* cannot. Only the immediate coroutine body can use co_await.
|
||||
* This is acceptable for xrpld because all yield() sites are shallow.
|
||||
*/
|
||||
template <>
|
||||
class CoroTask<void>
|
||||
{
|
||||
public:
|
||||
// The C++ coroutine protocol mandates these names (promise_type,
|
||||
// initial_suspend, await_ready, ...) and instance-callable awaiter
|
||||
// methods, which conflict with the project naming/static conventions.
|
||||
// NOLINTBEGIN(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
struct promise_type;
|
||||
using Handle = std::coroutine_handle<promise_type>;
|
||||
|
||||
/**
|
||||
* Coroutine promise. Compiler uses this to manage coroutine state.
|
||||
* Stores the exception (if any) and the continuation handle for
|
||||
* symmetric transfer back to the awaiting coroutine.
|
||||
*/
|
||||
struct promise_type
|
||||
{
|
||||
// Captured exception from the coroutine body, rethrown in
|
||||
// await_resume() when this task is co_await-ed by a caller.
|
||||
std::exception_ptr exception_;
|
||||
|
||||
// Handle to the coroutine that is co_await-ing this task.
|
||||
// Set by await_suspend(). FinalAwaiter uses it for symmetric
|
||||
// transfer back to the caller. Null if this is a top-level task.
|
||||
std::coroutine_handle<> continuation_;
|
||||
|
||||
/**
|
||||
* Create the CoroTask return object.
|
||||
* Called by the compiler at coroutine creation.
|
||||
*/
|
||||
CoroTask
|
||||
get_return_object()
|
||||
{
|
||||
return CoroTask{Handle::from_promise(*this)};
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazy start. The coroutine body does not execute until the
|
||||
* handle is explicitly resumed (e.g. by CoroTaskRunner::resume).
|
||||
*/
|
||||
std::suspend_always
|
||||
initial_suspend() noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Awaiter returned by final_suspend(). Uses symmetric transfer:
|
||||
* if a continuation exists, transfers control directly to it
|
||||
* (tail-call, no stack growth). Otherwise returns noop_coroutine
|
||||
* so the coroutine frame stays alive for the owner to destroy.
|
||||
*/
|
||||
struct FinalAwaiter
|
||||
{
|
||||
/**
|
||||
* Always false. We need await_suspend to run for
|
||||
* symmetric transfer.
|
||||
*/
|
||||
bool
|
||||
await_ready() noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Symmetric transfer: returns the continuation handle so
|
||||
* the compiler emits a tail-call instead of a nested resume.
|
||||
* If no continuation is set, returns noop_coroutine to
|
||||
* suspend at final_suspend without destroying the frame.
|
||||
*
|
||||
* @param h Handle to this completing coroutine
|
||||
*
|
||||
* @return Continuation handle, or noop_coroutine
|
||||
*/
|
||||
std::coroutine_handle<>
|
||||
await_suspend(Handle h) noexcept
|
||||
{
|
||||
if (auto cont = h.promise().continuation_)
|
||||
return cont;
|
||||
return std::noop_coroutine();
|
||||
}
|
||||
|
||||
void
|
||||
await_resume() noexcept
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns FinalAwaiter for symmetric transfer at coroutine end.
|
||||
*/
|
||||
FinalAwaiter
|
||||
final_suspend() noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Called by the compiler for `co_return;` (void coroutine).
|
||||
*/
|
||||
void
|
||||
return_void()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Called by the compiler when an exception escapes the coroutine
|
||||
* body. Captures it for later rethrowing in await_resume().
|
||||
*/
|
||||
void
|
||||
unhandled_exception()
|
||||
{
|
||||
exception_ = std::current_exception();
|
||||
}
|
||||
};
|
||||
// NOLINTEND(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
|
||||
/**
|
||||
* Default constructor. Creates an empty (null handle) task.
|
||||
*/
|
||||
CoroTask() = default;
|
||||
|
||||
/**
|
||||
* Takes ownership of a compiler-generated coroutine handle.
|
||||
*
|
||||
* @param h Coroutine handle to own
|
||||
*/
|
||||
explicit CoroTask(Handle h) : handle_(h)
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroys the coroutine frame if this task owns one.
|
||||
*/
|
||||
~CoroTask()
|
||||
{
|
||||
if (handle_)
|
||||
handle_.destroy();
|
||||
}
|
||||
|
||||
/**
|
||||
* Move constructor. Transfers handle ownership, leaves other empty.
|
||||
*/
|
||||
CoroTask(CoroTask&& other) noexcept : handle_(std::exchange(other.handle_, {}))
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Move assignment. Destroys current frame (if any), takes other's.
|
||||
*/
|
||||
CoroTask&
|
||||
operator=(CoroTask&& other) noexcept
|
||||
{
|
||||
if (this != &other)
|
||||
{
|
||||
if (handle_)
|
||||
handle_.destroy();
|
||||
handle_ = std::exchange(other.handle_, {});
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
CoroTask(CoroTask const&) = delete;
|
||||
CoroTask&
|
||||
operator=(CoroTask const&) = delete;
|
||||
|
||||
/**
|
||||
* @return The underlying coroutine_handle
|
||||
*/
|
||||
[[nodiscard]] Handle
|
||||
handle() const
|
||||
{
|
||||
return handle_;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if the coroutine has run to completion (or thrown)
|
||||
*/
|
||||
[[nodiscard]] bool
|
||||
done() const
|
||||
{
|
||||
return handle_ && handle_.done();
|
||||
}
|
||||
|
||||
// -- Awaiter interface: allows `co_await someCoroTask;` --
|
||||
|
||||
/**
|
||||
* Always false. This task is lazy, so co_await always suspends
|
||||
* the caller to set up the continuation link.
|
||||
*/
|
||||
// NOLINTBEGIN(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
[[nodiscard]] bool
|
||||
await_ready() const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores the caller's handle as our continuation, then returns
|
||||
* our handle for symmetric transfer (caller suspends, we resume).
|
||||
*
|
||||
* @param caller Handle of the coroutine doing co_await on us
|
||||
*
|
||||
* @return Our handle for symmetric transfer
|
||||
*/
|
||||
std::coroutine_handle<>
|
||||
await_suspend(std::coroutine_handle<> caller) noexcept
|
||||
{
|
||||
XRPL_ASSERT(handle_, "xrpl::CoroTask<void>::await_suspend : handle is valid");
|
||||
handle_.promise().continuation_ = caller;
|
||||
return handle_; // Symmetric transfer
|
||||
}
|
||||
|
||||
/**
|
||||
* Called in the awaiting coroutine's context after this task
|
||||
* completes. Rethrows any exception captured by
|
||||
* unhandled_exception().
|
||||
*/
|
||||
void
|
||||
await_resume()
|
||||
{
|
||||
XRPL_ASSERT(handle_, "xrpl::CoroTask<void>::await_resume : handle is valid");
|
||||
if (auto& ep = handle_.promise().exception_)
|
||||
std::rethrow_exception(ep);
|
||||
}
|
||||
// NOLINTEND(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
|
||||
private:
|
||||
// Exclusively-owned coroutine handle. Null after move or default
|
||||
// construction. Destroyed in the destructor.
|
||||
Handle handle_;
|
||||
};
|
||||
|
||||
/**
|
||||
* CoroTask<T> -- coroutine return type for value-returning coroutines.
|
||||
*
|
||||
* Class / Dependency Diagram
|
||||
* ==========================
|
||||
*
|
||||
* CoroTask<T>
|
||||
* +-----------------------------------------------+
|
||||
* | - handle_ : Handle (coroutine_handle<promise>) |
|
||||
* +-----------------------------------------------+
|
||||
* | + handle(), done() |
|
||||
* | + await_ready/suspend/resume (Awaiter iface) |
|
||||
* +-----------------------------------------------+
|
||||
* | owns
|
||||
* v
|
||||
* promise_type
|
||||
* +-----------------------------------------------+
|
||||
* | - result_ : variant<monostate, T, |
|
||||
* | exception_ptr> |
|
||||
* | - continuation_ : std::coroutine_handle<> |
|
||||
* +-----------------------------------------------+
|
||||
* | + get_return_object() -> CoroTask |
|
||||
* | + initial_suspend() -> suspend_always (lazy) |
|
||||
* | + final_suspend() -> FinalAwaiter |
|
||||
* | + return_value(T) -> stores in result_[1] |
|
||||
* | + unhandled_exception -> stores in result_[2] |
|
||||
* +-----------------------------------------------+
|
||||
* | returns at final_suspend
|
||||
* v
|
||||
* FinalAwaiter (same symmetric-transfer pattern as CoroTask<void>)
|
||||
*
|
||||
* Value Extraction
|
||||
* ----------------
|
||||
* await_resume() inspects the variant:
|
||||
* - index 2 (exception_ptr) -> rethrow
|
||||
* - index 1 (T) -> return value via move
|
||||
*
|
||||
* Usage Examples
|
||||
* ==============
|
||||
*
|
||||
* 1. Simple value return:
|
||||
*
|
||||
* CoroTask<int> computeAnswer() { co_return 42; }
|
||||
*
|
||||
* CoroTask<void> caller() {
|
||||
* int v = co_await computeAnswer(); // v == 42
|
||||
* }
|
||||
*
|
||||
* 2. Chaining value-returning coroutines:
|
||||
*
|
||||
* CoroTask<int> add(int a, int b) { co_return a + b; }
|
||||
* CoroTask<int> doubleSum(int a, int b) {
|
||||
* int s = co_await add(a, b);
|
||||
* co_return s * 2;
|
||||
* }
|
||||
*
|
||||
* 3. Exception propagation from inner to outer:
|
||||
*
|
||||
* CoroTask<int> failing() {
|
||||
* throw std::runtime_error("bad");
|
||||
* co_return 0; // never reached
|
||||
* }
|
||||
* CoroTask<void> caller() {
|
||||
* try {
|
||||
* int v = co_await failing(); // throws here
|
||||
* } catch (std::runtime_error const& e) {
|
||||
* // e.what() == "bad"
|
||||
* }
|
||||
* }
|
||||
*
|
||||
* Caveats / Pitfalls (in addition to CoroTask<void> caveats above)
|
||||
* ================================================================
|
||||
*
|
||||
* BUG-RISK: await_resume() moves the value out of the variant.
|
||||
* Calling co_await on the same CoroTask<T> instance twice is undefined
|
||||
* behavior -- the second call will see a moved-from T. CoroTask is
|
||||
* single-shot: one co_return, one co_await.
|
||||
*
|
||||
* BUG-RISK: T must be move-constructible.
|
||||
* return_value(T) takes by value and moves into the variant.
|
||||
* Types that are not movable cannot be used as T.
|
||||
*
|
||||
* LIMITATION: No co_yield support.
|
||||
* CoroTask<T> only supports a single co_return. It does not implement
|
||||
* yield_value(), so using co_yield inside a CoroTask<T> coroutine is a
|
||||
* compile error. For streaming values, a different return type
|
||||
* (e.g. Generator<T>) would be needed.
|
||||
*
|
||||
* LIMITATION: Result is only accessible via co_await.
|
||||
* There is no .get() or .result() method. The value can only be
|
||||
* extracted by co_await-ing the CoroTask<T> from inside another
|
||||
* coroutine. For extracting results in non-coroutine code, pass a
|
||||
* pointer to the caller and write through it (as the tests do).
|
||||
*/
|
||||
template <typename T>
|
||||
class CoroTask
|
||||
{
|
||||
static_assert(
|
||||
std::is_move_constructible_v<T>,
|
||||
"CoroTask<T> requires T to be move-constructible");
|
||||
|
||||
public:
|
||||
// The C++ coroutine protocol mandates these names (promise_type,
|
||||
// initial_suspend, await_ready, ...) and instance-callable awaiter
|
||||
// methods, which conflict with the project naming/static conventions.
|
||||
// NOLINTBEGIN(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
struct promise_type;
|
||||
using Handle = std::coroutine_handle<promise_type>;
|
||||
|
||||
/**
|
||||
* Coroutine promise for value-returning coroutines.
|
||||
* Stores the result as a variant: monostate (not yet set),
|
||||
* T (co_return value), or exception_ptr (unhandled exception).
|
||||
*/
|
||||
struct promise_type
|
||||
{
|
||||
// Tri-state result:
|
||||
// index 0 (monostate) -- coroutine has not yet completed
|
||||
// index 1 (T) -- co_return value stored here
|
||||
// index 2 (exception) -- unhandled exception captured here
|
||||
std::variant<std::monostate, T, std::exception_ptr> result_;
|
||||
|
||||
// Handle to the coroutine co_await-ing this task. Used by
|
||||
// FinalAwaiter for symmetric transfer. Null for top-level tasks.
|
||||
std::coroutine_handle<> continuation_;
|
||||
|
||||
/**
|
||||
* Create the CoroTask return object.
|
||||
* Called by the compiler at coroutine creation.
|
||||
*/
|
||||
CoroTask
|
||||
get_return_object()
|
||||
{
|
||||
return CoroTask{Handle::from_promise(*this)};
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazy start. Coroutine body does not run until explicitly resumed.
|
||||
*/
|
||||
std::suspend_always
|
||||
initial_suspend() noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Symmetric-transfer awaiter at coroutine completion.
|
||||
* Same pattern as CoroTask<void>::FinalAwaiter.
|
||||
*/
|
||||
struct FinalAwaiter
|
||||
{
|
||||
bool
|
||||
await_ready() noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns continuation for symmetric transfer, or
|
||||
* noop_coroutine if this is a top-level task.
|
||||
*
|
||||
* @param h Handle to this completing coroutine
|
||||
*
|
||||
* @return Continuation handle, or noop_coroutine
|
||||
*/
|
||||
std::coroutine_handle<>
|
||||
await_suspend(Handle h) noexcept
|
||||
{
|
||||
if (auto cont = h.promise().continuation_)
|
||||
return cont;
|
||||
return std::noop_coroutine();
|
||||
}
|
||||
|
||||
void
|
||||
await_resume() noexcept
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
FinalAwaiter
|
||||
final_suspend() noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Called by the compiler for `co_return value;`.
|
||||
* Moves the value into result_ at index 1.
|
||||
*
|
||||
* @param value The value to store
|
||||
*/
|
||||
void
|
||||
return_value(T value)
|
||||
{
|
||||
result_.template emplace<1>(std::move(value));
|
||||
}
|
||||
|
||||
/**
|
||||
* Captures unhandled exceptions at index 2 of result_.
|
||||
* Rethrown later in await_resume().
|
||||
*/
|
||||
void
|
||||
unhandled_exception()
|
||||
{
|
||||
result_.template emplace<2>(std::current_exception());
|
||||
}
|
||||
};
|
||||
// NOLINTEND(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
|
||||
/**
|
||||
* Default constructor. Creates an empty (null handle) task.
|
||||
*/
|
||||
CoroTask() = default;
|
||||
|
||||
/**
|
||||
* Takes ownership of a compiler-generated coroutine handle.
|
||||
*
|
||||
* @param h Coroutine handle to own
|
||||
*/
|
||||
explicit CoroTask(Handle h) : handle_(h)
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroys the coroutine frame if this task owns one.
|
||||
*/
|
||||
~CoroTask()
|
||||
{
|
||||
if (handle_)
|
||||
handle_.destroy();
|
||||
}
|
||||
|
||||
/**
|
||||
* Move constructor. Transfers handle ownership, leaves other empty.
|
||||
*/
|
||||
CoroTask(CoroTask&& other) noexcept : handle_(std::exchange(other.handle_, {}))
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Move assignment. Destroys current frame (if any), takes other's.
|
||||
*/
|
||||
CoroTask&
|
||||
operator=(CoroTask&& other) noexcept
|
||||
{
|
||||
if (this != &other)
|
||||
{
|
||||
if (handle_)
|
||||
handle_.destroy();
|
||||
handle_ = std::exchange(other.handle_, {});
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
CoroTask(CoroTask const&) = delete;
|
||||
CoroTask&
|
||||
operator=(CoroTask const&) = delete;
|
||||
|
||||
/**
|
||||
* @return The underlying coroutine_handle
|
||||
*/
|
||||
[[nodiscard]] Handle
|
||||
handle() const
|
||||
{
|
||||
return handle_;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if the coroutine has run to completion (or thrown)
|
||||
*/
|
||||
[[nodiscard]] bool
|
||||
done() const
|
||||
{
|
||||
return handle_ && handle_.done();
|
||||
}
|
||||
|
||||
// -- Awaiter interface: allows `T val = co_await someCoroTask;` --
|
||||
|
||||
/**
|
||||
* Always false. co_await always suspends to set up continuation.
|
||||
*/
|
||||
// NOLINTBEGIN(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
[[nodiscard]] bool
|
||||
await_ready() const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores caller as continuation, returns our handle for
|
||||
* symmetric transfer.
|
||||
*
|
||||
* @param caller Handle of the coroutine doing co_await on us
|
||||
*
|
||||
* @return Our handle for symmetric transfer
|
||||
*/
|
||||
std::coroutine_handle<>
|
||||
await_suspend(std::coroutine_handle<> caller) noexcept
|
||||
{
|
||||
XRPL_ASSERT(handle_, "xrpl::CoroTask<T>::await_suspend : handle is valid");
|
||||
handle_.promise().continuation_ = caller;
|
||||
return handle_;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the result: rethrows if exception, otherwise moves
|
||||
* the T value out of the variant. Single-shot: calling twice
|
||||
* on the same task is undefined (moved-from T).
|
||||
*
|
||||
* @return The co_return-ed value
|
||||
*/
|
||||
T
|
||||
await_resume()
|
||||
{
|
||||
XRPL_ASSERT(handle_, "xrpl::CoroTask<T>::await_resume : handle is valid");
|
||||
auto& result = handle_.promise().result_;
|
||||
if (auto* ep = std::get_if<2>(&result))
|
||||
std::rethrow_exception(*ep);
|
||||
return std::get<1>(std::move(result));
|
||||
}
|
||||
// NOLINTEND(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
|
||||
private:
|
||||
// Exclusively-owned coroutine handle. Null after move or default
|
||||
// construction. Destroyed in the destructor.
|
||||
Handle handle_;
|
||||
};
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -1,448 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
/**
|
||||
* @file CoroTaskRunner.ipp
|
||||
*
|
||||
* CoroTaskRunner inline implementation.
|
||||
*
|
||||
* This file contains the business logic for managing C++20 coroutines
|
||||
* on the JobQueue. It is included at the bottom of JobQueue.h.
|
||||
*
|
||||
* Data Flow: suspend / post / resume cycle
|
||||
* =========================================
|
||||
*
|
||||
* coroutine body CoroTaskRunner JobQueue
|
||||
* -------------- -------------- --------
|
||||
* |
|
||||
* co_await runner->suspend()
|
||||
* |
|
||||
* +--- await_suspend ------> onSuspend()
|
||||
* | ++nSuspend_ ------------> nSuspend_
|
||||
* | [coroutine is now suspended]
|
||||
* |
|
||||
* . (externally or by yieldAndPost())
|
||||
* .
|
||||
* +--- (caller calls) -----> post()
|
||||
* | ++runCount_
|
||||
* | addJob(resume) ----------> job enqueued
|
||||
* | |
|
||||
* | [worker picks up]
|
||||
* | |
|
||||
* +--- <----- resume() <-----------------------------------+
|
||||
* | --nSuspend_ ------> nSuspend_
|
||||
* | swap in LocalValues (lvs_)
|
||||
* | task_.handle().resume()
|
||||
* | |
|
||||
* | [coroutine body continues here]
|
||||
* | |
|
||||
* | swap out LocalValues
|
||||
* | --runCount_
|
||||
* | cv_.notify_all()
|
||||
* v
|
||||
*
|
||||
* Thread Safety
|
||||
* =============
|
||||
* - mutex_ : guards task_.handle().resume() so that post()-before-suspend
|
||||
* races cannot resume the coroutine while it is still running.
|
||||
* (See the race condition discussion in JobQueue.h)
|
||||
* - mutexRun_ : guards runCount_ counter; used by join() to wait until
|
||||
* all in-flight resume operations complete.
|
||||
* - jq_.mutex_: guards nSuspend_ increments/decrements.
|
||||
*
|
||||
* Common Mistakes When Modifying This File
|
||||
* =========================================
|
||||
*
|
||||
* 1. Changing lock ordering.
|
||||
* resume() acquires locks sequentially (never held simultaneously):
|
||||
* jq_.mutex_ (released immediately), then mutex_ (held across resume),
|
||||
* then mutexRun_ (released after decrement). post() acquires only
|
||||
* mutexRun_. Any new code path must follow the same order.
|
||||
*
|
||||
* 2. Removing the shared_from_this() capture in post().
|
||||
* The lambda passed to addJob captures [this, sp = shared_from_this()].
|
||||
* If you remove sp, 'this' can be destroyed before the job runs,
|
||||
* causing use-after-free. The sp capture is load-bearing.
|
||||
*
|
||||
* 3. Forgetting to decrement nSuspend_ on a new code path.
|
||||
* Every ++nSuspend_ must have a matching --nSuspend_. If you add a new
|
||||
* suspension path (e.g. a new awaiter) and forget to decrement on resume
|
||||
* or on failure, JobQueue::stop() will hang.
|
||||
*
|
||||
* 4. Calling task_.handle().resume() without holding mutex_.
|
||||
* This allows a race where the coroutine runs on two threads
|
||||
* simultaneously. Always hold mutex_ around resume().
|
||||
*
|
||||
* 5. Swapping LocalValues outside of the mutex_ critical section.
|
||||
* The swap-in and swap-out of LocalValues must bracket the resume()
|
||||
* call. If you move the swap-out before the lock_guard(mutex_) is
|
||||
* released, you break LocalValue isolation for any code that runs
|
||||
* after the coroutine suspends but before the lock is dropped.
|
||||
*/
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
/**
|
||||
* Construct a CoroTaskRunner. Sets runCount_ to 0; does not
|
||||
* create the coroutine. Call init() afterwards.
|
||||
*
|
||||
* @param jq The JobQueue this coroutine will run on
|
||||
* @param type Job type for scheduling priority
|
||||
* @param name Human-readable name for logging
|
||||
*/
|
||||
inline JobQueue::CoroTaskRunner::CoroTaskRunner(
|
||||
CreateT,
|
||||
JobQueue& jq,
|
||||
JobType type,
|
||||
std::string name)
|
||||
: jq_(jq), type_(type), name_(std::move(name))
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize with a coroutine-returning callable.
|
||||
* Stores the callable on the heap (FuncStore) so it outlives the
|
||||
* coroutine frame. Coroutine frames store a reference to the
|
||||
* callable's implicit object parameter (the lambda). If the callable
|
||||
* is a temporary, that reference dangles after the caller returns.
|
||||
* Keeping the callable alive here ensures the coroutine's captures
|
||||
* remain valid.
|
||||
*
|
||||
* @param f Callable: CoroTask<void>(shared_ptr<CoroTaskRunner>)
|
||||
*/
|
||||
template <class F>
|
||||
void
|
||||
JobQueue::CoroTaskRunner::init(F&& f)
|
||||
{
|
||||
using Fn = std::decay_t<F>;
|
||||
auto store = std::make_unique<FuncStore<Fn>>(std::forward<F>(f));
|
||||
task_ = store->func(shared_from_this());
|
||||
storedFunc_ = std::move(store);
|
||||
}
|
||||
|
||||
/**
|
||||
* Destructor. Waits for any in-flight resume() to complete, then
|
||||
* asserts (debug) that the coroutine has finished or
|
||||
* expectEarlyExit() was called.
|
||||
*
|
||||
* The join() call is necessary because with async dispatch the
|
||||
* coroutine runs on a worker thread. The gate signal (which wakes
|
||||
* the test thread) can arrive before resume() has set finished_.
|
||||
* join() synchronizes via mutexRun_, establishing a happens-before
|
||||
* edge: finished_ = true -> unlock(mutexRun_) in resume() ->
|
||||
* lock(mutexRun_) in join() -> read finished_.
|
||||
*/
|
||||
inline JobQueue::CoroTaskRunner::~CoroTaskRunner()
|
||||
{
|
||||
#ifndef NDEBUG
|
||||
join();
|
||||
XRPL_ASSERT(finished_, "xrpl::JobQueue::CoroTaskRunner::~CoroTaskRunner : is finished");
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment the JobQueue's suspended-coroutine count (nSuspend_).
|
||||
*/
|
||||
inline void
|
||||
JobQueue::CoroTaskRunner::onSuspend()
|
||||
{
|
||||
std::scoped_lock const lock(jq_.mutex_);
|
||||
++jq_.nSuspend_;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrement nSuspend_ without resuming.
|
||||
*/
|
||||
inline void
|
||||
JobQueue::CoroTaskRunner::onUndoSuspend()
|
||||
{
|
||||
std::scoped_lock const lock(jq_.mutex_);
|
||||
--jq_.nSuspend_;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return a SuspendAwaiter whose await_suspend() increments nSuspend_
|
||||
* before the coroutine actually suspends. The caller must later call
|
||||
* post() or resume() to continue execution.
|
||||
*
|
||||
* @return Awaiter for use with `co_await runner->suspend()`
|
||||
*/
|
||||
inline auto
|
||||
JobQueue::CoroTaskRunner::suspend()
|
||||
{
|
||||
/**
|
||||
* Custom awaiter for suspend(). Always suspends (await_ready
|
||||
* returns false) and increments nSuspend_ in await_suspend().
|
||||
*/
|
||||
// The C++ coroutine protocol mandates these awaiter names and
|
||||
// instance-callable methods, which conflict with the project
|
||||
// naming/static conventions.
|
||||
// NOLINTBEGIN(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
struct SuspendAwaiter
|
||||
{
|
||||
CoroTaskRunner& runner_; // The runner that owns this coroutine.
|
||||
|
||||
/**
|
||||
* Always returns false so the coroutine suspends.
|
||||
*/
|
||||
[[nodiscard]] bool
|
||||
await_ready() const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when the coroutine suspends. Increments nSuspend_
|
||||
* so the JobQueue knows a coroutine is waiting.
|
||||
*/
|
||||
void
|
||||
await_suspend(std::coroutine_handle<>) const
|
||||
{
|
||||
runner_.onSuspend();
|
||||
}
|
||||
|
||||
void
|
||||
await_resume() const noexcept
|
||||
{
|
||||
}
|
||||
};
|
||||
// NOLINTEND(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
return SuspendAwaiter{*this};
|
||||
}
|
||||
|
||||
/**
|
||||
* Suspend and immediately repost on the JobQueue. Equivalent to
|
||||
* `co_await JobQueueAwaiter{runner}` but uses an inline struct
|
||||
* to work around a GCC-12 codegen bug (see declaration in JobQueue.h).
|
||||
*
|
||||
* If the JobQueue is stopping (post fails), the suspend count is
|
||||
* undone and the coroutine continues immediately via symmetric
|
||||
* transfer back to its own handle.
|
||||
*
|
||||
* @return An inline YieldPostAwaiter
|
||||
*/
|
||||
inline auto
|
||||
JobQueue::CoroTaskRunner::yieldAndPost()
|
||||
{
|
||||
// The C++ coroutine protocol mandates these awaiter names and
|
||||
// instance-callable methods, which conflict with the project
|
||||
// naming/static conventions.
|
||||
// NOLINTBEGIN(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
struct YieldPostAwaiter
|
||||
{
|
||||
CoroTaskRunner& runner_;
|
||||
|
||||
[[nodiscard]] bool
|
||||
await_ready() const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a coroutine_handle<> (symmetric transfer) rather than
|
||||
* void + h.resume(). Two reasons:
|
||||
*
|
||||
* 1. h.resume() runs the coroutine nested inside this frame. A
|
||||
* coroutine that yields in a loop against a stopping JobQueue
|
||||
* fails post() every iteration, so the stack grows without
|
||||
* bound. Symmetric transfer is a tail call and does not nest.
|
||||
*
|
||||
* 2. After h.resume() returns, the coroutine may have completed
|
||||
* and destroyed its frame -- the frame this awaiter lives in.
|
||||
* Returning from await_suspend would then touch freed memory.
|
||||
*
|
||||
* A bool return would also avoid nesting, but GCC-12 miscompiles
|
||||
* bool-returning await_suspend (see JobQueueAwaiter.h).
|
||||
*
|
||||
* @return noop_coroutine() to stay suspended (job posted);
|
||||
* the caller's handle to continue now (JQ stopping)
|
||||
*/
|
||||
std::coroutine_handle<>
|
||||
await_suspend(std::coroutine_handle<> h)
|
||||
{
|
||||
runner_.onSuspend();
|
||||
if (!runner_.post())
|
||||
{
|
||||
runner_.onUndoSuspend();
|
||||
return h;
|
||||
}
|
||||
return std::noop_coroutine();
|
||||
}
|
||||
|
||||
void
|
||||
await_resume() const noexcept
|
||||
{
|
||||
}
|
||||
};
|
||||
// NOLINTEND(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
return YieldPostAwaiter{*this};
|
||||
}
|
||||
|
||||
/**
|
||||
* Schedule coroutine resumption as a job on the JobQueue.
|
||||
* A shared_ptr capture (sp) prevents this CoroTaskRunner from being
|
||||
* destroyed while the job is queued but not yet executed.
|
||||
*
|
||||
* @return false if the JobQueue rejected the job (shutting down)
|
||||
*/
|
||||
inline bool
|
||||
JobQueue::CoroTaskRunner::post()
|
||||
{
|
||||
{
|
||||
std::scoped_lock const lk(mutexRun_);
|
||||
++runCount_;
|
||||
}
|
||||
|
||||
// sp prevents 'this' from being destroyed while the job is pending
|
||||
if (jq_.addJob(type_, name_, [this, sp = shared_from_this()]() { resume(); }))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// The coroutine will not run. Undo the runCount_ increment.
|
||||
std::scoped_lock const lk(mutexRun_);
|
||||
--runCount_;
|
||||
cv_.notify_all();
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resume the coroutine on the current thread.
|
||||
*
|
||||
* Steps:
|
||||
* 1. Decrement nSuspend_ (under jq_.mutex_)
|
||||
* 2. Swap in this coroutine's LocalValues for thread-local isolation
|
||||
* 3. Resume the coroutine handle (under mutex_)
|
||||
* 4. Swap out LocalValues, restoring the thread's previous state
|
||||
* 5. Decrement runCount_ and notify join() waiters
|
||||
*
|
||||
* @pre post() must have been called before resume(). Direct calls
|
||||
* without a prior post() will corrupt runCount_ and break join().
|
||||
* Note: runCount_ is NOT incremented here — post() already did that.
|
||||
* This ensures join() stays blocked for the entire post->resume lifetime.
|
||||
*/
|
||||
inline void
|
||||
JobQueue::CoroTaskRunner::resume()
|
||||
{
|
||||
{
|
||||
std::scoped_lock const lock(jq_.mutex_);
|
||||
--jq_.nSuspend_;
|
||||
}
|
||||
auto saved = detail::getLocalValues().release();
|
||||
detail::getLocalValues().reset(&lvs_);
|
||||
std::scoped_lock const lock(mutex_);
|
||||
XRPL_ASSERT(
|
||||
task_.handle() && !task_.done(),
|
||||
"xrpl::JobQueue::CoroTaskRunner::resume : task handle is valid and not done");
|
||||
if (task_.handle() && !task_.done())
|
||||
{
|
||||
task_.handle().resume();
|
||||
}
|
||||
else
|
||||
{
|
||||
// A resume() with no coroutine to run (e.g. a duplicate external
|
||||
// post() after completion). Resuming a null or finished handle is
|
||||
// undefined behavior, so skip it -- this matches the old
|
||||
// Coro::resume() `if (coro_)` guard. The bookkeeping below still
|
||||
// runs to balance the ++runCount_ done by the post() that
|
||||
// scheduled this call.
|
||||
JLOG(jq_.journal_.warn())
|
||||
<< "CoroTaskRunner::resume called for coroutine '" << name_
|
||||
<< "' with no runnable coroutine (duplicate post or already completed)";
|
||||
}
|
||||
detail::getLocalValues().release();
|
||||
detail::getLocalValues().reset(saved);
|
||||
if (task_.done())
|
||||
{
|
||||
finished_ = true;
|
||||
// An exception that escapes a top-level coroutine body is captured
|
||||
// by promise_type::unhandled_exception() but has no awaiter to
|
||||
// rethrow it, so it would vanish with the frame. Surface it in the
|
||||
// log. (The old Boost path propagated it out of resume() instead.)
|
||||
if (auto const& ep = task_.handle().promise().exception_)
|
||||
{
|
||||
try
|
||||
{
|
||||
std::rethrow_exception(ep);
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
JLOG(jq_.journal_.error())
|
||||
<< "Unhandled exception in coroutine '" << name_ << "': " << e.what();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
JLOG(jq_.journal_.error())
|
||||
<< "Unhandled non-standard exception in coroutine '" << name_ << "'";
|
||||
}
|
||||
}
|
||||
// Break the shared_ptr cycle: frame -> shared_ptr<runner> -> this.
|
||||
// Use std::move (not task_ = {}) so task_.handle_ is null BEFORE the
|
||||
// frame is destroyed. operator= would destroy the frame while handle_
|
||||
// still holds the old value -- a re-entrancy hazard on GCC-12 if
|
||||
// frame destruction triggers runner cleanup.
|
||||
[[maybe_unused]] auto completed = std::move(task_);
|
||||
}
|
||||
std::scoped_lock const lk(mutexRun_);
|
||||
--runCount_;
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if the coroutine has not yet run to completion
|
||||
*/
|
||||
inline bool
|
||||
JobQueue::CoroTaskRunner::runnable() const
|
||||
{
|
||||
// After normal completion, task_ is reset to break the shared_ptr cycle
|
||||
// (handle_ becomes null). A null handle means the coroutine is done.
|
||||
return task_.handle() && !task_.done();
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle early termination when the coroutine never ran (e.g. JobQueue
|
||||
* is stopping). Decrements nSuspend_ and destroys the coroutine frame
|
||||
* to break the shared_ptr cycle: frame -> lambda -> runner -> frame.
|
||||
*/
|
||||
inline void
|
||||
JobQueue::CoroTaskRunner::expectEarlyExit()
|
||||
{
|
||||
if (!finished_)
|
||||
{
|
||||
std::scoped_lock const lock(jq_.mutex_);
|
||||
--jq_.nSuspend_;
|
||||
finished_ = true;
|
||||
}
|
||||
// Break the shared_ptr cycle: frame -> shared_ptr<runner> -> this.
|
||||
// The coroutine is at initial_suspend and never ran user code, so
|
||||
// destroying it is safe. Use std::move (not task_ = {}) so
|
||||
// task_.handle_ is null before the frame is destroyed.
|
||||
{
|
||||
[[maybe_unused]] auto completed = std::move(task_);
|
||||
}
|
||||
storedFunc_.reset();
|
||||
}
|
||||
|
||||
/**
|
||||
* Block until all pending/active resume operations complete.
|
||||
* Uses cv_ + mutexRun_ to wait until runCount_ reaches 0 or
|
||||
* finished_ becomes true. The finished_ check handles the case
|
||||
* where resume() is called directly (without post()), which
|
||||
* decrements runCount_ below zero. In that scenario runCount_
|
||||
* never returns to 0, but finished_ becoming true guarantees
|
||||
* the coroutine is done and no more resumes will occur.
|
||||
*
|
||||
* Note: when join() returns via the finished_ disjunct, the final
|
||||
* resume() call may still be executing its post-completion
|
||||
* bookkeeping (the --runCount_ / notify after finished_ is set).
|
||||
* That is safe -- the coroutine body has fully completed and the
|
||||
* runner is kept alive by the resume job's shared_ptr -- but
|
||||
* callers must not assume resume() itself has returned.
|
||||
*/
|
||||
inline void
|
||||
JobQueue::CoroTaskRunner::join()
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mutexRun_);
|
||||
cv_.wait(lk, [this]() { return runCount_ == 0 || finished_; });
|
||||
}
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
#include <xrpl/basics/LocalValue.h>
|
||||
#include <xrpl/core/ClosureCounter.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/core/JobTypeData.h>
|
||||
#include <xrpl/core/detail/Workers.h>
|
||||
#include <xrpl/json/json_value.h>
|
||||
@@ -153,420 +152,6 @@ public:
|
||||
join();
|
||||
};
|
||||
|
||||
/**
|
||||
* C++20 coroutine lifecycle manager. Replaces Coro for new code.
|
||||
*
|
||||
* Class / Inheritance / Dependency Diagram
|
||||
* =========================================
|
||||
*
|
||||
* std::enable_shared_from_this<CoroTaskRunner>
|
||||
* ^
|
||||
* | (public inheritance)
|
||||
* |
|
||||
* CoroTaskRunner
|
||||
* +---------------------------------------------------+
|
||||
* | - lvs_ : detail::LocalValues |
|
||||
* | - jq_ : JobQueue& |
|
||||
* | - type_ : JobType |
|
||||
* | - name_ : std::string |
|
||||
* | - runCount_ : int (in-flight resumes) |
|
||||
* | - mutex_ : std::mutex (coroutine guard) |
|
||||
* | - mutexRun_ : std::mutex (join guard) |
|
||||
* | - cv_ : condition_variable |
|
||||
* | - task_ : CoroTask<void> |
|
||||
* | - storedFunc_ : unique_ptr<FuncBase> (type-erased)|
|
||||
* +---------------------------------------------------+
|
||||
* | + init(F&&) : set up coroutine callable |
|
||||
* | + onSuspend() : ++jq_.nSuspend_ |
|
||||
* | + onUndoSuspend() : --jq_.nSuspend_ |
|
||||
* | + suspend() : returns SuspendAwaiter |
|
||||
* | + post() : schedule resume on JobQueue |
|
||||
* | + resume() : resume coroutine on caller |
|
||||
* | + runnable() : !task_.done() |
|
||||
* | + expectEarlyExit() : teardown for failed post |
|
||||
* | + join() : block until not running |
|
||||
* +---------------------------------------------------+
|
||||
* | |
|
||||
* | owns | references
|
||||
* v v
|
||||
* CoroTask<void> JobQueue
|
||||
* (coroutine frame) (thread pool + nSuspend_)
|
||||
*
|
||||
* FuncBase / FuncStore<F> (type-erased heap storage
|
||||
* for the coroutine lambda)
|
||||
*
|
||||
* Coroutine Lifecycle (Control Flow)
|
||||
* ===================================
|
||||
*
|
||||
* Caller thread JobQueue worker thread
|
||||
* ------------- ----------------------
|
||||
* postCoroTask(f)
|
||||
* |
|
||||
* +-- reserve a jobCounter_ slot (reject if JQ shutting down)
|
||||
* +-- ++nSuspend_ (lazy start counts as suspended)
|
||||
* +-- make_shared<CoroTaskRunner>
|
||||
* +-- init(f)
|
||||
* | +-- store lambda on heap (FuncStore)
|
||||
* | +-- task_ = f(shared_from_this())
|
||||
* | [coroutine created, suspended at initial_suspend]
|
||||
* +-- post()
|
||||
* | +-- ++runCount_
|
||||
* | +-- addJob(type_, [resume]{})
|
||||
* | resume()
|
||||
* | |
|
||||
* | +-- --nSuspend_
|
||||
* | +-- swap in LocalValues
|
||||
* | +-- task_.handle().resume()
|
||||
* | | [coroutine body runs]
|
||||
* | | ...
|
||||
* | | co_await suspend()
|
||||
* | | +-- ++nSuspend_
|
||||
* | | [coroutine suspends]
|
||||
* | +-- swap out LocalValues
|
||||
* | +-- --runCount_
|
||||
* | +-- cv_.notify_all()
|
||||
* |
|
||||
* post() <-- called externally or by yieldAndPost()
|
||||
* +-- ++runCount_
|
||||
* +-- addJob(type_, [resume]{})
|
||||
* resume()
|
||||
* |
|
||||
* +-- [coroutine body continues]
|
||||
* +-- co_return
|
||||
* +-- --runCount_
|
||||
* +-- cv_.notify_all()
|
||||
* join()
|
||||
* +-- cv_.wait([]{runCount_ == 0})
|
||||
* +-- [done]
|
||||
*
|
||||
* Usage Examples
|
||||
* ==============
|
||||
*
|
||||
* 1. Fire-and-forget coroutine (most common pattern):
|
||||
*
|
||||
* jq.postCoroTask(JtClient, "MyWork",
|
||||
* [](auto runner) -> CoroTask<void> {
|
||||
* doSomeWork();
|
||||
* co_await runner->suspend(); // yield to other jobs
|
||||
* doMoreWork();
|
||||
* co_return;
|
||||
* });
|
||||
*
|
||||
* 2. Manually controlling suspend / resume (external trigger):
|
||||
*
|
||||
* auto runner = jq.postCoroTask(JtClient, "ExtTrigger",
|
||||
* [&result](auto runner) -> CoroTask<void> {
|
||||
* startAsyncOperation(callback);
|
||||
* co_await runner->suspend();
|
||||
* // callback called runner->post() to get here
|
||||
* result = collectResult();
|
||||
* co_return;
|
||||
* });
|
||||
* // ... later, from the callback:
|
||||
* runner->post(); // reschedule the coroutine on the JobQueue
|
||||
*
|
||||
* 3. Using yieldAndPost() for automatic suspend + repost:
|
||||
*
|
||||
* jq.postCoroTask(JtClient, "AutoRepost",
|
||||
* [](auto runner) -> CoroTask<void> {
|
||||
* step1();
|
||||
* co_await runner->yieldAndPost(); // yield + auto-repost
|
||||
* step2();
|
||||
* co_await runner->yieldAndPost();
|
||||
* step3();
|
||||
* co_return;
|
||||
* });
|
||||
*
|
||||
* 4. Checking shutdown after co_await (cooperative cancellation):
|
||||
*
|
||||
* jq.postCoroTask(JtClient, "Cancellable",
|
||||
* [&jq](auto runner) -> CoroTask<void> {
|
||||
* while (moreWork()) {
|
||||
* co_await runner->yieldAndPost();
|
||||
* if (jq.isStopping())
|
||||
* co_return; // bail out cleanly
|
||||
* processNextItem();
|
||||
* }
|
||||
* co_return;
|
||||
* });
|
||||
*
|
||||
* Caveats / Pitfalls
|
||||
* ==================
|
||||
*
|
||||
* BUG-RISK: Calling suspend() without a matching post()/resume().
|
||||
* After co_await runner->suspend(), the coroutine is parked and
|
||||
* nSuspend_ is incremented. If nothing ever calls post() or
|
||||
* resume(), the coroutine is leaked and JobQueue::stop() will
|
||||
* hang forever waiting for nSuspend_ to reach zero.
|
||||
*
|
||||
* BUG-RISK: Calling post() on an already-running coroutine.
|
||||
* post() schedules a resume() job. If the coroutine has not
|
||||
* actually suspended yet (no co_await executed), the resume job
|
||||
* will try to call handle().resume() while the coroutine is still
|
||||
* running on another thread. This is UB. The mutex_ prevents
|
||||
* data corruption but the logic is wrong — always co_await
|
||||
* suspend() before calling post(). (The test incorrect_order()
|
||||
* shows this works only because mutex_ serializes the calls.)
|
||||
*
|
||||
* BUG-RISK: Dropping the shared_ptr<CoroTaskRunner> before join().
|
||||
* The CoroTaskRunner destructor asserts that finished_ is true
|
||||
* (the coroutine completed). If you let the last shared_ptr die
|
||||
* while the coroutine is still running or suspended, you get an
|
||||
* assertion failure in debug and UB in release. Always call
|
||||
* join() or expectEarlyExit() first.
|
||||
*
|
||||
* BUG-RISK: Lambda captures outliving the coroutine frame.
|
||||
* The lambda passed to postCoroTask is heap-allocated (FuncStore)
|
||||
* to prevent dangling. But objects captured by pointer still need
|
||||
* their own lifetime management. If you capture a raw pointer to
|
||||
* a stack variable, and the stack frame exits before the coroutine
|
||||
* finishes, the pointer dangles. Use shared_ptr or ensure the
|
||||
* pointed-to object outlives the coroutine.
|
||||
*
|
||||
* BUG-RISK: Forgetting co_return in a void coroutine.
|
||||
* If the coroutine body falls off the end without co_return,
|
||||
* the compiler may silently treat it as co_return (per standard),
|
||||
* but some compilers warn. Always write explicit co_return.
|
||||
*
|
||||
* LIMITATION: CoroTaskRunner only supports CoroTask<void>.
|
||||
* The task_ member is CoroTask<void>. To return values from
|
||||
* the top-level coroutine, write through a captured pointer
|
||||
* (as the tests demonstrate), or co_await inner CoroTask<T>
|
||||
* coroutines that return values.
|
||||
*
|
||||
* LIMITATION: One coroutine per CoroTaskRunner.
|
||||
* init() must be called exactly once. You cannot reuse a
|
||||
* CoroTaskRunner to run a second coroutine. Create a new one
|
||||
* via postCoroTask() instead.
|
||||
*
|
||||
* LIMITATION: No timeout on join().
|
||||
* join() blocks indefinitely. If the coroutine is suspended
|
||||
* and never posted, join() will deadlock. Use timed waits
|
||||
* on the gate pattern (condition_variable + wait_for) in tests.
|
||||
*/
|
||||
class CoroTaskRunner : public std::enable_shared_from_this<CoroTaskRunner>
|
||||
{
|
||||
private:
|
||||
// Per-coroutine thread-local storage. Swapped in before resume()
|
||||
// and swapped out after, so each coroutine sees its own LocalValue
|
||||
// state regardless of which worker thread executes it.
|
||||
detail::LocalValues lvs_;
|
||||
|
||||
// Back-reference to the owning JobQueue. Used to post jobs,
|
||||
// increment/decrement nSuspend_, and acquire jq_.mutex_.
|
||||
JobQueue& jq_;
|
||||
|
||||
// Job type passed to addJob() when posting this coroutine.
|
||||
JobType type_;
|
||||
|
||||
// Human-readable name for this coroutine job (for logging).
|
||||
std::string name_;
|
||||
|
||||
// Number of in-flight resume operations (pending + active).
|
||||
// Incremented by post(), decremented when resume() finishes.
|
||||
// Guarded by mutexRun_. join() blocks until this reaches 0.
|
||||
//
|
||||
// A counter (not a bool) is needed because post() can be called
|
||||
// from within the coroutine body (e.g. via yieldAndPost()),
|
||||
// enqueuing a second resume while the first is still running.
|
||||
// A bool would be clobbered: R2.post() sets true, then R1's
|
||||
// cleanup sets false — losing the fact that R2 is still pending.
|
||||
int runCount_ = 0;
|
||||
|
||||
// Serializes all coroutine resume() calls, preventing concurrent
|
||||
// execution of the coroutine body on multiple threads. Handles the
|
||||
// race where post() enqueues a resume before the coroutine has
|
||||
// actually suspended (post-before-suspend pattern).
|
||||
std::mutex mutex_;
|
||||
|
||||
// Guards runCount_. Used with cv_ for join() to wait
|
||||
// until all pending/active resume operations complete.
|
||||
std::mutex mutexRun_;
|
||||
|
||||
// Notified when runCount_ reaches zero, allowing
|
||||
// join() waiters to wake up.
|
||||
std::condition_variable cv_;
|
||||
|
||||
// The coroutine handle wrapper. Owns the coroutine frame.
|
||||
// Set by init(). Reset to empty in resume() upon coroutine
|
||||
// completion (to break the shared_ptr cycle) or in
|
||||
// expectEarlyExit() on early termination.
|
||||
CoroTask<void> task_;
|
||||
|
||||
/**
|
||||
* Type-erased base for heap-stored callables.
|
||||
* Prevents the coroutine lambda from being destroyed before
|
||||
* the coroutine frame is done with it.
|
||||
*
|
||||
* @see FuncStore
|
||||
*/
|
||||
struct FuncBase
|
||||
{
|
||||
virtual ~FuncBase() = default;
|
||||
};
|
||||
|
||||
/**
|
||||
* Concrete type-erased storage for a callable of type F.
|
||||
* The coroutine frame stores a reference to the lambda's implicit
|
||||
* object parameter. If the lambda is a temporary, that reference
|
||||
* dangles after the call returns. FuncStore keeps it alive on
|
||||
* the heap for the lifetime of the CoroTaskRunner.
|
||||
*/
|
||||
template <class F>
|
||||
struct FuncStore : FuncBase
|
||||
{
|
||||
F func; // The stored callable (coroutine lambda).
|
||||
explicit FuncStore(F&& f) : func(std::move(f))
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
// Heap-allocated callable storage. Set by init(), ensures the
|
||||
// lambda outlives the coroutine frame that references it.
|
||||
std::unique_ptr<FuncBase> storedFunc_;
|
||||
|
||||
// True once the coroutine has completed or expectEarlyExit() was
|
||||
// called. Asserted in the destructor (debug) to catch leaked
|
||||
// runners. Available in all builds to guard expectEarlyExit()
|
||||
// against double-decrementing nSuspend_.
|
||||
bool finished_ = false;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Tag type for private construction. Prevents external code
|
||||
* from constructing CoroTaskRunner directly. Use postCoroTask().
|
||||
*/
|
||||
struct CreateT
|
||||
{
|
||||
explicit CreateT() = default;
|
||||
};
|
||||
|
||||
/**
|
||||
* Construct a CoroTaskRunner. Private by convention (CreateT tag).
|
||||
*
|
||||
* @param jq The JobQueue this coroutine will run on
|
||||
* @param type Job type for scheduling priority
|
||||
* @param name Human-readable name for logging
|
||||
*/
|
||||
CoroTaskRunner(CreateT, JobQueue&, JobType, std::string);
|
||||
|
||||
CoroTaskRunner(CoroTaskRunner const&) = delete;
|
||||
CoroTaskRunner&
|
||||
operator=(CoroTaskRunner const&) = delete;
|
||||
|
||||
/**
|
||||
* Destructor. Asserts (debug) that the coroutine has finished
|
||||
* or expectEarlyExit() was called.
|
||||
*/
|
||||
~CoroTaskRunner();
|
||||
|
||||
/**
|
||||
* Initialize with a coroutine-returning callable.
|
||||
* Must be called exactly once, after the object is managed by
|
||||
* shared_ptr (because init uses shared_from_this internally).
|
||||
* This is handled automatically by postCoroTask().
|
||||
*
|
||||
* @param f Callable: CoroTask<void>(shared_ptr<CoroTaskRunner>)
|
||||
*/
|
||||
template <class F>
|
||||
void
|
||||
init(F&& f);
|
||||
|
||||
/**
|
||||
* Increment the JobQueue's suspended-coroutine count (nSuspend_).
|
||||
* Called when the coroutine is about to suspend. Every call
|
||||
* must be balanced by a corresponding decrement (via resume()
|
||||
* or onUndoSuspend()), or JobQueue::stop() will hang.
|
||||
*/
|
||||
void
|
||||
onSuspend();
|
||||
|
||||
/**
|
||||
* Decrement nSuspend_ without resuming.
|
||||
* Used to undo onSuspend() when a scheduled post() fails
|
||||
* (e.g. JobQueue is stopping).
|
||||
*/
|
||||
void
|
||||
onUndoSuspend();
|
||||
|
||||
/**
|
||||
* Suspend the coroutine.
|
||||
* The awaiter's await_suspend() increments nSuspend_ before the
|
||||
* coroutine actually suspends. The caller must later call post()
|
||||
* or resume() to continue execution.
|
||||
*
|
||||
* @return An awaiter for use with `co_await runner->suspend()`
|
||||
*/
|
||||
auto
|
||||
suspend();
|
||||
|
||||
/**
|
||||
* Suspend the coroutine and immediately repost it on the
|
||||
* JobQueue. Combines suspend() + post() atomically inside
|
||||
* await_suspend, so there is no window where an external
|
||||
* event could race between the two.
|
||||
*
|
||||
* Equivalent to JobQueueAwaiter but defined as an inline
|
||||
* awaiter returned from a member function. This avoids a
|
||||
* GCC-12 coroutine codegen bug where an external awaiter
|
||||
* struct (JobQueueAwaiter) used at multiple co_await points
|
||||
* corrupts the coroutine state machine's resume index,
|
||||
* causing the coroutine to hang on the third resumption.
|
||||
*
|
||||
* @return An awaiter for use with `co_await runner->yieldAndPost()`
|
||||
*/
|
||||
auto
|
||||
yieldAndPost();
|
||||
|
||||
/**
|
||||
* Schedule coroutine resumption as a job on the JobQueue.
|
||||
* Captures shared_from_this() to prevent this runner from being
|
||||
* destroyed while the job is queued.
|
||||
*
|
||||
* @return true if the job was accepted; false if the JobQueue
|
||||
* is stopping (caller must handle cleanup)
|
||||
*/
|
||||
bool
|
||||
post();
|
||||
|
||||
/**
|
||||
* Resume the coroutine on the current thread.
|
||||
* Decrements nSuspend_, swaps in LocalValues, resumes the
|
||||
* coroutine handle, swaps out LocalValues, and notifies join()
|
||||
* waiters. Lock ordering (sequential, non-overlapping):
|
||||
* jq_.mutex_ -> mutex_ -> mutexRun_.
|
||||
*
|
||||
* @pre post() must have been called before resume(). Direct
|
||||
* calls without a prior post() will corrupt runCount_
|
||||
* and break join().
|
||||
*/
|
||||
void
|
||||
resume();
|
||||
|
||||
/**
|
||||
* @return true if the coroutine has not yet run to completion
|
||||
*/
|
||||
bool
|
||||
runnable() const;
|
||||
|
||||
/**
|
||||
* Handle early termination when the coroutine never ran.
|
||||
* Decrements nSuspend_ and destroys the coroutine frame to
|
||||
* break the shared_ptr cycle (frame -> lambda -> runner -> frame).
|
||||
* Called by postCoroTask() when post() fails.
|
||||
*/
|
||||
void
|
||||
expectEarlyExit();
|
||||
|
||||
/**
|
||||
* Block until all pending/active resume operations complete.
|
||||
* Uses cv_ + mutexRun_ to wait until runCount_ reaches 0.
|
||||
* Warning: deadlocks if the coroutine is suspended and never posted.
|
||||
*/
|
||||
void
|
||||
join();
|
||||
};
|
||||
|
||||
using JobFunction = std::function<void()>;
|
||||
|
||||
JobQueue(
|
||||
@@ -612,20 +197,6 @@ public:
|
||||
std::shared_ptr<Coro>
|
||||
postCoro(JobType t, std::string const& name, F&& f);
|
||||
|
||||
/**
|
||||
* Creates a C++20 coroutine and adds a job to the queue to run it.
|
||||
*
|
||||
* @param t The type of job.
|
||||
* @param name Name of the job.
|
||||
* @param f Callable with signature
|
||||
* CoroTask<void>(std::shared_ptr<CoroTaskRunner>).
|
||||
*
|
||||
* @return shared_ptr to posted CoroTaskRunner. nullptr if not successful.
|
||||
*/
|
||||
template <class F>
|
||||
std::shared_ptr<CoroTaskRunner>
|
||||
postCoroTask(JobType t, std::string const& name, F&& f);
|
||||
|
||||
/**
|
||||
* Jobs waiting at this priority.
|
||||
*/
|
||||
@@ -848,8 +419,7 @@ private:
|
||||
|
||||
} // namespace xrpl
|
||||
|
||||
#include <xrpl/core/Coro.ipp> // IWYU pragma: keep
|
||||
#include <xrpl/core/CoroTaskRunner.ipp> // IWYU pragma: keep
|
||||
#include <xrpl/core/Coro.ipp> // IWYU pragma: keep
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
@@ -872,82 +442,4 @@ JobQueue::postCoro(JobType t, std::string const& name, F&& f)
|
||||
return coro;
|
||||
}
|
||||
|
||||
// postCoroTask — entry point for launching a C++20 coroutine on the JobQueue.
|
||||
//
|
||||
// Control Flow
|
||||
// ============
|
||||
//
|
||||
// postCoroTask(t, name, f)
|
||||
// |
|
||||
// +-- 1. Reserve a jobCounter_ slot — reject if JQ shutting down
|
||||
// |
|
||||
// +-- 2. ++nSuspend_ (mirrors Boost Coro ctor's implicit yield)
|
||||
// | The coroutine is "suspended" from the JobQueue's perspective
|
||||
// | even though it hasn't run yet — this keeps the JQ shutdown
|
||||
// | logic correct (it waits for nSuspend_ to reach 0).
|
||||
// |
|
||||
// +-- 3. Create CoroTaskRunner (shared_ptr, ref-counted)
|
||||
// |
|
||||
// +-- 4. runner->init(f)
|
||||
// | +-- Heap-allocate the lambda (FuncStore) to prevent
|
||||
// | | dangling captures in the coroutine frame
|
||||
// | +-- task_ = f(shared_from_this())
|
||||
// | [coroutine created but NOT started — lazy initial_suspend]
|
||||
// |
|
||||
// +-- 5. runner->post()
|
||||
// | +-- addJob(type_, [resume]{}) → resume on worker thread
|
||||
// | +-- failure (JQ stopping):
|
||||
// | +-- runner->expectEarlyExit()
|
||||
// | | --nSuspend_, destroy coroutine frame
|
||||
// | +-- return nullptr
|
||||
// |
|
||||
// +-- 6. Release the jobCounter_ slot (on return)
|
||||
//
|
||||
// Why reserve a jobCounter_ slot?
|
||||
// ===============================
|
||||
// JobQueue::stop() joins jobCounter_ before it acquires mutex_ and asserts
|
||||
// nSuspend_ == 0. Without a reservation, stop() could run to completion in
|
||||
// the window between the ++nSuspend_ in step 2 and the balancing post() or
|
||||
// expectEarlyExit() in step 5, tripping that assert. Holding a slot blocks
|
||||
// stop()'s join() for the whole function, closing the window. wrap() also
|
||||
// returns nullopt once the counter is joined, so it doubles as the shutdown
|
||||
// check — a plain stopping_ read cannot do this, because the read and the
|
||||
// ++nSuspend_ are not a single atomic step.
|
||||
//
|
||||
// Why async post() instead of synchronous resume()?
|
||||
// ==================================================
|
||||
// The initial dispatch MUST use async post() so the coroutine body runs on
|
||||
// a JobQueue worker thread, not the caller's thread. resume() swaps the
|
||||
// caller's thread-local LocalValues with the coroutine's private copy.
|
||||
// If the coroutine mutates LocalValues (e.g. thread_specific_storage test),
|
||||
// those mutations bleed back into the caller's thread-local state after the
|
||||
// swap-out, corrupting subsequent tests that share the same thread pool.
|
||||
// Async post() avoids this by running the coroutine on a worker thread whose
|
||||
// LocalValues are managed by the thread pool, not by the caller.
|
||||
//
|
||||
template <class F>
|
||||
std::shared_ptr<JobQueue::CoroTaskRunner>
|
||||
JobQueue::postCoroTask(JobType t, std::string const& name, F&& f)
|
||||
{
|
||||
// Held until this function returns. Null once jobCounter_ is joined,
|
||||
// which is how a shutting-down JobQueue rejects new coroutines.
|
||||
auto const shutdownGuard = jobCounter_.wrap([]() {});
|
||||
if (!shutdownGuard)
|
||||
return nullptr;
|
||||
|
||||
{
|
||||
std::scoped_lock const lock(mutex_);
|
||||
++nSuspend_;
|
||||
}
|
||||
|
||||
auto runner = std::make_shared<CoroTaskRunner>(CoroTaskRunner::CreateT{}, *this, t, name);
|
||||
runner->init(std::forward<F>(f));
|
||||
if (!runner->post())
|
||||
{
|
||||
runner->expectEarlyExit();
|
||||
runner.reset();
|
||||
}
|
||||
return runner;
|
||||
}
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -1,212 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <xrpl/beast/utility/instrumentation.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
|
||||
#include <coroutine>
|
||||
#include <memory>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
/**
|
||||
* Awaiter that suspends and immediately reschedules on the JobQueue.
|
||||
* Equivalent to calling yield() followed by post() in the old Coro API.
|
||||
*
|
||||
* Usage:
|
||||
* co_await JobQueueAwaiter{runner};
|
||||
*
|
||||
* What it waits for: The coroutine is re-queued as a job and resumes
|
||||
* when a worker thread picks it up.
|
||||
*
|
||||
* Which thread resumes: A JobQueue worker thread.
|
||||
*
|
||||
* What await_resume() returns: void.
|
||||
*
|
||||
* Dependency Diagram
|
||||
* ==================
|
||||
*
|
||||
* JobQueueAwaiter
|
||||
* +----------------------------------------------+
|
||||
* | + runner : shared_ptr<CoroTaskRunner> |
|
||||
* +----------------------------------------------+
|
||||
* | + await_ready() -> false (always suspend) |
|
||||
* | + await_suspend() -> bool (suspend or cancel) |
|
||||
* | + await_resume() -> void |
|
||||
* +----------------------------------------------+
|
||||
* | |
|
||||
* | uses | uses
|
||||
* v v
|
||||
* CoroTaskRunner JobQueue
|
||||
* .onSuspend() (via runner->post() -> addJob)
|
||||
* .onUndoSuspend()
|
||||
* .post()
|
||||
*
|
||||
* Control Flow (await_suspend)
|
||||
* ============================
|
||||
*
|
||||
* co_await JobQueueAwaiter{runner}
|
||||
* |
|
||||
* +-- await_ready() -> false
|
||||
* +-- await_suspend(handle)
|
||||
* |
|
||||
* +-- runner->onSuspend() // ++nSuspend_
|
||||
* +-- runner->post() // addJob to JobQueue
|
||||
* | |
|
||||
* | +-- success? return noop_coroutine()
|
||||
* | | // coroutine stays suspended;
|
||||
* | | // worker thread will call resume()
|
||||
* | +-- failure? (JQ stopping)
|
||||
* | +-- runner->onUndoSuspend() // --nSuspend_
|
||||
* | +-- return handle // symmetric transfer back
|
||||
* | // coroutine continues immediately
|
||||
* | // so it can clean up and co_return
|
||||
*
|
||||
* DEPRECATED — prefer `co_await runner->yieldAndPost()`
|
||||
* =====================================================
|
||||
*
|
||||
* GCC-12 has a coroutine codegen bug where using this external awaiter
|
||||
* struct at multiple co_await points in the same coroutine corrupts the
|
||||
* state machine's resume index. After the second co_await, the third
|
||||
* resumption enters handle().resume() but never reaches await_resume()
|
||||
* or any subsequent user code — the coroutine hangs indefinitely.
|
||||
*
|
||||
* The fix is `co_await runner->yieldAndPost()`, which defines the
|
||||
* awaiter as an inline struct inside a CoroTaskRunner member function.
|
||||
* GCC-12 handles inline awaiters correctly at multiple co_await points.
|
||||
*
|
||||
* This struct is retained for single-use scenarios and documentation
|
||||
* purposes. For any code that may use co_await in a loop or at
|
||||
* multiple points, always use `runner->yieldAndPost()`.
|
||||
*
|
||||
* Usage Examples
|
||||
* ==============
|
||||
*
|
||||
* 1. Yield and auto-repost (preferred — works on all compilers):
|
||||
*
|
||||
* CoroTask<void> handler(auto runner) {
|
||||
* doPartA();
|
||||
* co_await runner->yieldAndPost(); // yield + repost
|
||||
* doPartB(); // runs on a worker thread
|
||||
* co_return;
|
||||
* }
|
||||
*
|
||||
* 2. Multiple yield points in a loop:
|
||||
*
|
||||
* CoroTask<void> batchProcessor(auto runner) {
|
||||
* for (auto& item : items) {
|
||||
* process(item);
|
||||
* co_await runner->yieldAndPost(); // let other jobs run
|
||||
* }
|
||||
* co_return;
|
||||
* }
|
||||
*
|
||||
* 3. Graceful shutdown — checking after resume:
|
||||
*
|
||||
* CoroTask<void> longTask(auto runner, JobQueue& jq) {
|
||||
* while (hasWork()) {
|
||||
* co_await runner->yieldAndPost();
|
||||
* // If JQ is stopping, await_suspend resumes the coroutine
|
||||
* // immediately without re-queuing. Always check
|
||||
* // isStopping() to decide whether to proceed:
|
||||
* if (jq.isStopping())
|
||||
* co_return;
|
||||
* doNextChunk();
|
||||
* }
|
||||
* co_return;
|
||||
* }
|
||||
*
|
||||
* Caveats / Pitfalls
|
||||
* ==================
|
||||
*
|
||||
* BUG-RISK: Using a stale or null runner.
|
||||
* The runner shared_ptr must be valid and point to the CoroTaskRunner
|
||||
* that owns the coroutine currently executing. Passing a runner from
|
||||
* a different coroutine, or a default-constructed shared_ptr, is UB.
|
||||
*
|
||||
* BUG-RISK: Assuming resume happens on the same thread.
|
||||
* After co_await, the coroutine resumes on whatever worker thread
|
||||
* picks up the job. Do not rely on thread-local state unless it is
|
||||
* managed through LocalValue (which CoroTaskRunner automatically
|
||||
* swaps in/out).
|
||||
*
|
||||
* BUG-RISK: Ignoring the shutdown path.
|
||||
* When the JobQueue is stopping, post() fails and await_suspend()
|
||||
* resumes the coroutine immediately (symmetric transfer back to h).
|
||||
* The coroutine body continues on the same thread. If your code
|
||||
* after co_await assumes it was re-queued and is running on a worker
|
||||
* thread, that assumption breaks during shutdown. Always handle the
|
||||
* "JQ is stopping" case, either by checking jq.isStopping() or by
|
||||
* letting the coroutine fall through to co_return naturally.
|
||||
*
|
||||
* DIFFERENCE from runner->suspend() + runner->post():
|
||||
* Both JobQueueAwaiter and yieldAndPost() combine suspend + post
|
||||
* in one atomic operation. With the manual suspend()/post() pattern,
|
||||
* there is a window between the two calls where an external event
|
||||
* could race. The atomic awaiters remove that window — onSuspend()
|
||||
* and post() happen within the same await_suspend() call while the
|
||||
* coroutine is guaranteed to be suspended. Use yieldAndPost() unless
|
||||
* you need an external party to decide *when* to call post().
|
||||
*/
|
||||
struct JobQueueAwaiter
|
||||
{
|
||||
// The CoroTaskRunner that owns the currently executing coroutine.
|
||||
std::shared_ptr<JobQueue::CoroTaskRunner> runner;
|
||||
|
||||
/**
|
||||
* Always returns false so the coroutine suspends.
|
||||
*/
|
||||
// The C++ coroutine protocol mandates these awaiter names and
|
||||
// instance-callable methods, which conflict with the project
|
||||
// naming/static conventions.
|
||||
// NOLINTBEGIN(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
[[nodiscard]] bool
|
||||
await_ready() const noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment nSuspend (equivalent to yield()) and schedule resume
|
||||
* on the JobQueue (equivalent to post()). If the JobQueue is
|
||||
* stopping, undoes the suspend count and transfers back to the
|
||||
* coroutine so it can clean up and co_return.
|
||||
*
|
||||
* Returns a coroutine_handle<> (symmetric transfer) instead of
|
||||
* bool to work around a GCC-12 codegen bug where bool-returning
|
||||
* await_suspend leaves the coroutine in an invalid state —
|
||||
* neither properly suspended nor resumed — causing a hang.
|
||||
*
|
||||
* WARNING: GCC-12 has an additional codegen bug where using this
|
||||
* external awaiter struct at multiple co_await points in the same
|
||||
* coroutine corrupts the state machine's resume index, causing the
|
||||
* coroutine to hang on the third resumption. Prefer
|
||||
* `co_await runner->yieldAndPost()` which uses an inline awaiter
|
||||
* that GCC-12 handles correctly.
|
||||
*
|
||||
* @return noop_coroutine() to stay suspended (job posted);
|
||||
* the caller's handle to resume immediately (JQ stopping)
|
||||
*/
|
||||
std::coroutine_handle<>
|
||||
await_suspend(std::coroutine_handle<> h)
|
||||
{
|
||||
XRPL_ASSERT(runner, "xrpl::JobQueueAwaiter::await_suspend : runner is valid");
|
||||
runner->onSuspend();
|
||||
if (!runner->post())
|
||||
{
|
||||
// JobQueue is stopping. Undo the suspend count and
|
||||
// transfer back to the coroutine so it can clean up
|
||||
// and co_return.
|
||||
runner->onUndoSuspend();
|
||||
return h;
|
||||
}
|
||||
return std::noop_coroutine();
|
||||
}
|
||||
|
||||
void
|
||||
await_resume() const noexcept
|
||||
{
|
||||
}
|
||||
// NOLINTEND(readability-identifier-naming, readability-convert-member-functions-to-static)
|
||||
};
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -396,6 +396,7 @@ JSS(mean); // out: get_aggregate_price
|
||||
JSS(median); // out: get_aggregate_price
|
||||
JSS(median_fee); // out: TxQ
|
||||
JSS(median_level); // out: TxQ
|
||||
JSS(memory_limit); // out: server_info
|
||||
JSS(message); // error.
|
||||
JSS(meta); // out: NetworkOPs, AccountTx*, Tx
|
||||
JSS(meta_blob); // out: NetworkOPs, AccountTx*, Tx
|
||||
@@ -436,7 +437,6 @@ JSS(node_read_retries); // out: GetCounts
|
||||
JSS(node_reads_hit); // out: GetCounts
|
||||
JSS(node_reads_total); // out: GetCounts
|
||||
JSS(node_reads_duration_us); // out: GetCounts
|
||||
JSS(node_size); // out: server_info
|
||||
JSS(nodes); // out: VaultInfo
|
||||
JSS(nodestore); // out: GetCounts
|
||||
JSS(node_writes); // out: GetCounts
|
||||
|
||||
+49
-66
@@ -10,12 +10,11 @@ a build configured with `-Dvalidator_keys=ON`.
|
||||
package/
|
||||
build_pkg.py Staging and build script (called by the CMake `package` target and CI)
|
||||
sign_rpm.py Signs the built RPMs (called by CI when publishing)
|
||||
images/
|
||||
packaging/
|
||||
Dockerfile Packaging image, built by `build-packaging-images.yml`; installs its tooling with `bin/install-packaging-tools.sh`
|
||||
publish_pkg.py Uploads built packages to the XRPLF Nexus repositories (called by CI, and shipped in that image)
|
||||
xrpld/
|
||||
Dockerfile The xrpld Docker images, installing the built DEB on Ubuntu (see "Docker images")
|
||||
docker/
|
||||
Dockerfile Packaging image, built by `build-packaging-images.yml`; installs its tooling with `bin/install-packaging-tools.sh`
|
||||
publish_pkg.py Uploads built packages to the XRPLF Nexus repositories (called by CI, and shipped in that image)
|
||||
image/
|
||||
Dockerfile The xrpld Docker image, installing the built DEB on Ubuntu (see "Docker image")
|
||||
rpm/
|
||||
xrpld.spec RPM spec
|
||||
debian/ Debian control files (control.in, lintian-overrides.in, rules, copyright, docs, links, source/format).
|
||||
@@ -120,8 +119,8 @@ Caller workflows (`on-pr.yml`, `on-tag.yml`, `on-trigger.yml`) call
|
||||
in the container of every distro that format targets and running the binaries
|
||||
there, so one that cannot be installed never reaches Nexus.
|
||||
3. `publish` uploads both artifacts, or lists what it would upload.
|
||||
4. `docker` builds the [Docker images](#docker-images) from the tested DEB, and
|
||||
pushes them when publishing.
|
||||
4. `docker` builds the [Docker image](#docker-image) from the tested DEB, and
|
||||
pushes it when publishing.
|
||||
|
||||
The packaging script derives the package version from the downloaded binary's
|
||||
`xrpld --version` output; no CMake configure or build step is needed inside the
|
||||
@@ -210,36 +209,32 @@ rejects request bodies over 100 MB, so uploads use the DNS-only host instead.
|
||||
The `release-info` action decides the channel from the event, and
|
||||
`publish_pkg.py` maps that channel to its repositories:
|
||||
|
||||
| Event | Version | Channel | DEB repository | RPM upload repository |
|
||||
| ------------------------ | ------------------ | --------- | -------------- | --------------------- |
|
||||
| tag | `X.Y.Z` | `stable` | `deb-stable` | `rpm-stable-hosted` |
|
||||
| tag | `X.Y.Z-rcN` | `rc` | `deb-rc` | `rpm-rc-hosted` |
|
||||
| tag | `X.Y.Z-bN` | `beta` | `deb-beta` | `rpm-beta-hosted` |
|
||||
| tag, any other | `xrpld --version` | `custom` | `deb-custom` | `rpm-custom-hosted` |
|
||||
| push to `develop` | `0.0.0-dev+<hash>` | `develop` | `deb-develop` | `rpm-develop-hosted` |
|
||||
| tag, non-public codebase | _any_ | `private` | `deb-private` | `rpm-private-hosted` |
|
||||
| Event | Version | Channel | DEB repository | RPM upload repository |
|
||||
| ------------------------ | ----------------- | --------- | -------------- | --------------------- |
|
||||
| tag | `X.Y.Z` | `stable` | `deb-stable` | `rpm-stable-hosted` |
|
||||
| tag | `X.Y.Z-rcN` | `rc` | `deb-rc` | `rpm-rc-hosted` |
|
||||
| tag | `X.Y.Z-bN` | `beta` | `deb-beta` | `rpm-beta-hosted` |
|
||||
| tag, any other | `xrpld --version` | `custom` | `deb-custom` | `rpm-custom-hosted` |
|
||||
| push to `develop` | `xrpld --version` | `develop` | `deb-develop` | `rpm-develop-hosted` |
|
||||
| tag, non-public codebase | _any_ | `private` | `deb-private` | `rpm-private-hosted` |
|
||||
|
||||
A variant is published to the same channel under its own name, so
|
||||
`xrpld-assert` never overwrites `xrpld`.
|
||||
|
||||
Only a tag picks a release channel. Versions sort in row order, so moving to a
|
||||
more mature channel never downgrades. A tag matching none of the release
|
||||
patterns, such as `X.Y.Z-hotfix1`, publishes to `custom`, which sits outside
|
||||
that order.
|
||||
Only a tag names a channel — do not extend that to `develop`, where
|
||||
`BuildInfo.cpp`'s `versionString` moves through `-bN`, `-rcN` and even the final
|
||||
version during a release cycle, which would send develop builds into `stable`.
|
||||
Versions sort in row order, so moving to a more mature channel never downgrades.
|
||||
A tag matching none of the release patterns, such as `X.Y.Z-hotfix1`, publishes
|
||||
to `custom`, which sits outside that order.
|
||||
|
||||
Every untagged build reports `0.0.0-dev+<hash>`, which the `develop` channel
|
||||
packages as `0.0.0~dev`, below every release.
|
||||
|
||||
The action also picks the package release number:
|
||||
|
||||
- A tag: `1`, since a tag's version is never reused.
|
||||
- Anything else: `<run number>.<commit date>git<commit hash>`, e.g.
|
||||
`857.20260826gitb6a8995`. Develop packages all share `0.0.0~dev`, so the run
|
||||
number orders them, and the date and hash name the commit a package on
|
||||
`packages.xrplf.org` came from.
|
||||
|
||||
Both reach the packaging scripts as arguments, so neither script derives
|
||||
anything itself.
|
||||
The action decides the package release number on the same split: a tag's version
|
||||
is unique, so its packages are release 1, while develop repeats the same version
|
||||
and takes `<run number>.<commit date>git<commit hash>`, e.g.
|
||||
`857.20260826gitb6a8995` — the leading run number keeps each push superseding
|
||||
the last, and the date and hash say which commit a package on
|
||||
`packages.xrplf.org` came from. Both reach the packaging scripts as arguments,
|
||||
so neither script derives anything itself.
|
||||
|
||||
Publishing is its own job, gated behind the install tests, uploading from the same
|
||||
image that built the packages with the `publish_pkg.py` shipped in it — the
|
||||
@@ -276,22 +271,14 @@ Nexus owns the repository metadata; nothing here indexes anything. Worth knowing
|
||||
installs it at `/usr/local/bin/publish_pkg.py` for other XRPLF repositories that
|
||||
build their packages elsewhere.
|
||||
|
||||
## Docker images
|
||||
## Docker image
|
||||
|
||||
The `docker` job installs the tested `xrpld` DEB on `ubuntu:26.04` using
|
||||
[`images/xrpld/Dockerfile`](images/xrpld/Dockerfile), once per target, and
|
||||
checks that the server starts in each image. A tag's images are tagged with the
|
||||
tag name, a develop image as `develop`. With `publish: true`:
|
||||
|
||||
- `xrpld` is pushed to `xrplf/xrpld` on Docker Hub using the `DOCKERHUB_TOKEN`
|
||||
secret, an organization access token for `xrplf`. Private builds are never
|
||||
pushed there.
|
||||
- `voidstar` replaces `/usr/bin/xrpld` with the binary of the `voidstar` build
|
||||
config, adds `libvoidstar.so` and links the binary into `/symbols`, as
|
||||
Antithesis expects. It is pushed as `xrpld-voidstar` to the Antithesis
|
||||
registry, `${ANTITHESIS_DOCKER_HOST}/${ANTITHESIS_DOCKER_PATH}`, logging in
|
||||
with the `ANTITHESIS_DOCKER_CREDENTIALS` service account key. The registry is
|
||||
private, so private builds are pushed too.
|
||||
[`image/Dockerfile`](image/Dockerfile), checks that the server starts, and,
|
||||
with `publish: true`, pushes it to `xrplf/xrpld` on Docker Hub using the
|
||||
`DOCKERHUB_TOKEN` secret, an organization access token for `xrplf`. A tag's
|
||||
image is tagged with the tag name, `xrplf/xrpld:<version>`, and a develop image
|
||||
as `xrplf/xrpld:develop`. Private builds are never pushed.
|
||||
|
||||
## How `build_pkg.py` works
|
||||
|
||||
@@ -320,30 +307,26 @@ values; DEB writes them as `${pkg_version}-${PKG_RELEASE}` in
|
||||
|
||||
With `PKG_RELEASE=1`, the package metadata becomes:
|
||||
|
||||
| Input version | RPM version/release | Debian version |
|
||||
| ------------------- | -------------------------- | ------------------ |
|
||||
| `3.2.0` | `3.2.0-1%{?dist}` | `3.2.0-1` |
|
||||
| `3.2.0-b1` | `3.2.0~b1-1%{?dist}` | `3.2.0~b1-1` |
|
||||
| `3.2.0-rc1` | `3.2.0~rc1-1%{?dist}` | `3.2.0~rc1-1` |
|
||||
| `3.2.0-custom-1` | `3.2.0~custom.1-1%{?dist}` | `3.2.0~custom.1-1` |
|
||||
| `0.0.0-dev+abc1234` | `0.0.0~dev-1%{?dist}` | `0.0.0~dev-1` |
|
||||
| Input version | RPM version/release | Debian version |
|
||||
| ------------------ | ---------------------------- | -------------------- |
|
||||
| `3.2.0` | `3.2.0-1%{?dist}` | `3.2.0-1` |
|
||||
| `3.2.0-b0+abc1234` | `3.2.0~b0+abc1234-1%{?dist}` | `3.2.0~b0+abc1234-1` |
|
||||
| `3.2.0-b1` | `3.2.0~b1-1%{?dist}` | `3.2.0~b1-1` |
|
||||
| `3.2.0-rc1` | `3.2.0~rc1-1%{?dist}` | `3.2.0~rc1-1` |
|
||||
| `3.2.0-custom-1` | `3.2.0~custom.1-1%{?dist}` | `3.2.0~custom.1-1` |
|
||||
|
||||
`build_pkg.py` defines `dist` as `.el9` rather than letting rpmbuild take it
|
||||
from the build host, so the RHEL image can track a newer release without
|
||||
changing what the packages claim to target.
|
||||
|
||||
The Debian changelog entry carries the channel passed as `--channel`: one of
|
||||
the channels in the table above, or `UNRELEASED`, the Debian convention for a
|
||||
build that targets no channel, which local and CMake builds pass. Each channel
|
||||
accepts:
|
||||
|
||||
- `stable`, `rc`, `beta`: `X.Y.Z`, or a `bN`/`rcN` pre-release, the only kind
|
||||
that may carry build metadata. `3.2.0-b1` packages as `3.2.0~b1`.
|
||||
- `custom`, `private`: any version, with each `-` inside the pre-release or
|
||||
build metadata switched to `.`. `3.4.0-custom-1` packages as `3.4.0~custom.1`.
|
||||
- `develop`: only `0.0.0-dev`, without its build metadata. `0.0.0-dev+abc1234`
|
||||
packages as `0.0.0~dev`.
|
||||
- `UNRELEASED`: `0.0.0-dev` as `develop` does, anything else as `stable` does.
|
||||
The Debian changelog entry carries the channel passed as `--channel`, which
|
||||
only accepts the channels in the table above plus `UNRELEASED`, the Debian
|
||||
convention for a build that targets no channel at all — what local and CMake
|
||||
builds pass, since nothing publishes them. An unsupported pre-release, and
|
||||
build metadata on a final release such as `3.2.0+abc123`, are both rejected,
|
||||
except in the `custom` and `private` channels, which accept any version and
|
||||
switch each `-` inside the pre-release or build metadata to `.`, so
|
||||
`3.4.0-custom-1` packages as `3.4.0~custom.1`.
|
||||
|
||||
The RPM path intentionally uses `~` in `Version`, matching the Debian
|
||||
pre-release ordering convention, so RPM filenames/NVRs begin with forms like
|
||||
|
||||
+4
-20
@@ -24,12 +24,6 @@ PRE_RELEASE = re.compile(r"^(b|rc)(0|[1-9][0-9]*)$")
|
||||
# Channels a tag of any version is published to, rather than only bN/rcN.
|
||||
ANY_VERSION_CHANNELS = ("custom", "private")
|
||||
|
||||
# The version of any untagged build, which is all the develop channel publishes.
|
||||
DEV_VERSION = "0.0.0-dev"
|
||||
|
||||
# Channels an untagged build is packaged for: develop, or none for a local build.
|
||||
DEV_VERSION_CHANNELS = ("develop", "UNRELEASED")
|
||||
|
||||
# The package name a variant suffixes, and the name every variant keeps for its
|
||||
# on-disk paths (/usr/bin/xrpld, /etc/xrpld, xrpld.service).
|
||||
BASE_NAME = "xrpld"
|
||||
@@ -80,23 +74,13 @@ def capture(*command: object) -> str:
|
||||
def package_version(reported: str, channel: str) -> str:
|
||||
"""Normalise a reported version into one the package formats accept.
|
||||
|
||||
- Release: unchanged (3.2.0 -> 3.2.0).
|
||||
- bN/rcN pre-release: '-' becomes '~', to sort before the release
|
||||
(3.2.0-b1 -> 3.2.0~b1).
|
||||
- custom, private: any version, '-' inside the pre-release and build
|
||||
metadata becomes '.' (3.4.0-custom-1 -> 3.4.0~custom.1).
|
||||
- develop, UNRELEASED: 0.0.0-dev without build metadata, so builds sort by
|
||||
package release, not commit hash (0.0.0-dev+abc1234 -> 0.0.0~dev).
|
||||
develop accepts nothing else.
|
||||
A pre-release switches to '~' (3.2.0-b1 -> 3.2.0~b1), which also sorts before
|
||||
the final 3.2.0; a no-op for a final release. The custom and private
|
||||
channels accept any pre-release and build metadata, with any '-' inside
|
||||
either switched to '.' (3.4.0-custom-1 -> 3.4.0~custom.1).
|
||||
"""
|
||||
# Metadata first, as it may contain '-' too.
|
||||
release, plus, metadata = reported.partition("+")
|
||||
if release == DEV_VERSION and channel in DEV_VERSION_CHANNELS:
|
||||
return DEV_VERSION.replace("-", "~")
|
||||
assert channel != "develop", (
|
||||
f"unsupported version {reported!r}: "
|
||||
f"the develop channel only accepts {DEV_VERSION}."
|
||||
)
|
||||
base, _, pre_release = release.partition("-")
|
||||
if channel in ANY_VERSION_CHANNELS:
|
||||
pre_release = pre_release.replace("-", ".")
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
ARG BASE_IMAGE=debian:trixie
|
||||
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
# Bind-mounted rather than copied in, so the installer never lands in a layer.
|
||||
RUN --mount=type=bind,source=bin/install-packaging-tools.sh,target=/install-packaging-tools.sh \
|
||||
/install-packaging-tools.sh
|
||||
|
||||
# See ../README.md, "Publishing from other repositories".
|
||||
COPY package/docker/publish_pkg.py /usr/local/bin/publish_pkg.py
|
||||
File renamed without changes.
@@ -0,0 +1,15 @@
|
||||
FROM ubuntu:26.04
|
||||
|
||||
RUN --mount=type=bind,target=/tmp/package \
|
||||
apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends /tmp/package/*.deb \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -sf /dev/stdout /var/log/xrpld/debug.log
|
||||
|
||||
USER xrpld
|
||||
WORKDIR /var/lib/xrpld
|
||||
|
||||
EXPOSE 2459 5005 6006 50051
|
||||
|
||||
ENTRYPOINT ["/usr/bin/xrpld"]
|
||||
CMD ["--net", "--silent", "--conf", "/etc/xrpld/xrpld.cfg"]
|
||||
@@ -1,9 +0,0 @@
|
||||
ARG BASE_IMAGE=debian:trixie
|
||||
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
RUN --mount=type=bind,source=bin/install-packaging-tools.sh,target=/install-packaging-tools.sh \
|
||||
/install-packaging-tools.sh
|
||||
|
||||
# See package/README.md, "Publishing from other repositories".
|
||||
COPY package/images/packaging/publish_pkg.py /usr/local/bin/publish_pkg.py
|
||||
@@ -1,31 +0,0 @@
|
||||
FROM ubuntu:26.04 AS xrpld
|
||||
|
||||
RUN --mount=type=bind,target=/tmp/package \
|
||||
apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends /tmp/package/*.deb \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ln -sf /dev/stdout /var/log/xrpld/debug.log
|
||||
|
||||
USER xrpld
|
||||
WORKDIR /var/lib/xrpld
|
||||
|
||||
EXPOSE 2459 5005 6006 50051
|
||||
|
||||
ENTRYPOINT ["/usr/bin/xrpld"]
|
||||
CMD ["--net", "--silent", "--conf", "/etc/xrpld/xrpld.cfg"]
|
||||
|
||||
# The same image with the Antithesis-instrumented binary, which loads
|
||||
# libvoidstar.so; Antithesis reads the debug symbols from /symbols.
|
||||
FROM xrpld AS voidstar
|
||||
|
||||
USER root
|
||||
|
||||
ADD --chmod=644 --checksum=sha256:d080cc85f1d8d96452bdaf0021a10fa3e4cc3c319840f1fadd2d33904e607095 \
|
||||
https://antithesis.com/assets/instrumentation/libvoidstar.so /usr/lib/libvoidstar.so
|
||||
|
||||
RUN --mount=type=bind,source=xrpld,target=/tmp/xrpld \
|
||||
install -m 755 /tmp/xrpld /usr/bin/xrpld \
|
||||
&& mkdir /symbols \
|
||||
&& ln -s /usr/bin/xrpld /symbols/xrpld
|
||||
|
||||
USER xrpld
|
||||
@@ -13,29 +13,34 @@
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#ifndef XRPLD_VERSION
|
||||
#error "XRPLD_VERSION must be defined"
|
||||
#endif
|
||||
|
||||
namespace xrpl::build_info {
|
||||
|
||||
namespace {
|
||||
|
||||
// Set by cmake/XrplVersion.cmake, see http://semver.org/
|
||||
constexpr char kVersionString[] = XRPLD_VERSION;
|
||||
//--------------------------------------------------------------------------
|
||||
// The build version number. You must edit this for each release
|
||||
// and follow the format described at http://semver.org/
|
||||
//------------------------------------------------------------------------------
|
||||
// clang-format off
|
||||
// NOLINTNEXTLINE(readability-identifier-naming)
|
||||
char const* const versionString = "3.5.0-b0"
|
||||
// clang-format on
|
||||
;
|
||||
|
||||
//
|
||||
// Don't touch anything below this line
|
||||
//
|
||||
|
||||
std::string
|
||||
buildVersionString()
|
||||
{
|
||||
std::string version = kVersionString;
|
||||
std::string version = versionString;
|
||||
|
||||
#if defined(DEBUG) || defined(SANITIZERS)
|
||||
// A version derived from git already names its commit in the metadata.
|
||||
bool const hasMetadata = version.contains('+');
|
||||
std::string metadata;
|
||||
|
||||
std::string const& commitHash = xrpl::git::getCommitHash();
|
||||
if (!hasMetadata && !commitHash.empty())
|
||||
if (!commitHash.empty())
|
||||
metadata += commitHash + ".";
|
||||
|
||||
#ifdef DEBUG
|
||||
@@ -51,7 +56,7 @@ buildVersionString()
|
||||
#endif
|
||||
|
||||
if (!metadata.empty())
|
||||
version += (hasMetadata ? "." : "+") + metadata;
|
||||
version += "+" + metadata;
|
||||
#endif
|
||||
|
||||
return version;
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/beast/unit_test/suite.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/json/json_value.h>
|
||||
@@ -129,6 +128,7 @@ public:
|
||||
.ledgerMaster = app.getLedgerMaster(),
|
||||
.consumer = c,
|
||||
.role = Role::USER,
|
||||
.coro = {},
|
||||
.infoSub = {},
|
||||
.apiVersion = rpc::kApiVersionIfUnspecified},
|
||||
{},
|
||||
@@ -140,28 +140,22 @@ public:
|
||||
numSrc.reserve(rpc::tuning::kMaxSrcCur);
|
||||
for (std::uint8_t i = 0; i < rpc::tuning::kMaxSrcCur; ++i)
|
||||
numSrc.push_back(makeMptID(i, bob));
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = xrpl::test::detail::rpf(alice, bob, usd, numSrc);
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
BEAST_EXPECT(!result.isMember(jss::error));
|
||||
|
||||
// Test more than rpc::tuning::max_src_cur source currencies.
|
||||
numSrc.push_back(makeMptID(rpc::tuning::kMaxSrcCur, bob));
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = xrpl::test::detail::rpf(alice, bob, usd, numSrc);
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
BEAST_EXPECT(result.isMember(jss::error));
|
||||
@@ -173,28 +167,22 @@ public:
|
||||
auto curm = MPTTester({.env = env, .issuer = alice, .holders = {bob}});
|
||||
numSrc.push_back(curm.issuanceID());
|
||||
}
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = xrpl::test::detail::rpf(alice, bob, usd, {});
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
BEAST_EXPECT(!result.isMember(jss::error));
|
||||
|
||||
// Test more than rpc::tuning::max_auto_src_cur source currencies.
|
||||
auto curm = MPTTester({.env = env, .issuer = alice, .holders = {bob}});
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = xrpl::test::detail::rpf(alice, bob, usd, {});
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
BEAST_EXPECT(result.isMember(jss::error));
|
||||
|
||||
+12
-26
@@ -27,7 +27,6 @@
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/basics/hardened_hash.h>
|
||||
#include <xrpl/beast/unit_test/suite.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/json/json_reader.h>
|
||||
@@ -170,6 +169,7 @@ public:
|
||||
.ledgerMaster = app.getLedgerMaster(),
|
||||
.consumer = c,
|
||||
.role = Role::USER,
|
||||
.coro = {},
|
||||
.infoSub = {},
|
||||
.apiVersion = rpc::kApiVersionIfUnspecified},
|
||||
{},
|
||||
@@ -198,14 +198,11 @@ public:
|
||||
|
||||
json::Value result;
|
||||
Gate g;
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = std::move(params);
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
@@ -288,6 +285,7 @@ public:
|
||||
.ledgerMaster = app.getLedgerMaster(),
|
||||
.consumer = c,
|
||||
.role = Role::USER,
|
||||
.coro = {},
|
||||
.infoSub = {},
|
||||
.apiVersion = rpc::kApiVersionIfUnspecified},
|
||||
{},
|
||||
@@ -295,27 +293,21 @@ public:
|
||||
json::Value result;
|
||||
Gate g;
|
||||
// Test rpc::tuning::max_src_cur source currencies.
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = rpf(Account("alice"), Account("bob"), rpc::tuning::kMaxSrcCur);
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
BEAST_EXPECT(!result.isMember(jss::error));
|
||||
|
||||
// Test more than rpc::tuning::max_src_cur source currencies.
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = rpf(Account("alice"), Account("bob"), rpc::tuning::kMaxSrcCur + 1);
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
BEAST_EXPECT(result.isMember(jss::error));
|
||||
@@ -323,28 +315,22 @@ public:
|
||||
// Test rpc::tuning::max_auto_src_cur source currencies.
|
||||
for (auto i = 0; i < (rpc::tuning::kMaxAutoSrcCur - 1); ++i)
|
||||
env.trust(Account("alice")[std::to_string(i + 100)](100), "bob");
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = rpf(Account("alice"), Account("bob"), 0);
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
BEAST_EXPECT(!result.isMember(jss::error));
|
||||
|
||||
// Test more than rpc::tuning::max_auto_src_cur source currencies.
|
||||
env.trust(Account("alice")["AUD"](100), "bob");
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = rpf(Account("alice"), Account("bob"), 0);
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
BEAST_EXPECT(result.isMember(jss::error));
|
||||
|
||||
@@ -23,9 +23,8 @@
|
||||
#include <xrpl/basics/chrono.h>
|
||||
#include <xrpl/basics/strHex.h>
|
||||
#include <xrpl/beast/unit_test/suite.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/beast/utility/Zero.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/core/ServiceRegistry.h>
|
||||
#include <xrpl/json/json_value.h>
|
||||
#include <xrpl/json/to_string.h>
|
||||
@@ -1639,6 +1638,7 @@ struct PayChan_test : public beast::unit_test::Suite
|
||||
.ledgerMaster = app.getLedgerMaster(),
|
||||
.consumer = c,
|
||||
.role = Role::USER,
|
||||
.coro = {},
|
||||
.infoSub = {},
|
||||
.apiVersion = rpc::kApiVersionIfUnspecified},
|
||||
{},
|
||||
@@ -1654,13 +1654,10 @@ struct PayChan_test : public beast::unit_test::Suite
|
||||
BEAST_EXPECT(context.loadType == resource::kFeeReferenceRpc);
|
||||
json::Value result;
|
||||
Gate g;
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.coro = coro;
|
||||
result = doChannelVerify(context);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
@@ -1708,6 +1705,7 @@ struct PayChan_test : public beast::unit_test::Suite
|
||||
.ledgerMaster = app.getLedgerMaster(),
|
||||
.consumer = c,
|
||||
.role = Role::ADMIN, // channel_authorize requires ADMIN or canSign()
|
||||
.coro = {},
|
||||
.infoSub = {},
|
||||
.apiVersion = rpc::kApiVersionIfUnspecified},
|
||||
{},
|
||||
@@ -1722,13 +1720,10 @@ struct PayChan_test : public beast::unit_test::Suite
|
||||
BEAST_EXPECT(context.loadType == resource::kFeeReferenceRpc);
|
||||
json::Value result;
|
||||
Gate g;
|
||||
// Safe capture: the test blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.coro = coro;
|
||||
result = doChannelAuthorize(context);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
@@ -1207,7 +1207,7 @@ public:
|
||||
|
||||
auto backend{node_store::Manager::instance().makeBackend(
|
||||
section,
|
||||
megabytes(env.app().config().getValueFor(SizedItem::BurstSize, std::nullopt)),
|
||||
megabytes(env.app().config().getValueFor(SizedItem::BurstSize)),
|
||||
scheduler,
|
||||
env.app().getJournal("NodeStoreTest"))};
|
||||
backend->open();
|
||||
@@ -1229,22 +1229,12 @@ public:
|
||||
// Normally, SHAMapStoreImp handles all these details.
|
||||
auto nscfg = env.app().config().section(Sections::kNodeDatabase);
|
||||
|
||||
// Provide default values.
|
||||
// Provide default values (mirrors SHAMapStoreImp::makeNodeStore).
|
||||
if (!nscfg.exists(Keys::kCacheSize))
|
||||
{
|
||||
nscfg.set(
|
||||
Keys::kCacheSize,
|
||||
std::to_string(
|
||||
env.app().config().getValueFor(SizedItem::TreeCacheSize, std::nullopt)));
|
||||
}
|
||||
nscfg.set(Keys::kCacheSize, "16384");
|
||||
|
||||
if (!nscfg.exists(Keys::kCacheAge))
|
||||
{
|
||||
nscfg.set(
|
||||
Keys::kCacheAge,
|
||||
std::to_string(
|
||||
env.app().config().getValueFor(SizedItem::TreeCacheAge, std::nullopt)));
|
||||
}
|
||||
nscfg.set(Keys::kCacheAge, "5");
|
||||
|
||||
NodeStoreScheduler scheduler(env.app().getJobQueue());
|
||||
|
||||
|
||||
@@ -595,6 +595,108 @@ main
|
||||
BEAST_EXPECT(c.networkId == 10000);
|
||||
}
|
||||
|
||||
void
|
||||
testMemoryLimit()
|
||||
{
|
||||
testcase("memory limit");
|
||||
|
||||
{
|
||||
Config c;
|
||||
c.loadFromString("");
|
||||
BEAST_EXPECT(!c.memoryLimit);
|
||||
}
|
||||
|
||||
auto const parse = [](std::string const& value) {
|
||||
Config c;
|
||||
c.loadFromString("[memory_limit]\n" + value + "\n");
|
||||
return c;
|
||||
};
|
||||
|
||||
BEAST_EXPECT(parse("16").memoryLimit == std::uint64_t{16} << 30);
|
||||
BEAST_EXPECT(parse("0").memoryLimit == std::uint64_t{0});
|
||||
BEAST_EXPECT(parse("0").cacheMemoryBudget() == 0);
|
||||
|
||||
// Garbage and out-of-range values are rejected.
|
||||
expectException([&parse] { parse("banana"); });
|
||||
expectException([&parse] { parse("2000"); });
|
||||
|
||||
// Standalone mode does not change the budget: detected RAM unless
|
||||
// a limit is configured.
|
||||
{
|
||||
Config c;
|
||||
c.setupControl(true, true, true);
|
||||
c.loadFromString("[memory_limit]\n8\n");
|
||||
BEAST_EXPECT(c.cacheMemoryBudget() == std::uint64_t{8} << 30);
|
||||
}
|
||||
|
||||
// Values derive from the budget: half of it at 8 KiB per entry for
|
||||
// the tree cache; 0 yields the floors.
|
||||
BEAST_EXPECT(parse("16").getValueFor(SizedItem::TreeCacheSize) == 1048576);
|
||||
BEAST_EXPECT(parse("64").getValueFor(SizedItem::TreeCacheSize) == 4194304);
|
||||
BEAST_EXPECT(parse("0").getValueFor(SizedItem::TreeCacheSize) == 16384);
|
||||
BEAST_EXPECT(parse("16").getValueFor(SizedItem::TxnDbCache) == 32);
|
||||
BEAST_EXPECT(parse("0").getValueFor(SizedItem::TxnDbCache) == 4);
|
||||
BEAST_EXPECT(parse("16").getValueFor(SizedItem::SweepInterval) == 30);
|
||||
BEAST_EXPECT(parse("16").getValueFor(SizedItem::LedgerSize) == 96);
|
||||
BEAST_EXPECT(parse("16").getValueFor(SizedItem::BurstSize) == 16);
|
||||
BEAST_EXPECT(parse("1024").getValueFor(SizedItem::BurstSize) == 48);
|
||||
|
||||
// Deprecated [node_size] tiers are aliases for budgets (by name,
|
||||
// case-insensitively, or legacy 0-4 index); an explicit
|
||||
// [memory_limit] wins.
|
||||
auto const alias = [](std::string const& value) {
|
||||
Config c;
|
||||
c.loadFromString("[node_size]\n" + value + "\n");
|
||||
return c;
|
||||
};
|
||||
|
||||
BEAST_EXPECT(alias("large").cacheMemoryBudget() == std::uint64_t{64} << 30);
|
||||
BEAST_EXPECT(alias("large").getValueFor(SizedItem::TreeCacheSize) == 4194304);
|
||||
BEAST_EXPECT(alias("HUGE").cacheMemoryBudget() == std::uint64_t{128} << 30);
|
||||
BEAST_EXPECT(alias("3").cacheMemoryBudget() == std::uint64_t{64} << 30);
|
||||
BEAST_EXPECT(alias("9").cacheMemoryBudget() == std::uint64_t{128} << 30);
|
||||
|
||||
{
|
||||
Config c;
|
||||
c.loadFromString("[node_size]\nsmall\n\n[memory_limit]\n100\n");
|
||||
BEAST_EXPECT(c.cacheMemoryBudget() == std::uint64_t{100} << 30);
|
||||
}
|
||||
|
||||
// Policy values are fixed but individually overridable.
|
||||
{
|
||||
Config c;
|
||||
c.loadFromString("[tree_cache_age]\n900\n\n[ledger_fetch_size]\n8\n");
|
||||
BEAST_EXPECT(c.getValueFor(SizedItem::TreeCacheAge) == 900);
|
||||
BEAST_EXPECT(c.getValueFor(SizedItem::LedgerFetch) == 8);
|
||||
BEAST_EXPECT(c.getValueFor(SizedItem::LedgerAge) == 180);
|
||||
}
|
||||
expectException([] {
|
||||
Config c;
|
||||
c.loadFromString("[ledger_fetch_size]\n100\n");
|
||||
});
|
||||
}
|
||||
|
||||
void
|
||||
testMountinfoPath()
|
||||
{
|
||||
testcase("mountinfo path");
|
||||
|
||||
using detail::decodeMountinfoPath;
|
||||
|
||||
// The kernel escapes space, tab, newline and backslash as octal.
|
||||
BEAST_EXPECT(decodeMountinfoPath("/sys/fs/cgroup") == "/sys/fs/cgroup");
|
||||
BEAST_EXPECT(decodeMountinfoPath("/var/lib/cgroup\\040mounts") == "/var/lib/cgroup mounts");
|
||||
BEAST_EXPECT(decodeMountinfoPath("a\\011b\\012c\\134d") == "a\tb\nc\\d");
|
||||
BEAST_EXPECT(decodeMountinfoPath("\\040\\040") == " ");
|
||||
|
||||
// Anything that is not a backslash and three octal digits is kept.
|
||||
BEAST_EXPECT(decodeMountinfoPath("\\04") == "\\04");
|
||||
BEAST_EXPECT(decodeMountinfoPath("\\") == "\\");
|
||||
BEAST_EXPECT(decodeMountinfoPath("\\0x0") == "\\0x0");
|
||||
BEAST_EXPECT(decodeMountinfoPath("\\400") == "\\400");
|
||||
BEAST_EXPECT(decodeMountinfoPath("") == "");
|
||||
}
|
||||
|
||||
void
|
||||
testValidatorsFile()
|
||||
{
|
||||
@@ -1676,6 +1778,8 @@ r.ripple.com:51235
|
||||
testAmendment();
|
||||
testOverlay();
|
||||
testNetworkID();
|
||||
testMemoryLimit();
|
||||
testMountinfoPath();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,581 +0,0 @@
|
||||
#include <test/jtx/Env.h>
|
||||
#include <test/jtx/envconfig.h>
|
||||
|
||||
#include <xrpld/core/Config.h>
|
||||
|
||||
#include <xrpl/basics/LocalValue.h>
|
||||
#include <xrpl/beast/unit_test/suite.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/core/JobQueueAwaiter.h>
|
||||
|
||||
#include <array>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// Tests intentionally capture state in coroutine lambdas; lifetimes are
|
||||
// controlled by Gate synchronization and join() before scope exit.
|
||||
// NOLINTBEGIN(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
|
||||
namespace xrpl::test {
|
||||
|
||||
/**
|
||||
* Test suite for the C++20 coroutine primitives: CoroTask, CoroTaskRunner,
|
||||
* and JobQueueAwaiter.
|
||||
*
|
||||
* Dependency Diagram
|
||||
* ==================
|
||||
*
|
||||
* CoroTask_test
|
||||
* +-------------------------------------------------+
|
||||
* | + Gate (inner class) : condition_variable helper |
|
||||
* +-------------------------------------------------+
|
||||
* | uses
|
||||
* v
|
||||
* jtx::Env --> JobQueue::postCoroTask()
|
||||
* |
|
||||
* +-- CoroTaskRunner (suspend / post / resume)
|
||||
* +-- CoroTask<void> / CoroTask<T>
|
||||
* +-- JobQueueAwaiter
|
||||
*
|
||||
* Test Coverage Matrix
|
||||
* ====================
|
||||
*
|
||||
* Test | Primitives exercised
|
||||
* --------------------------+----------------------------------------------
|
||||
* testVoidCompletion | CoroTask<void> basic lifecycle
|
||||
* testCorrectOrder | suspend() -> join() -> post() -> complete
|
||||
* testIncorrectOrder | post() before suspend() (race-safe path)
|
||||
* testJobQueueAwaiter | JobQueueAwaiter + yieldAndPost suspend/repost
|
||||
* testThreadSpecificStorage | LocalValue isolation across coroutines
|
||||
* testExceptionPropagation | CoroTask<void> exception via co_await
|
||||
* testMultipleYields | N sequential suspend/resume cycles
|
||||
* testValueReturn | CoroTask<T> co_return value
|
||||
* testValueException | CoroTask<T> exception via co_await
|
||||
* testValueChaining | nested CoroTask<T> -> CoroTask<T>
|
||||
* testShutdownRejection | postCoroTask returns nullptr when stopping
|
||||
*/
|
||||
class CoroTask_test : public beast::unit_test::Suite
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Simple one-shot gate for synchronizing between test thread
|
||||
* and coroutine worker threads. signal() sets the flag;
|
||||
* waitFor() blocks until signaled or timeout.
|
||||
*/
|
||||
class Gate
|
||||
{
|
||||
private:
|
||||
std::condition_variable cv_;
|
||||
std::mutex mutex_;
|
||||
bool signaled_ = false;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Block until signaled or timeout expires.
|
||||
*
|
||||
* @param relTime Maximum duration to wait
|
||||
*
|
||||
* @return true if signaled before timeout
|
||||
*/
|
||||
template <class Rep, class Period>
|
||||
bool
|
||||
waitFor(std::chrono::duration<Rep, Period> const& relTime)
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mutex_);
|
||||
auto b = cv_.wait_for(lk, relTime, [this] { return signaled_; });
|
||||
signaled_ = false;
|
||||
return b;
|
||||
}
|
||||
|
||||
/**
|
||||
* Signal the gate, waking any waiting thread.
|
||||
*/
|
||||
void
|
||||
signal()
|
||||
{
|
||||
std::scoped_lock const lk(mutex_);
|
||||
signaled_ = true;
|
||||
cv_.notify_all();
|
||||
}
|
||||
};
|
||||
|
||||
// NOTE: All coroutine lambdas passed to postCoroTask use explicit
|
||||
// pointer-by-value captures instead of [&] to work around a GCC 14
|
||||
// bug where reference captures in coroutine lambdas are corrupted
|
||||
// in the coroutine frame.
|
||||
|
||||
/**
|
||||
* CoroTask<void> runs to completion and runner becomes non-runnable.
|
||||
*/
|
||||
void
|
||||
testVoidCompletion()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("void completion");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
Gate g;
|
||||
auto runner = env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [gp = &g](auto) -> CoroTask<void> {
|
||||
gp->signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(runner);
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
runner->join();
|
||||
BEAST_EXPECT(!runner->runnable());
|
||||
}
|
||||
|
||||
/**
|
||||
* Correct order: suspend, join, post, complete.
|
||||
* Mirrors existing Coroutine_test::correct_order.
|
||||
*/
|
||||
void
|
||||
testCorrectOrder()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("correct order");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
Gate g1, g2;
|
||||
auto runner = env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [g1p = &g1, g2p = &g2](auto runner) -> CoroTask<void> {
|
||||
g1p->signal();
|
||||
co_await runner->suspend();
|
||||
g2p->signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(runner);
|
||||
if (!BEAST_EXPECT(g1.waitFor(5s)))
|
||||
return;
|
||||
runner->join();
|
||||
runner->post();
|
||||
if (!BEAST_EXPECT(g2.waitFor(5s)))
|
||||
return;
|
||||
runner->join();
|
||||
}
|
||||
|
||||
/**
|
||||
* Incorrect order: post() before suspend(). Verifies the
|
||||
* race-safe path. Mirrors Coroutine_test::incorrect_order.
|
||||
*/
|
||||
void
|
||||
testIncorrectOrder()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("incorrect order");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
Gate g;
|
||||
env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [gp = &g](auto runner) -> CoroTask<void> {
|
||||
runner->post();
|
||||
co_await runner->suspend();
|
||||
gp->signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
}
|
||||
|
||||
/**
|
||||
* Suspend + auto-repost across multiple yield points, using the
|
||||
* external JobQueueAwaiter struct for the first suspension and the
|
||||
* inline yieldAndPost() awaiter for the second. JobQueueAwaiter is
|
||||
* used at only one co_await point per coroutine (see the GCC-12
|
||||
* multi-use warning in JobQueueAwaiter.h).
|
||||
*/
|
||||
void
|
||||
testJobQueueAwaiter()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("JobQueueAwaiter");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
Gate g;
|
||||
std::vector<int> steps;
|
||||
auto runner = env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [sp = &steps, gp = &g](auto runner) -> CoroTask<void> {
|
||||
sp->push_back(1);
|
||||
co_await JobQueueAwaiter{runner};
|
||||
sp->push_back(2);
|
||||
co_await runner->yieldAndPost();
|
||||
sp->push_back(3);
|
||||
gp->signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(runner);
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
runner->join();
|
||||
BEAST_EXPECT(steps == std::vector<int>({1, 2, 3}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-coroutine LocalValue isolation. Each coroutine sees its own
|
||||
* copy of thread-local state. Mirrors Coroutine_test::thread_specific_storage.
|
||||
*/
|
||||
void
|
||||
testThreadSpecificStorage()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("thread specific storage");
|
||||
Env env(*this);
|
||||
|
||||
auto& jq = env.app().getJobQueue();
|
||||
|
||||
static constexpr int kN = 4;
|
||||
std::array<std::shared_ptr<JobQueue::CoroTaskRunner>, kN> a;
|
||||
|
||||
LocalValue<int> lv(-1);
|
||||
BEAST_EXPECT(*lv == -1);
|
||||
|
||||
Gate g;
|
||||
jq.addJob(JtClient, "LocalValTest", [&]() {
|
||||
this->BEAST_EXPECT(*lv == -1);
|
||||
*lv = -2;
|
||||
this->BEAST_EXPECT(*lv == -2);
|
||||
g.signal();
|
||||
});
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
BEAST_EXPECT(*lv == -1);
|
||||
|
||||
for (int i = 0; i < kN; ++i)
|
||||
{
|
||||
jq.postCoroTask(
|
||||
JtClient,
|
||||
"CoroTaskTest",
|
||||
[this, ap = &a, gp = &g, lvp = &lv, id = i](auto runner) -> CoroTask<void> {
|
||||
(*ap)[id] = runner;
|
||||
gp->signal();
|
||||
co_await runner->suspend();
|
||||
|
||||
this->BEAST_EXPECT(**lvp == -1);
|
||||
**lvp = id;
|
||||
this->BEAST_EXPECT(**lvp == id);
|
||||
gp->signal();
|
||||
co_await runner->suspend();
|
||||
|
||||
this->BEAST_EXPECT(**lvp == id);
|
||||
co_return;
|
||||
});
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
a[i]->join();
|
||||
}
|
||||
for (auto const& r : a)
|
||||
{
|
||||
r->post();
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
r->join();
|
||||
}
|
||||
for (auto const& r : a)
|
||||
{
|
||||
r->post();
|
||||
r->join();
|
||||
}
|
||||
|
||||
jq.addJob(JtClient, "LocalValTest", [&]() {
|
||||
this->BEAST_EXPECT(*lv == -2);
|
||||
g.signal();
|
||||
});
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
BEAST_EXPECT(*lv == -1);
|
||||
}
|
||||
|
||||
/**
|
||||
* An exception thrown in an awaited CoroTask<void> is rethrown into
|
||||
* the awaiting coroutine by await_resume(), with the original
|
||||
* message intact. (An exception escaping the top-level body has no
|
||||
* awaiter to rethrow it; it is captured by unhandled_exception()
|
||||
* and logged by CoroTaskRunner::resume().)
|
||||
*/
|
||||
void
|
||||
testExceptionPropagation()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("exception propagation");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
Gate g;
|
||||
std::string what;
|
||||
auto runner = env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [wp = &what, gp = &g](auto) -> CoroTask<void> {
|
||||
auto inner = []() -> CoroTask<void> {
|
||||
throw std::runtime_error("test exception");
|
||||
co_return;
|
||||
};
|
||||
try
|
||||
{
|
||||
co_await inner();
|
||||
}
|
||||
catch (std::runtime_error const& e)
|
||||
{
|
||||
*wp = e.what();
|
||||
}
|
||||
gp->signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(runner);
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
runner->join();
|
||||
BEAST_EXPECT(what == "test exception");
|
||||
BEAST_EXPECT(!runner->runnable());
|
||||
}
|
||||
|
||||
/**
|
||||
* Multiple sequential suspend/resume cycles via co_await.
|
||||
*/
|
||||
void
|
||||
testMultipleYields()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("multiple yields");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
Gate g;
|
||||
int counter = 0;
|
||||
auto runner = env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [cp = &counter, gp = &g](auto runner) -> CoroTask<void> {
|
||||
++(*cp);
|
||||
gp->signal();
|
||||
co_await runner->suspend();
|
||||
++(*cp);
|
||||
gp->signal();
|
||||
co_await runner->suspend();
|
||||
++(*cp);
|
||||
gp->signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(runner);
|
||||
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
BEAST_EXPECT(counter == 1);
|
||||
runner->join();
|
||||
|
||||
runner->post();
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
BEAST_EXPECT(counter == 2);
|
||||
runner->join();
|
||||
|
||||
runner->post();
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
BEAST_EXPECT(counter == 3);
|
||||
runner->join();
|
||||
BEAST_EXPECT(!runner->runnable());
|
||||
}
|
||||
|
||||
/**
|
||||
* CoroTask<T> returns a value via co_return. Outer coroutine
|
||||
* extracts it with co_await.
|
||||
*/
|
||||
void
|
||||
testValueReturn()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("value return");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
Gate g;
|
||||
int result = 0;
|
||||
auto runner = env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [rp = &result, gp = &g](auto) -> CoroTask<void> {
|
||||
auto inner = []() -> CoroTask<int> { co_return 42; };
|
||||
*rp = co_await inner();
|
||||
gp->signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(runner);
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
runner->join();
|
||||
BEAST_EXPECT(result == 42);
|
||||
BEAST_EXPECT(!runner->runnable());
|
||||
}
|
||||
|
||||
/**
|
||||
* CoroTask<T> propagates exceptions from inner coroutines.
|
||||
* Outer coroutine catches via try/catch around co_await.
|
||||
*/
|
||||
void
|
||||
testValueException()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("value exception");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
Gate g;
|
||||
std::string what;
|
||||
auto runner = env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [wp = &what, gp = &g](auto) -> CoroTask<void> {
|
||||
auto inner = []() -> CoroTask<int> {
|
||||
throw std::runtime_error("inner error");
|
||||
co_return 0;
|
||||
};
|
||||
try
|
||||
{
|
||||
co_await inner();
|
||||
}
|
||||
catch (std::runtime_error const& e)
|
||||
{
|
||||
*wp = e.what();
|
||||
}
|
||||
gp->signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(runner);
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
runner->join();
|
||||
BEAST_EXPECT(what == "inner error");
|
||||
BEAST_EXPECT(!runner->runnable());
|
||||
}
|
||||
|
||||
/**
|
||||
* CoroTask<T> chaining. Nested value-returning coroutines
|
||||
* compose via co_await.
|
||||
*/
|
||||
void
|
||||
testValueChaining()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("value chaining");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
Gate g;
|
||||
int result = 0;
|
||||
auto runner = env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [rp = &result, gp = &g](auto) -> CoroTask<void> {
|
||||
auto add = [](int a, int b) -> CoroTask<int> { co_return a + b; };
|
||||
auto mul = [add](int a, int b) -> CoroTask<int> {
|
||||
int const sum = co_await add(a, b);
|
||||
co_return sum * 2;
|
||||
};
|
||||
*rp = co_await mul(3, 4);
|
||||
gp->signal();
|
||||
co_return;
|
||||
});
|
||||
BEAST_EXPECT(runner);
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
runner->join();
|
||||
BEAST_EXPECT(result == 14); // (3 + 4) * 2
|
||||
BEAST_EXPECT(!runner->runnable());
|
||||
}
|
||||
|
||||
/**
|
||||
* postCoroTask returns nullptr when JobQueue is stopping.
|
||||
*/
|
||||
void
|
||||
testShutdownRejection()
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
using namespace jtx;
|
||||
|
||||
testcase("shutdown rejection");
|
||||
|
||||
Env env(*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->forceMultiThread = true;
|
||||
return cfg;
|
||||
}));
|
||||
|
||||
// Stop the JobQueue
|
||||
env.app().getJobQueue().stop();
|
||||
|
||||
auto runner = env.app().getJobQueue().postCoroTask(
|
||||
JtClient, "CoroTaskTest", [](auto) -> CoroTask<void> { co_return; });
|
||||
BEAST_EXPECT(!runner);
|
||||
}
|
||||
|
||||
void
|
||||
run() override
|
||||
{
|
||||
testVoidCompletion();
|
||||
testCorrectOrder();
|
||||
testIncorrectOrder();
|
||||
testJobQueueAwaiter();
|
||||
testThreadSpecificStorage();
|
||||
testExceptionPropagation();
|
||||
testMultipleYields();
|
||||
testValueReturn();
|
||||
testValueException();
|
||||
testValueChaining();
|
||||
testShutdownRejection();
|
||||
}
|
||||
};
|
||||
|
||||
BEAST_DEFINE_TESTSUITE(CoroTask, core, xrpl);
|
||||
|
||||
} // namespace xrpl::test
|
||||
|
||||
// NOLINTEND(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
@@ -6,7 +6,6 @@
|
||||
|
||||
#include <xrpl/basics/LocalValue.h>
|
||||
#include <xrpl/beast/unit_test/suite.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
|
||||
@@ -50,11 +49,6 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
// NOTE: All coroutine lambdas passed to postCoroTask use explicit
|
||||
// pointer-by-value captures instead of [&] to work around a GCC 14
|
||||
// bug where reference captures in coroutine lambdas are corrupted
|
||||
// in the coroutine frame.
|
||||
|
||||
void
|
||||
correctOrder()
|
||||
{
|
||||
@@ -69,22 +63,14 @@ public:
|
||||
}));
|
||||
|
||||
Gate g1, g2;
|
||||
std::shared_ptr<JobQueue::CoroTaskRunner> c;
|
||||
env.app().getJobQueue().postCoroTask(
|
||||
JtClient,
|
||||
"CoroTest",
|
||||
// Safe capture: the test blocks on the Gates until the coroutine
|
||||
// completes, so the captured pointers outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
[cp = &c, g1p = &g1, g2p = &g2](auto runner) -> CoroTask<void> {
|
||||
*cp = runner;
|
||||
g1p->signal();
|
||||
co_await runner->suspend();
|
||||
g2p->signal();
|
||||
co_return;
|
||||
});
|
||||
if (!BEAST_EXPECT(g1.waitFor(5s)))
|
||||
return;
|
||||
std::shared_ptr<JobQueue::Coro> c;
|
||||
env.app().getJobQueue().postCoro(JtClient, "CoroTest", [&](auto const& cr) {
|
||||
c = cr;
|
||||
g1.signal();
|
||||
c->yield();
|
||||
g2.signal();
|
||||
});
|
||||
BEAST_EXPECT(g1.waitFor(5s));
|
||||
c->join();
|
||||
c->post();
|
||||
BEAST_EXPECT(g2.waitFor(5s));
|
||||
@@ -104,22 +90,11 @@ public:
|
||||
}));
|
||||
|
||||
Gate g;
|
||||
env.app().getJobQueue().postCoroTask(
|
||||
JtClient,
|
||||
"CoroTest",
|
||||
// Safe capture: the test blocks on the Gate until the coroutine
|
||||
// completes, so the captured pointer outlives the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
[gp = &g](auto runner) -> CoroTask<void> {
|
||||
// Schedule a resume before suspending. The posted job
|
||||
// cannot actually call resume() until the current resume()
|
||||
// releases CoroTaskRunner::mutex_, which only happens after
|
||||
// the coroutine suspends at co_await.
|
||||
runner->post();
|
||||
co_await runner->suspend();
|
||||
gp->signal();
|
||||
co_return;
|
||||
});
|
||||
env.app().getJobQueue().postCoro(JtClient, "CoroTest", [&](auto const& c) {
|
||||
c->post();
|
||||
c->yield();
|
||||
g.signal();
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
}
|
||||
|
||||
@@ -135,7 +110,7 @@ public:
|
||||
auto& jq = env.app().getJobQueue();
|
||||
|
||||
static int const kN = 4;
|
||||
std::array<std::shared_ptr<JobQueue::CoroTaskRunner>, kN> a;
|
||||
std::array<std::shared_ptr<JobQueue::Coro>, kN> a;
|
||||
|
||||
LocalValue<int> lv(-1);
|
||||
BEAST_EXPECT(*lv == -1);
|
||||
@@ -152,36 +127,26 @@ public:
|
||||
|
||||
for (int i = 0; i < kN; ++i)
|
||||
{
|
||||
jq.postCoroTask(
|
||||
JtClient,
|
||||
"CoroTest",
|
||||
// Safe capture: the test drives every coroutine to completion
|
||||
// via the Gate/post()/join() sequences below, so the captured
|
||||
// pointers outlive the coroutines.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
[this, ap = &a, gp = &g, lvp = &lv, id = i](auto runner) -> CoroTask<void> {
|
||||
(*ap)[id] = runner;
|
||||
gp->signal();
|
||||
co_await runner->suspend();
|
||||
jq.postCoro(JtClient, "CoroTest", [&, id = i](auto const& c) {
|
||||
a[id] = c;
|
||||
g.signal();
|
||||
c->yield();
|
||||
|
||||
this->BEAST_EXPECT(**lvp == -1);
|
||||
**lvp = id;
|
||||
this->BEAST_EXPECT(**lvp == id);
|
||||
gp->signal();
|
||||
co_await runner->suspend();
|
||||
this->BEAST_EXPECT(*lv == -1);
|
||||
*lv = id;
|
||||
this->BEAST_EXPECT(*lv == id);
|
||||
g.signal();
|
||||
c->yield();
|
||||
|
||||
this->BEAST_EXPECT(**lvp == id);
|
||||
co_return;
|
||||
});
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
this->BEAST_EXPECT(*lv == id);
|
||||
});
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
a[i]->join();
|
||||
}
|
||||
for (auto const& c : a)
|
||||
{
|
||||
c->post();
|
||||
if (!BEAST_EXPECT(g.waitFor(5s)))
|
||||
return;
|
||||
BEAST_EXPECT(g.waitFor(5s));
|
||||
c->join();
|
||||
}
|
||||
for (auto const& c : a)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
#include <test/jtx/Env.h>
|
||||
|
||||
#include <xrpl/beast/unit_test/suite.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
|
||||
namespace xrpl::test {
|
||||
|
||||
@@ -46,114 +46,87 @@ class JobQueue_test : public beast::unit_test::Suite
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: All coroutine lambdas passed to postCoroTask use explicit
|
||||
// pointer-by-value captures instead of [&] to work around a GCC 14
|
||||
// bug where reference captures in coroutine lambdas are corrupted
|
||||
// in the coroutine frame.
|
||||
|
||||
void
|
||||
testPostCoroTask()
|
||||
testPostCoro()
|
||||
{
|
||||
jtx::Env env{*this};
|
||||
|
||||
JobQueue& jQueue = env.app().getJobQueue();
|
||||
{
|
||||
// Test repeated post()s until the coroutine completes.
|
||||
// Test repeated post()s until the Coro completes.
|
||||
std::atomic<int> yieldCount{0};
|
||||
auto const runner = jQueue.postCoroTask(
|
||||
auto const coro = jQueue.postCoro(
|
||||
JtClient,
|
||||
"PostCoroTest1",
|
||||
// Safe capture: the test blocks below until the coroutine
|
||||
// completes, so the captured pointer outlives the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
[ycp = &yieldCount](auto runner) -> CoroTask<void> {
|
||||
while (++(*ycp) < 4)
|
||||
co_await runner->suspend();
|
||||
co_return;
|
||||
[&yieldCount](std::shared_ptr<JobQueue::Coro> const& coroCopy) {
|
||||
while (++yieldCount < 4)
|
||||
coroCopy->yield();
|
||||
});
|
||||
BEAST_EXPECT(runner != nullptr);
|
||||
BEAST_EXPECT(coro != nullptr);
|
||||
|
||||
// Wait for the Job to run and yield.
|
||||
while (yieldCount == 0)
|
||||
;
|
||||
|
||||
// Now re-post until the CoroTaskRunner says it is done.
|
||||
// Now re-post until the Coro says it is done.
|
||||
int old = yieldCount;
|
||||
while (runner->runnable())
|
||||
while (coro->runnable())
|
||||
{
|
||||
BEAST_EXPECT(runner->post());
|
||||
BEAST_EXPECT(coro->post());
|
||||
while (old == yieldCount)
|
||||
{
|
||||
}
|
||||
runner->join();
|
||||
coro->join();
|
||||
BEAST_EXPECT(++old == yieldCount);
|
||||
}
|
||||
BEAST_EXPECT(yieldCount == 4);
|
||||
}
|
||||
{
|
||||
// Test repeated post()+join()s until the coroutine completes.
|
||||
// Test repeated resume()s until the Coro completes.
|
||||
int yieldCount{0};
|
||||
auto const runner = jQueue.postCoroTask(
|
||||
auto const coro = jQueue.postCoro(
|
||||
JtClient,
|
||||
"PostCoroTest2",
|
||||
// Safe capture: the test blocks below until the coroutine
|
||||
// completes, so the captured pointer outlives the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
[ycp = &yieldCount](auto runner) -> CoroTask<void> {
|
||||
while (++(*ycp) < 4)
|
||||
co_await runner->suspend();
|
||||
co_return;
|
||||
[&yieldCount](std::shared_ptr<JobQueue::Coro> const& coroCopy) {
|
||||
while (++yieldCount < 4)
|
||||
coroCopy->yield();
|
||||
});
|
||||
if (!runner)
|
||||
if (!coro)
|
||||
{
|
||||
// There's no good reason we should not get a runner, but we
|
||||
// There's no good reason we should not get a Coro, but we
|
||||
// can't continue without one.
|
||||
BEAST_EXPECT(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait for the Job to run and yield.
|
||||
runner->join();
|
||||
coro->join();
|
||||
|
||||
// Now post()+join() until the CoroTaskRunner says it is done.
|
||||
// resume() requires a prior post() (see the precondition on
|
||||
// CoroTaskRunner::resume()), so the posted job performs the
|
||||
// resume and join() blocks until it completes. yieldCount is
|
||||
// deliberately not atomic: the mutexRun_ handoff inside join()
|
||||
// must provide the happens-before edge that makes the
|
||||
// increment visible to this thread.
|
||||
// Now resume until the Coro says it is done.
|
||||
int old = yieldCount;
|
||||
while (runner->runnable())
|
||||
while (coro->runnable())
|
||||
{
|
||||
BEAST_EXPECT(runner->post());
|
||||
runner->join();
|
||||
coro->resume(); // Resume runs synchronously on this thread.
|
||||
BEAST_EXPECT(++old == yieldCount);
|
||||
}
|
||||
BEAST_EXPECT(yieldCount == 4);
|
||||
}
|
||||
{
|
||||
// If the JobQueue is stopped, we should no
|
||||
// longer be able to post a coroutine (and calling postCoroTask()
|
||||
// should return nullptr).
|
||||
// longer be able to add a Coro (and calling postCoro() should
|
||||
// return false).
|
||||
using namespace std::chrono_literals;
|
||||
jQueue.stop();
|
||||
|
||||
// The coroutine should never run, so having it access this
|
||||
// The Coro should never run, so having the Coro access this
|
||||
// unprotected variable on the stack should be completely safe.
|
||||
// Not recommended for the faint of heart...
|
||||
bool unprotected = false;
|
||||
auto const runner = jQueue.postCoroTask(
|
||||
JtClient,
|
||||
"PostCoroTest3",
|
||||
// Safe capture: the JobQueue is stopped, so the coroutine is
|
||||
// never started and the captured pointer is never dereferenced.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
[up = &unprotected](auto) -> CoroTask<void> {
|
||||
*up = true;
|
||||
co_return;
|
||||
auto const coro = jQueue.postCoro(
|
||||
JtClient, "PostCoroTest3", [&unprotected](std::shared_ptr<JobQueue::Coro> const&) {
|
||||
unprotected = false;
|
||||
});
|
||||
BEAST_EXPECT(runner == nullptr);
|
||||
BEAST_EXPECT(unprotected == false);
|
||||
BEAST_EXPECT(coro == nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,7 +135,7 @@ public:
|
||||
run() override
|
||||
{
|
||||
testAddJob();
|
||||
testPostCoroTask();
|
||||
testPostCoro();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -23,9 +23,7 @@
|
||||
#include <xrpl/basics/strHex.h>
|
||||
#include <xrpl/beast/unit_test/suite.h>
|
||||
#include <xrpl/beast/utility/instrumentation.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/core/ServiceRegistry.h>
|
||||
#include <xrpl/json/json_value.h>
|
||||
#include <xrpl/json/to_string.h>
|
||||
@@ -225,6 +223,7 @@ findPathsRequest(
|
||||
.ledgerMaster = app.getLedgerMaster(),
|
||||
.consumer = c,
|
||||
.role = Role::USER,
|
||||
.coro = {},
|
||||
.infoSub = {},
|
||||
.apiVersion = rpc::kApiVersionIfUnspecified},
|
||||
{},
|
||||
@@ -251,14 +250,11 @@ findPathsRequest(
|
||||
|
||||
json::Value result;
|
||||
Gate g;
|
||||
// Safe capture: the caller blocks on g.waitFor() until the coroutine
|
||||
// completes, so the captured locals outlive the coroutine.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
app.getJobQueue().postCoroTask(JtClient, "RPC-Client", [&](auto) -> CoroTask<void> {
|
||||
app.getJobQueue().postCoro(JtClient, "RPC-Client", [&](auto const& coro) {
|
||||
context.params = std::move(params);
|
||||
context.coro = coro;
|
||||
rpc::doCommand(context, result);
|
||||
g.signal();
|
||||
co_return;
|
||||
});
|
||||
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
#include <xrpl/beast/core/SemanticVersion.h>
|
||||
#include <xrpl/beast/unit_test/suite.h>
|
||||
#include <xrpl/protocol/BuildInfo.h>
|
||||
|
||||
@@ -70,12 +69,8 @@ public:
|
||||
auto vFF = 0xFFFF'FFFF'FFFF'FFFFLLU;
|
||||
BEAST_EXPECT(!build_info::isNewerVersion(vFF));
|
||||
|
||||
// 1.5.9 is newer than a development build (0.0.0-dev), but not a release.
|
||||
auto v159 = build_info::encodeSoftwareVersion("1.5.9");
|
||||
BEAST_EXPECT(
|
||||
build_info::isNewerVersion(v159) ==
|
||||
(beast::SemanticVersion(build_info::getVersionString()) <
|
||||
beast::SemanticVersion("1.5.9")));
|
||||
BEAST_EXPECT(!build_info::isNewerVersion(v159));
|
||||
|
||||
auto vCurrent = build_info::getEncodedVersion();
|
||||
BEAST_EXPECT(!build_info::isNewerVersion(vCurrent));
|
||||
|
||||
@@ -301,7 +301,6 @@ public:
|
||||
using namespace std::chrono_literals;
|
||||
Env env{*this, envconfig([](std::unique_ptr<Config> cfg) {
|
||||
cfg->fees.referenceFee = 10;
|
||||
cfg->nodeSize = 0;
|
||||
return cfg;
|
||||
})};
|
||||
Account const gw{"gateway"};
|
||||
|
||||
@@ -80,6 +80,8 @@ admin = 127.0.0.1
|
||||
BEAST_EXPECT(result.isMember(jss::info));
|
||||
auto const& info = result[jss::info];
|
||||
BEAST_EXPECT(info.isMember(jss::build_version));
|
||||
// Admin request: reports the cache memory budget in GB.
|
||||
BEAST_EXPECT(info.isMember(jss::memory_limit));
|
||||
// Git info is not guaranteed to be present
|
||||
if (info.isMember(jss::git))
|
||||
{
|
||||
|
||||
@@ -4,15 +4,19 @@
|
||||
#include <xrpl/basics/IntrusiveRefCounts.h>
|
||||
#include <xrpl/basics/TaggedCache.ipp> // IWYU pragma: keep
|
||||
#include <xrpl/basics/chrono.h>
|
||||
#include <xrpl/beast/insight/NullCollector.h>
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
#include <xrpl/protocol/Protocol.h>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <helpers/TestSink.h>
|
||||
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
@@ -243,4 +247,290 @@ TEST(TaggedCacheTest, tagged_cache)
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TaggedCacheTest, hard_cap_enforced_on_insert)
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
beast::Journal const journal{TestSink::instance()};
|
||||
|
||||
TestStopwatch clock;
|
||||
clock.set(0);
|
||||
|
||||
using Key = LedgerIndex;
|
||||
using Value = std::string;
|
||||
using Cache = TaggedCache<Key, Value>;
|
||||
|
||||
// A cap-enabled cache must never let the strong-cache count exceed the
|
||||
// cap, enforced on the insert path alone (no sweep). The large targetSize
|
||||
// and long age make the periodic sweep irrelevant here, so only
|
||||
// evictForHardCap can be bounding it.
|
||||
int const cap = 100;
|
||||
Cache capped(
|
||||
"capped", 1'000'000, 3600s, clock, journal, beast::insight::NullCollector::make(), cap);
|
||||
|
||||
bool everExceeded = false;
|
||||
for (Key k = 1; k <= 1000; ++k)
|
||||
{
|
||||
capped.insert(k, "v");
|
||||
if (capped.getCacheSize() > cap)
|
||||
everExceeded = true;
|
||||
}
|
||||
EXPECT_FALSE(everExceeded);
|
||||
EXPECT_LE(capped.getCacheSize(), cap);
|
||||
EXPECT_GT(capped.getCacheSize(), 0);
|
||||
}
|
||||
|
||||
TEST(TaggedCacheTest, hard_cap_enforced_across_partitions)
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
beast::Journal const journal{TestSink::instance()};
|
||||
|
||||
TestStopwatch clock;
|
||||
clock.set(0);
|
||||
|
||||
using Key = LedgerIndex;
|
||||
using Value = std::string;
|
||||
using Cache = TaggedCache<Key, Value>;
|
||||
|
||||
// The cap is well below the partition count, so with sequential keys
|
||||
// (partitioned round-robin) the partition a given insert lands in holds
|
||||
// only that one entry when the count first crosses the cap: eviction
|
||||
// must fall through to another partition rather than stop at an empty
|
||||
// home partition and leave cacheCount_ stuck above cacheHardCap_.
|
||||
int const cap = 4;
|
||||
std::size_t const partitions = 16;
|
||||
Cache capped(
|
||||
"capped-partitions",
|
||||
1'000'000,
|
||||
3600s,
|
||||
clock,
|
||||
journal,
|
||||
beast::insight::NullCollector::make(),
|
||||
cap,
|
||||
partitions);
|
||||
|
||||
bool everExceeded = false;
|
||||
for (Key k = 1; k <= 2000; ++k)
|
||||
{
|
||||
capped.insert(k, "v");
|
||||
if (capped.getCacheSize() > cap)
|
||||
everExceeded = true;
|
||||
}
|
||||
EXPECT_FALSE(everExceeded);
|
||||
EXPECT_LE(capped.getCacheSize(), cap);
|
||||
EXPECT_GT(capped.getCacheSize(), 0);
|
||||
}
|
||||
|
||||
TEST(TaggedCacheTest, hard_cap_enforced_in_sparse_partition)
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
beast::Journal const journal{TestSink::instance()};
|
||||
|
||||
TestStopwatch clock;
|
||||
clock.set(0);
|
||||
|
||||
using Key = LedgerIndex;
|
||||
using Value = std::string;
|
||||
using Cache = TaggedCache<Key, Value>;
|
||||
|
||||
// A single partition whose bucket array grows into the thousands while
|
||||
// an external owner (held in `heldRefs`, the common case for a value
|
||||
// also referenced from elsewhere) keeps every demoted entry in the
|
||||
// table as weakly-tracked rather than erased: the handful of strong
|
||||
// survivors end up sparse among thousands of weak entries, and
|
||||
// evictForHardCap must keep finding one.
|
||||
int const cap = 5;
|
||||
std::size_t const partitions = 1;
|
||||
Cache capped(
|
||||
"capped-sparse",
|
||||
1'000'000,
|
||||
3600s,
|
||||
clock,
|
||||
journal,
|
||||
beast::insight::NullCollector::make(),
|
||||
cap,
|
||||
partitions);
|
||||
|
||||
std::vector<std::shared_ptr<Value>> heldRefs;
|
||||
bool everExceeded = false;
|
||||
for (Key k = 1; k <= 2000; ++k)
|
||||
{
|
||||
capped.insert(k, "v");
|
||||
heldRefs.push_back(capped.fetch(k));
|
||||
if (capped.getCacheSize() > cap)
|
||||
everExceeded = true;
|
||||
}
|
||||
EXPECT_FALSE(everExceeded);
|
||||
EXPECT_LE(capped.getCacheSize(), cap);
|
||||
EXPECT_GT(capped.getCacheSize(), 0);
|
||||
}
|
||||
|
||||
TEST(TaggedCacheTest, hard_cap_bounded_work_with_weak_entries)
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
beast::Journal const journal{TestSink::instance()};
|
||||
|
||||
TestStopwatch clock;
|
||||
clock.set(0);
|
||||
|
||||
using Key = LedgerIndex;
|
||||
using Value = std::string;
|
||||
using Cache = TaggedCache<Key, Value>;
|
||||
|
||||
// Every demoted entry stays weakly tracked because `heldRefs` owns it,
|
||||
// so weak entries come to outnumber strong ones 100 to 1. Eviction work
|
||||
// per insert must stay constant rather than grow with the weak count.
|
||||
int const cap = 100;
|
||||
Key const inserts = 10'000;
|
||||
Cache capped(
|
||||
"capped-weak-heavy",
|
||||
1'000'000,
|
||||
3600s,
|
||||
clock,
|
||||
journal,
|
||||
beast::insight::NullCollector::make(),
|
||||
cap);
|
||||
|
||||
std::vector<std::shared_ptr<Value>> heldRefs;
|
||||
bool everExceeded = false;
|
||||
for (Key k = 1; k <= inserts; ++k)
|
||||
{
|
||||
capped.insert(k, "v");
|
||||
heldRefs.push_back(capped.fetch(k));
|
||||
if (capped.getCacheSize() > cap)
|
||||
everExceeded = true;
|
||||
}
|
||||
EXPECT_FALSE(everExceeded);
|
||||
EXPECT_EQ(capped.getCacheSize(), cap);
|
||||
EXPECT_EQ(capped.getTrackSize(), inserts);
|
||||
EXPECT_LE(capped.getEvictVisits(), 4u * inserts);
|
||||
}
|
||||
|
||||
TEST(TaggedCacheTest, hard_cap_enforced_via_fetch_handler)
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
beast::Journal const journal{TestSink::instance()};
|
||||
|
||||
TestStopwatch clock;
|
||||
clock.set(0);
|
||||
|
||||
using Key = LedgerIndex;
|
||||
using Value = std::string;
|
||||
using Cache = TaggedCache<Key, Value>;
|
||||
|
||||
// fetch(key, handler) is the miss path a NodeStore- or database-backed
|
||||
// lookup takes; it must count and cap an inserted entry the same as the
|
||||
// canonicalize path uses, not insert one uncounted and uncapped.
|
||||
int const cap = 100;
|
||||
Cache capped(
|
||||
"capped-handler",
|
||||
1'000'000,
|
||||
3600s,
|
||||
clock,
|
||||
journal,
|
||||
beast::insight::NullCollector::make(),
|
||||
cap);
|
||||
|
||||
bool everExceeded = false;
|
||||
for (Key k = 1; k <= 1000; ++k)
|
||||
{
|
||||
auto const ptr = capped.fetch(k, [] { return std::make_shared<Value>("v"); });
|
||||
EXPECT_NE(ptr, nullptr);
|
||||
if (capped.getCacheSize() > cap)
|
||||
everExceeded = true;
|
||||
}
|
||||
EXPECT_FALSE(everExceeded);
|
||||
EXPECT_LE(capped.getCacheSize(), cap);
|
||||
EXPECT_GT(capped.getCacheSize(), 0);
|
||||
}
|
||||
|
||||
TEST(TaggedCacheTest, hard_cap_disabled)
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
beast::Journal const journal{TestSink::instance()};
|
||||
|
||||
TestStopwatch clock;
|
||||
clock.set(0);
|
||||
|
||||
using Key = LedgerIndex;
|
||||
using Value = std::string;
|
||||
using Cache = TaggedCache<Key, Value>;
|
||||
|
||||
// cacheHardCap = 0: growth is bounded only by the periodic sweep.
|
||||
Cache uncapped(
|
||||
"uncapped", 1'000'000, 3600s, clock, journal, beast::insight::NullCollector::make(), 0);
|
||||
|
||||
for (Key k = 1; k <= 1000; ++k)
|
||||
uncapped.insert(k, "v");
|
||||
EXPECT_EQ(uncapped.getCacheSize(), 1000);
|
||||
}
|
||||
|
||||
// A key the cache can hash, compare and partition on but not default
|
||||
// construct. The requirements TaggedCache places on key_type are those
|
||||
// three; the eviction path must not add a fourth.
|
||||
struct ExplicitKey
|
||||
{
|
||||
std::size_t value;
|
||||
|
||||
explicit ExplicitKey(std::size_t v) : value(v)
|
||||
{
|
||||
}
|
||||
|
||||
operator std::size_t() const
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
bool
|
||||
operator==(ExplicitKey const&) const = default;
|
||||
};
|
||||
static_assert(!std::is_default_constructible_v<ExplicitKey>);
|
||||
|
||||
struct ExplicitKeyHash
|
||||
{
|
||||
std::size_t
|
||||
operator()(ExplicitKey const& key) const
|
||||
{
|
||||
return key.value;
|
||||
}
|
||||
};
|
||||
|
||||
TEST(TaggedCacheTest, hard_cap_with_non_default_constructible_key)
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
beast::Journal const journal{TestSink::instance()};
|
||||
|
||||
TestStopwatch clock;
|
||||
clock.set(0);
|
||||
|
||||
using Value = std::string;
|
||||
using Cache = TaggedCache<
|
||||
ExplicitKey,
|
||||
Value,
|
||||
/*IsKeyCache*/ false,
|
||||
SharedWeakCachePointer<Value>,
|
||||
std::shared_ptr<Value>,
|
||||
ExplicitKeyHash>;
|
||||
|
||||
int const cap = 100;
|
||||
Cache capped(
|
||||
"capped-explicit-key",
|
||||
1'000'000,
|
||||
3600s,
|
||||
clock,
|
||||
journal,
|
||||
beast::insight::NullCollector::make(),
|
||||
cap);
|
||||
|
||||
bool everExceeded = false;
|
||||
for (std::size_t k = 1; k <= 1000; ++k)
|
||||
{
|
||||
capped.insert(ExplicitKey{k}, "v");
|
||||
if (capped.getCacheSize() > cap)
|
||||
everExceeded = true;
|
||||
}
|
||||
EXPECT_FALSE(everExceeded);
|
||||
EXPECT_LE(capped.getCacheSize(), cap);
|
||||
EXPECT_GT(capped.getCacheSize(), 0);
|
||||
}
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -294,9 +294,8 @@ public:
|
||||
|
||||
auto const cores = std::thread::hardware_concurrency();
|
||||
|
||||
// Use a single thread when running on under-provisioned systems
|
||||
// or if we are configured to use minimal resources.
|
||||
if ((cores == 1) || ((config.nodeSize == 0) && (cores == 2)))
|
||||
// Use a single thread on under-provisioned systems.
|
||||
if (cores <= 2)
|
||||
return 1;
|
||||
|
||||
// Otherwise, prefer six threads.
|
||||
@@ -339,14 +338,12 @@ public:
|
||||
|
||||
auto count = static_cast<int>(std::thread::hardware_concurrency());
|
||||
|
||||
// Be more aggressive about the number of threads to use
|
||||
// for the job queue if the server is configured as
|
||||
// "large" or "huge" if there are enough cores.
|
||||
if (config->nodeSize >= 4 && count >= 16)
|
||||
// Scale the job queue with the available cores.
|
||||
if (count >= 16)
|
||||
{
|
||||
count = 6 + std::min(count, 8);
|
||||
}
|
||||
else if (config->nodeSize >= 3 && count >= 8)
|
||||
else if (count >= 8)
|
||||
{
|
||||
count = 4 + std::min(count, 6);
|
||||
}
|
||||
@@ -872,7 +869,7 @@ public:
|
||||
node_store::DummyScheduler dummyScheduler;
|
||||
std::unique_ptr<node_store::Database> source =
|
||||
node_store::Manager::instance().makeDatabase(
|
||||
megabytes(config_->getValueFor(SizedItem::BurstSize, std::nullopt)),
|
||||
megabytes(config_->getValueFor(SizedItem::BurstSize)),
|
||||
dummyScheduler,
|
||||
0,
|
||||
config_->section(Sections::kImportNodeDatabase),
|
||||
@@ -1485,6 +1482,7 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline)
|
||||
.ledgerMaster = getLedgerMaster(),
|
||||
.consumer = c,
|
||||
.role = Role::ADMIN,
|
||||
.coro = {},
|
||||
.infoSub = {},
|
||||
.apiVersion = rpc::kApiMaximumSupportedVersion},
|
||||
jvCommand};
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
#include <xrpl/beast/utility/instrumentation.h>
|
||||
#include <xrpl/config/BasicConfig.h>
|
||||
#include <xrpl/config/Constants.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/protocol/ErrorCodes.h>
|
||||
@@ -148,19 +147,13 @@ GRPCServerImpl::CallData<Request, Response>::process()
|
||||
// ensures that finished is always true when this CallData object
|
||||
// is returned as a tag in handleRpcs(), after sending the response
|
||||
finished_ = true;
|
||||
auto runner = app_.getJobQueue().postCoroTask(
|
||||
JobType::JtRpc,
|
||||
"gRPC-Client",
|
||||
// Safe capture: postCoroTask heap-allocates the lambda (FuncStore)
|
||||
// and thisShared keeps the CallData alive until the coroutine ends.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
[thisShared](auto) -> CoroTask<void> {
|
||||
thisShared->processRequest();
|
||||
co_return;
|
||||
auto coro = app_.getJobQueue().postCoro(
|
||||
JobType::JtRpc, "gRPC-Client", [thisShared](std::shared_ptr<JobQueue::Coro> coro) {
|
||||
thisShared->process(coro);
|
||||
});
|
||||
|
||||
// If runner is null, then the JobQueue has already been shutdown
|
||||
if (!runner)
|
||||
// If coro is null, then the JobQueue has already been shutdown
|
||||
if (!coro)
|
||||
{
|
||||
grpc::Status const status{grpc::StatusCode::INTERNAL, "Job Queue is already stopped"};
|
||||
responder_.FinishWithError(status, this);
|
||||
@@ -169,7 +162,7 @@ GRPCServerImpl::CallData<Request, Response>::process()
|
||||
|
||||
template <class Request, class Response>
|
||||
void
|
||||
GRPCServerImpl::CallData<Request, Response>::processRequest()
|
||||
GRPCServerImpl::CallData<Request, Response>::process(std::shared_ptr<JobQueue::Coro> coro)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -211,6 +204,7 @@ GRPCServerImpl::CallData<Request, Response>::processRequest()
|
||||
app_.getLedgerMaster(),
|
||||
usage,
|
||||
role,
|
||||
coro,
|
||||
InfoSub::pointer(),
|
||||
kApiVersion},
|
||||
request_};
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
#include <xrpld/rpc/detail/Handler.h>
|
||||
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
|
||||
#include <xrpl/resource/Charge.h>
|
||||
#include <xrpl/resource/Consumer.h>
|
||||
@@ -227,12 +228,9 @@ private:
|
||||
clone() override;
|
||||
|
||||
private:
|
||||
/**
|
||||
* Process the gRPC request. Called inside the CoroTask lambda
|
||||
* posted to the JobQueue by process().
|
||||
*/
|
||||
// process the request. Called inside the coroutine passed to JobQueue
|
||||
void
|
||||
processRequest();
|
||||
process(std::shared_ptr<JobQueue::Coro> coro);
|
||||
|
||||
// return load type of this RPC
|
||||
resource::Charge
|
||||
|
||||
@@ -2882,27 +2882,13 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters)
|
||||
|
||||
if (admin)
|
||||
{
|
||||
// Note: By default the node size is "tiny". When parsing it's an error if the final
|
||||
// NODE_SIZE is over 4 so below code should be safe.
|
||||
// NOLINTNEXTLINE(bugprone-switch-missing-default-case)
|
||||
switch (registry_.get().getApp().config().nodeSize)
|
||||
{
|
||||
case 0:
|
||||
info[jss::node_size] = "tiny";
|
||||
break;
|
||||
case 1:
|
||||
info[jss::node_size] = "small";
|
||||
break;
|
||||
case 2:
|
||||
info[jss::node_size] = "medium";
|
||||
break;
|
||||
case 3:
|
||||
info[jss::node_size] = "large";
|
||||
break;
|
||||
case 4:
|
||||
info[jss::node_size] = "huge";
|
||||
break;
|
||||
}
|
||||
// The cache memory budget in GB, rounded up: a nonzero sub-GiB
|
||||
// budget (a cgroup limit under 1 GiB, kept nonzero so enforcement
|
||||
// stays on) would otherwise truncate to 0, the same value reported
|
||||
// when enforcement is disabled.
|
||||
constexpr std::uint64_t oneGiB = std::uint64_t{1} << 30;
|
||||
auto const budget = registry_.get().getApp().config().cacheMemoryBudget();
|
||||
info[jss::memory_limit] = static_cast<json::UInt>((budget + oneGiB - 1) >> 30);
|
||||
|
||||
auto when = registry_.get().getValidators().expires();
|
||||
|
||||
|
||||
@@ -122,7 +122,9 @@ SHAMapStoreImp::SHAMapStoreImp(
|
||||
Keys::kCacheMb, std::to_string(config.getValueFor(SizedItem::HashNodeDbCache)));
|
||||
}
|
||||
|
||||
if (!section.exists(Keys::kFilterBits) && (config.nodeSize >= 2))
|
||||
// 32 GB is the budget the deprecated medium node_size tier maps to,
|
||||
// preserving the old medium-and-up gate for the bloom filter.
|
||||
if (!section.exists(Keys::kFilterBits) && config.cacheMemoryBudget() >= (32ull << 30))
|
||||
section.set(Keys::kFilterBits, "10");
|
||||
}
|
||||
|
||||
@@ -190,20 +192,13 @@ SHAMapStoreImp::makeNodeStore(int readThreads)
|
||||
{
|
||||
auto nscfg = app_.config().section(Sections::kNodeDatabase);
|
||||
|
||||
// Provide default values.
|
||||
// Documented defaults: 16384 records, 5 minutes (DatabaseNodeImp reads
|
||||
// cache_age in minutes).
|
||||
if (!nscfg.exists(Keys::kCacheSize))
|
||||
{
|
||||
nscfg.set(
|
||||
Keys::kCacheSize,
|
||||
std::to_string(app_.config().getValueFor(SizedItem::TreeCacheSize, std::nullopt)));
|
||||
}
|
||||
nscfg.set(Keys::kCacheSize, "16384");
|
||||
|
||||
if (!nscfg.exists(Keys::kCacheAge))
|
||||
{
|
||||
nscfg.set(
|
||||
Keys::kCacheAge,
|
||||
std::to_string(app_.config().getValueFor(SizedItem::TreeCacheAge, std::nullopt)));
|
||||
}
|
||||
nscfg.set(Keys::kCacheAge, "5");
|
||||
|
||||
std::unique_ptr<node_store::Database> db;
|
||||
|
||||
@@ -235,7 +230,7 @@ SHAMapStoreImp::makeNodeStore(int readThreads)
|
||||
else
|
||||
{
|
||||
db = node_store::Manager::instance().makeDatabase(
|
||||
megabytes(app_.config().getValueFor(SizedItem::BurstSize, std::nullopt)),
|
||||
megabytes(app_.config().getValueFor(SizedItem::BurstSize)),
|
||||
scheduler_,
|
||||
readThreads,
|
||||
nscfg,
|
||||
@@ -619,7 +614,7 @@ SHAMapStoreImp::makeBackendRotating(std::string path)
|
||||
|
||||
auto backend{node_store::Manager::instance().makeBackend(
|
||||
section,
|
||||
megabytes(app_.config().getValueFor(SizedItem::BurstSize, std::nullopt)),
|
||||
megabytes(app_.config().getValueFor(SizedItem::BurstSize)),
|
||||
scheduler_,
|
||||
app_.getJournal(kNodeStoreName))};
|
||||
backend->open();
|
||||
|
||||
+42
-22
@@ -38,9 +38,7 @@ enum class SizedItem : std::size_t {
|
||||
HashNodeDbCache,
|
||||
TxnDbCache,
|
||||
LgrDbCache,
|
||||
OpenFinalLimit,
|
||||
BurstSize,
|
||||
RamSizeGb,
|
||||
AccountIdCacheSize,
|
||||
};
|
||||
|
||||
@@ -138,7 +136,11 @@ private:
|
||||
*/
|
||||
bool signingEnabled_ = false;
|
||||
|
||||
// The amount of RAM, in bytes, that we detected on this system.
|
||||
// The amount of RAM, in bytes, that we detected on this system. Kept at
|
||||
// byte granularity (not rounded to GiB) so a cgroup limit under 1 GiB
|
||||
// still reads as a nonzero budget instead of collapsing into the
|
||||
// detection-failed case below.
|
||||
// 0 when detection failed.
|
||||
std::uint64_t const ramSize_;
|
||||
|
||||
public:
|
||||
@@ -209,10 +211,10 @@ public:
|
||||
std::uint32_t ledgerHistory = 256;
|
||||
std::uint32_t fetchDepth = 1000000000;
|
||||
|
||||
// Tunable that adjusts various parameters, typically associated
|
||||
// with hardware parameters (RAM size and CPU cores). The default
|
||||
// is 'tiny'.
|
||||
std::size_t nodeSize = 0;
|
||||
// Cache memory budget in bytes, from [memory_limit] (gigabytes). Unset
|
||||
// defaults to detected physical RAM; 0 disables enforcement. The
|
||||
// deprecated [node_size] tiers map onto this budget.
|
||||
std::optional<std::uint64_t> memoryLimit;
|
||||
|
||||
bool sslVerify = true;
|
||||
std::string sslVerifyFile;
|
||||
@@ -250,6 +252,11 @@ public:
|
||||
// size, but we allow admins to explicitly set it in the config.
|
||||
std::optional<int> sweepInterval;
|
||||
|
||||
// Optional overrides for the fixed cache policy values.
|
||||
std::optional<int> treeCacheAge; // [tree_cache_age], seconds
|
||||
std::optional<int> ledgerCacheAge; // [ledger_cache_age], seconds
|
||||
std::optional<int> ledgerFetchSize; // [ledger_fetch_size], ledgers per fetch pass
|
||||
|
||||
// Reduce-relay - Experimental parameters to control p2p routing algorithms
|
||||
|
||||
// Enable base squelching of duplicate validation/proposal messages
|
||||
@@ -375,25 +382,22 @@ public:
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve the default value for the item at the specified node size
|
||||
*
|
||||
* @param item The item for which the default value is needed
|
||||
* @param node Optional value, used to adjust the result to match the
|
||||
* size of a node (0: tiny, ..., 4: huge). If unseated,
|
||||
* uses the configured size (NODE_SIZE).
|
||||
*
|
||||
* @throws This method can throw std::out_of_range if you ask for values
|
||||
* that it does not recognize or request a non-default node-size.
|
||||
* Retrieve the value for the item, derived from the memory budget.
|
||||
*
|
||||
* @param item The item for which the value is needed
|
||||
* @return The value for the requested item.
|
||||
*
|
||||
* @note The defaults are selected so as to be reasonable, but the node
|
||||
* size is an imprecise metric that combines multiple aspects of
|
||||
* the underlying system; this means that we can't provide optimal
|
||||
* defaults in the code for every case.
|
||||
*/
|
||||
[[nodiscard]] int
|
||||
getValueFor(SizedItem item, std::optional<std::size_t> node = std::nullopt) const;
|
||||
getValueFor(SizedItem item) const;
|
||||
|
||||
/**
|
||||
* The effective cache memory budget in bytes.
|
||||
*
|
||||
* [memory_limit] if set, otherwise detected physical RAM. 0 means
|
||||
* enforcement is disabled (explicit 0, or RAM detection failed).
|
||||
*/
|
||||
[[nodiscard]] std::uint64_t
|
||||
cacheMemoryBudget() const;
|
||||
|
||||
[[nodiscard]] beast::Journal
|
||||
journal() const
|
||||
@@ -408,4 +412,20 @@ setupFeeVote(Section const& section);
|
||||
DatabaseCon::Setup
|
||||
setupDatabaseCon(Config const& c, std::optional<beast::Journal> j = std::nullopt);
|
||||
|
||||
namespace detail {
|
||||
|
||||
/**
|
||||
* A /proc/self/mountinfo path field with its octal escapes decoded.
|
||||
*
|
||||
* The kernel writes space, tab, newline and backslash in the root and
|
||||
* mount point fields as `\040`, `\011`, `\012` and `\134`.
|
||||
*
|
||||
* @param escaped The field as read from the file.
|
||||
* @return The path as the kernel names it.
|
||||
*/
|
||||
[[nodiscard]] std::string
|
||||
decodeMountinfoPath(std::string_view escaped);
|
||||
|
||||
} // namespace detail
|
||||
|
||||
} // namespace xrpl
|
||||
+361
-101
@@ -35,7 +35,6 @@
|
||||
#include <filesystem>
|
||||
#include <format>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
@@ -43,12 +42,41 @@
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <system_error>
|
||||
#include <thread>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace xrpl::detail {
|
||||
|
||||
std::string
|
||||
decodeMountinfoPath(std::string_view escaped)
|
||||
{
|
||||
// A backslash and three octal digits, the first at most 3, encode one byte.
|
||||
auto const octal = [](char c, char max = '7') { return c >= '0' && c <= max; };
|
||||
|
||||
std::string path;
|
||||
path.reserve(escaped.size());
|
||||
for (std::size_t i = 0; i < escaped.size(); ++i)
|
||||
{
|
||||
if (escaped[i] == '\\' && i + 3 < escaped.size() && octal(escaped[i + 1], '3') &&
|
||||
octal(escaped[i + 2]) && octal(escaped[i + 3]))
|
||||
{
|
||||
int const byte = ((escaped[i + 1] - '0') << 6) | ((escaped[i + 2] - '0') << 3) |
|
||||
(escaped[i + 3] - '0');
|
||||
path.push_back(static_cast<char>(byte));
|
||||
i += 3;
|
||||
}
|
||||
else
|
||||
{
|
||||
path.push_back(escaped[i]);
|
||||
}
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
} // namespace xrpl::detail
|
||||
|
||||
#if BOOST_OS_WINDOWS
|
||||
#include <sysinfoapi.h>
|
||||
|
||||
@@ -69,16 +97,204 @@ getMemorySize()
|
||||
|
||||
#if BOOST_OS_LINUX
|
||||
#include <sys/sysinfo.h> // IWYU pragma: keep
|
||||
#include <sys/types.h>
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
#include <fstream>
|
||||
|
||||
namespace xrpl::detail {
|
||||
|
||||
// This process's cgroup path from /proc/self/cgroup: the v2 line is
|
||||
// "0::<path>"; a v1 line is "<id>:<controllers>:<path>". Empty when absent.
|
||||
[[nodiscard]] std::string
|
||||
getOwnCgroupPath(std::string_view controller)
|
||||
{
|
||||
std::ifstream in("/proc/self/cgroup");
|
||||
std::string line;
|
||||
|
||||
while (std::getline(in, line))
|
||||
{
|
||||
auto const first = line.find(':');
|
||||
auto const second = line.find(':', first + 1);
|
||||
if (first == std::string::npos || second == std::string::npos)
|
||||
continue;
|
||||
|
||||
std::string_view const controllers(line.data() + first + 1, second - first - 1);
|
||||
if (controller.empty())
|
||||
{
|
||||
// The v2 entry is exactly "0::<path>".
|
||||
if (first == 1 && line[0] == '0' && controllers.empty())
|
||||
return line.substr(second + 1);
|
||||
}
|
||||
else if (controllers.contains(controller))
|
||||
{
|
||||
return line.substr(second + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
// The value in a cgroup limit file; 0 when absent or unlimited. "max" (v2)
|
||||
// fails the read, and the page-counter maximum (v1) both mean unlimited.
|
||||
[[nodiscard]] std::uint64_t
|
||||
readCgroupLimit(std::string const& path)
|
||||
{
|
||||
std::ifstream in(path);
|
||||
std::uint64_t limit = 0;
|
||||
|
||||
if (in >> limit && limit < (std::uint64_t{1} << 62))
|
||||
return limit;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Whether the cgroup directory contains this process. /proc/self/cgroup
|
||||
// paths are namespace-relative, so a resolved directory can name-collide
|
||||
// with a different cgroup when the cgroup mount shows another view; only
|
||||
// trust a directory this process is actually in.
|
||||
[[nodiscard]] bool
|
||||
cgroupContainsSelf(std::string const& dir)
|
||||
{
|
||||
std::ifstream in(dir + "/cgroup.procs");
|
||||
pid_t const self = ::getpid();
|
||||
pid_t pid = 0;
|
||||
|
||||
while (in >> pid)
|
||||
{
|
||||
if (pid == self)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// The smallest numeric limit in `file` from the leaf cgroup up through its
|
||||
// ancestors (the effective limit is the minimum over the hierarchy); 0 when
|
||||
// none is set or the leaf does not belong to this process.
|
||||
[[nodiscard]] std::uint64_t
|
||||
minCgroupLimit(std::string const& mount, std::string path, char const* file)
|
||||
{
|
||||
if (!cgroupContainsSelf(mount + path))
|
||||
return 0;
|
||||
|
||||
std::uint64_t best = 0;
|
||||
auto const consider = [&best](std::uint64_t limit) {
|
||||
if (limit != 0 && (best == 0 || limit < best))
|
||||
best = limit;
|
||||
};
|
||||
|
||||
while (!path.empty() && path != "/")
|
||||
{
|
||||
consider(readCgroupLimit(mount + path + "/" + file));
|
||||
|
||||
auto const slash = path.find_last_of('/');
|
||||
if (slash == std::string::npos)
|
||||
break;
|
||||
path.resize(slash);
|
||||
}
|
||||
consider(readCgroupLimit(mount + "/" + file));
|
||||
|
||||
return best;
|
||||
}
|
||||
|
||||
// The mount point of the cgroup v2 unified hierarchy (`controller` empty) or
|
||||
// of the cgroup v1 hierarchy whose co-mounted controllers include
|
||||
// `controller`; empty when neither is mounted. Read from /proc/self/mountinfo
|
||||
// rather than a fixed path, since a container runtime or an init system can
|
||||
// mount either hierarchy somewhere other than /sys/fs/cgroup. Field 5, the
|
||||
// mount point, is already expressed relative to this process's own root
|
||||
// (man 5 proc), so no further namespace translation is needed; its octal
|
||||
// escapes are decoded so the path opens as the kernel names it.
|
||||
[[nodiscard]] std::string
|
||||
findCgroupMount(std::string_view controller)
|
||||
{
|
||||
std::ifstream in("/proc/self/mountinfo");
|
||||
std::string line;
|
||||
|
||||
while (std::getline(in, line))
|
||||
{
|
||||
auto const dash = line.find(" - ");
|
||||
if (dash == std::string::npos)
|
||||
continue;
|
||||
|
||||
std::istringstream prefix(line.substr(0, dash));
|
||||
std::string id, parentId, majorMinor, root, mountPoint;
|
||||
prefix >> id >> parentId >> majorMinor >> root >> mountPoint;
|
||||
|
||||
std::istringstream suffix(line.substr(dash + 3));
|
||||
std::string fsType, source, superOptions;
|
||||
suffix >> fsType >> source >> superOptions;
|
||||
|
||||
if (controller.empty())
|
||||
{
|
||||
if (fsType == "cgroup2")
|
||||
return decodeMountinfoPath(mountPoint);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fsType != "cgroup")
|
||||
continue;
|
||||
|
||||
std::stringstream opts(superOptions);
|
||||
std::string opt;
|
||||
while (std::getline(opts, opt, ','))
|
||||
{
|
||||
if (opt == controller)
|
||||
return decodeMountinfoPath(mountPoint);
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
// The cgroup (v2, then v1) memory limit in bytes; 0 when absent or
|
||||
// unlimited. Checks this process's own cgroup and its ancestors (covering
|
||||
// nested limits such as systemd MemoryMax=) before the root-level files
|
||||
// containers expose.
|
||||
[[nodiscard]] std::uint64_t
|
||||
getCgroupMemoryLimit()
|
||||
{
|
||||
if (auto const mount = findCgroupMount(""); !mount.empty())
|
||||
{
|
||||
if (auto const path = getOwnCgroupPath(""); !path.empty() && path != "/")
|
||||
{
|
||||
if (auto const limit = minCgroupLimit(mount, path, "memory.max"))
|
||||
return limit;
|
||||
}
|
||||
|
||||
if (auto const limit = readCgroupLimit(mount + "/memory.max"))
|
||||
return limit;
|
||||
}
|
||||
|
||||
if (auto const mount = findCgroupMount("memory"); !mount.empty())
|
||||
{
|
||||
if (auto const path = getOwnCgroupPath("memory"); !path.empty() && path != "/")
|
||||
{
|
||||
if (auto const limit = minCgroupLimit(mount, path, "memory.limit_in_bytes"))
|
||||
return limit;
|
||||
}
|
||||
|
||||
if (auto const limit = readCgroupLimit(mount + "/memory.limit_in_bytes"))
|
||||
return limit;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
[[nodiscard]] std::uint64_t
|
||||
getMemorySize()
|
||||
{
|
||||
if (struct sysinfo si{}; sysinfo(&si) == 0)
|
||||
return static_cast<std::uint64_t>(si.totalram) * si.mem_unit;
|
||||
std::uint64_t ram = 0;
|
||||
|
||||
return 0;
|
||||
if (struct sysinfo si{}; sysinfo(&si) == 0)
|
||||
ram = static_cast<std::uint64_t>(si.totalram) * si.mem_unit;
|
||||
|
||||
if (auto const limit = getCgroupMemoryLimit(); limit != 0 && (ram == 0 || limit < ram))
|
||||
return limit;
|
||||
|
||||
return ram;
|
||||
}
|
||||
|
||||
} // namespace xrpl::detail
|
||||
@@ -109,50 +325,6 @@ getMemorySize()
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
// clang-format off
|
||||
// The configurable node sizes are "tiny", "small", "medium", "large", "huge"
|
||||
inline constexpr std::array<std::pair<SizedItem, std::array<int, 5>>, 13>
|
||||
kSizedItems
|
||||
{{
|
||||
// FIXME: We should document each of these items, explaining exactly
|
||||
// what they control and whether there exists an explicit
|
||||
// config option that can be used to override the default.
|
||||
|
||||
// tiny small medium large huge
|
||||
{SizedItem::SweepInterval, {{ 10, 30, 60, 90, 120 }}},
|
||||
{SizedItem::TreeCacheSize, {{ 262144, 524288, 2097152, 4194304, 8388608 }}},
|
||||
{SizedItem::TreeCacheAge, {{ 30, 60, 90, 120, 900 }}},
|
||||
{SizedItem::LedgerSize, {{ 32, 32, 64, 256, 384 }}},
|
||||
{SizedItem::LedgerAge, {{ 30, 60, 180, 300, 600 }}},
|
||||
{SizedItem::LedgerFetch, {{ 2, 3, 4, 5, 8 }}},
|
||||
{SizedItem::HashNodeDbCache, {{ 4, 12, 24, 64, 128 }}},
|
||||
{SizedItem::TxnDbCache, {{ 4, 12, 24, 64, 128 }}},
|
||||
{SizedItem::LgrDbCache, {{ 4, 8, 16, 32, 128 }}},
|
||||
{SizedItem::OpenFinalLimit, {{ 8, 16, 32, 64, 128 }}},
|
||||
{SizedItem::BurstSize, {{ 4, 8, 16, 32, 48 }}},
|
||||
{SizedItem::RamSizeGb, {{ 6, 8, 12, 24, 0 }}},
|
||||
{SizedItem::AccountIdCacheSize, {{ 20047, 50053, 77081, 150061, 300007 }}}
|
||||
}};
|
||||
// clang-format on
|
||||
|
||||
// Ensure that the order of entries in the table corresponds to the
|
||||
// order of entries in the enum:
|
||||
static_assert(
|
||||
[]() constexpr -> bool {
|
||||
std::underlying_type_t<SizedItem> idx = 0;
|
||||
|
||||
for (auto const& i : kSizedItems)
|
||||
{
|
||||
if (static_cast<std::underlying_type_t<SizedItem>>(i.first) != idx)
|
||||
return false;
|
||||
|
||||
++idx;
|
||||
}
|
||||
|
||||
return true;
|
||||
}(),
|
||||
"Mismatch between sized item enum & array indices");
|
||||
|
||||
//
|
||||
// TODO: Check permissions on config file before using it.
|
||||
//
|
||||
@@ -257,44 +429,16 @@ getEnvVar(char const* name)
|
||||
return value;
|
||||
}
|
||||
|
||||
Config::Config()
|
||||
: j_(beast::Journal::getNullSink()), ramSize_(detail::getMemorySize() / (1024 * 1024 * 1024))
|
||||
Config::Config() : j_(beast::Journal::getNullSink()), ramSize_(detail::getMemorySize())
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
Config::setupControl(bool bQuiet, bool bSilent, bool bStandalone)
|
||||
{
|
||||
XRPL_ASSERT(nodeSize == 0, "xrpl::Config::setupControl : node size not set");
|
||||
|
||||
quiet_ = bQuiet || bSilent;
|
||||
silent_ = bSilent;
|
||||
runStandalone_ = bStandalone;
|
||||
|
||||
// We try to autodetect the appropriate node size by checking available
|
||||
// RAM and CPU resources. We default to "tiny" for standalone mode.
|
||||
if (!bStandalone)
|
||||
{
|
||||
// First, check against 'minimum' RAM requirements per node size:
|
||||
auto const& threshold =
|
||||
kSizedItems[std::underlying_type_t<SizedItem>(SizedItem::RamSizeGb)];
|
||||
|
||||
auto ns = std::ranges::find_if(threshold.second, [this](std::size_t limit) {
|
||||
return (limit == 0) || (ramSize_ < limit);
|
||||
});
|
||||
|
||||
XRPL_ASSERT(ns != threshold.second.end(), "xrpl::Config::setupControl : valid node size");
|
||||
|
||||
if (ns != threshold.second.end())
|
||||
nodeSize = std::distance(threshold.second.begin(), ns);
|
||||
|
||||
// Adjust the size based on the number of hardware threads of
|
||||
// execution available to us:
|
||||
if (auto const hc = std::thread::hardware_concurrency(); hc != 0)
|
||||
nodeSize = std::min<std::size_t>(hc / 2, nodeSize);
|
||||
}
|
||||
|
||||
XRPL_ASSERT(nodeSize <= 4, "xrpl::Config::setupControl : node size is set");
|
||||
}
|
||||
|
||||
void
|
||||
@@ -577,31 +721,65 @@ Config::loadFromString(std::string const& fileContents)
|
||||
}
|
||||
}
|
||||
|
||||
if (getSingleSection(secConfig, Sections::kMemoryLimit, strTemp, j_))
|
||||
{
|
||||
// Gigabytes; 0 disables enforcement.
|
||||
auto const gb = beast::lexicalCastThrow<std::uint64_t>(strTemp);
|
||||
if (gb > 1024)
|
||||
{
|
||||
Throw<std::runtime_error>(
|
||||
"Invalid value '" + strTemp + "' for key '" + Sections::kMemoryLimit +
|
||||
"'; the limit is in gigabytes and may not exceed 1024");
|
||||
}
|
||||
memoryLimit = gb << 30;
|
||||
}
|
||||
|
||||
if (getSingleSection(secConfig, Sections::kNodeSize, strTemp, j_))
|
||||
{
|
||||
if (boost::iequals(strTemp, "tiny"))
|
||||
// Deprecated: each tier (by name or its legacy 0-4 index) is an
|
||||
// alias for a memory budget. [memory_limit], when present, wins.
|
||||
static constexpr std::array<std::pair<std::string_view, std::uint64_t>, 5> kTiers{
|
||||
{{"tiny", 4}, {"small", 8}, {"medium", 32}, {"large", 64}, {"huge", 128}}};
|
||||
|
||||
auto const tier = std::ranges::find_if(
|
||||
kTiers, [&strTemp](auto const& t) { return boost::iequals(strTemp, t.first); });
|
||||
|
||||
std::uint64_t const budgetGb = tier != kTiers.end()
|
||||
? tier->second
|
||||
: kTiers[std::min<std::size_t>(4, beast::lexicalCastThrow<std::size_t>(strTemp))]
|
||||
.second;
|
||||
|
||||
if (!memoryLimit)
|
||||
memoryLimit = budgetGb << 30;
|
||||
|
||||
if (!quiet_)
|
||||
{
|
||||
nodeSize = 0;
|
||||
std::cerr << "WARNING: [node_size] is deprecated and will be removed "
|
||||
"in a future release. Set [memory_limit] instead; thread "
|
||||
"counts derive from the core count and [workers] / "
|
||||
"[io_workers].\n";
|
||||
}
|
||||
else if (boost::iequals(strTemp, "small"))
|
||||
}
|
||||
|
||||
// A budget beyond the detected memory (physical RAM, or the cgroup limit
|
||||
// when one is set) cannot be honored and recreates the oversized-preset
|
||||
// OOM this setting exists to prevent.
|
||||
if (memoryLimit && ramSize_ != 0 && *memoryLimit > ramSize_ && !quiet_)
|
||||
{
|
||||
std::cerr << "WARNING: the configured memory budget (" << (*memoryLimit >> 30)
|
||||
<< " GB) exceeds detected memory (" << ramSize_
|
||||
<< " bytes, RAM or cgroup limit); ";
|
||||
// A whole-GB floor of 0 here is a nonzero byte budget rounded down,
|
||||
// not the "disabled" value: recommending it would turn enforcement
|
||||
// off, the opposite of this warning's point.
|
||||
if (auto const detectedGb = ramSize_ >> 30)
|
||||
{
|
||||
nodeSize = 1;
|
||||
}
|
||||
else if (boost::iequals(strTemp, "medium"))
|
||||
{
|
||||
nodeSize = 2;
|
||||
}
|
||||
else if (boost::iequals(strTemp, "large"))
|
||||
{
|
||||
nodeSize = 3;
|
||||
}
|
||||
else if (boost::iequals(strTemp, "huge"))
|
||||
{
|
||||
nodeSize = 4;
|
||||
std::cerr << "set [memory_limit] to " << detectedGb << " or less.\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
nodeSize = std::min<std::size_t>(4, beast::lexicalCastThrow<std::size_t>(strTemp));
|
||||
std::cerr << "[memory_limit] is in whole gigabytes and cannot represent it; "
|
||||
"remove the setting to use the detected budget automatically.\n";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -744,6 +922,42 @@ Config::loadFromString(std::string const& fileContents)
|
||||
}
|
||||
}
|
||||
|
||||
if (getSingleSection(secConfig, Sections::kTreeCacheAge, strTemp, j_))
|
||||
{
|
||||
treeCacheAge = beast::lexicalCastThrow<int>(strTemp);
|
||||
|
||||
if (*treeCacheAge < 10 || *treeCacheAge > 3600)
|
||||
{
|
||||
Throw<std::runtime_error>(
|
||||
std::string("Invalid ") + Sections::kTreeCacheAge +
|
||||
": must be between 10 and 3600 inclusive");
|
||||
}
|
||||
}
|
||||
|
||||
if (getSingleSection(secConfig, Sections::kLedgerCacheAge, strTemp, j_))
|
||||
{
|
||||
ledgerCacheAge = beast::lexicalCastThrow<int>(strTemp);
|
||||
|
||||
if (*ledgerCacheAge < 10 || *ledgerCacheAge > 3600)
|
||||
{
|
||||
Throw<std::runtime_error>(
|
||||
std::string("Invalid ") + Sections::kLedgerCacheAge +
|
||||
": must be between 10 and 3600 inclusive");
|
||||
}
|
||||
}
|
||||
|
||||
if (getSingleSection(secConfig, Sections::kLedgerFetchSize, strTemp, j_))
|
||||
{
|
||||
ledgerFetchSize = beast::lexicalCastThrow<int>(strTemp);
|
||||
|
||||
if (*ledgerFetchSize < 1 || *ledgerFetchSize > 16)
|
||||
{
|
||||
Throw<std::runtime_error>(
|
||||
std::string("Invalid ") + Sections::kLedgerFetchSize +
|
||||
": must be between 1 and 16 inclusive");
|
||||
}
|
||||
}
|
||||
|
||||
if (getSingleSection(secConfig, Sections::kWorkers, strTemp, j_))
|
||||
{
|
||||
workers = beast::lexicalCastThrow<int>(strTemp);
|
||||
@@ -1224,12 +1438,58 @@ Config::getDebugLogFile() const
|
||||
}
|
||||
|
||||
int
|
||||
Config::getValueFor(SizedItem item, std::optional<std::size_t> node) const
|
||||
Config::getValueFor(SizedItem item) const
|
||||
{
|
||||
auto const index = static_cast<std::underlying_type_t<SizedItem>>(item);
|
||||
XRPL_ASSERT(index < kSizedItems.size(), "xrpl::Config::getValueFor : valid index input");
|
||||
XRPL_ASSERT(!node || *node <= 4, "xrpl::Config::getValueFor : unset or valid node");
|
||||
return kSizedItems.at(index).second.at(node.value_or(nodeSize));
|
||||
// Memory-shaped items scale linearly with the budget between a floor and
|
||||
// a ceiling; time and policy items are fixed. A budget of 0 (enforcement
|
||||
// disabled) yields the floors. The 1024 bound keeps gb * 65536 within
|
||||
// int range (the config parser enforces it too).
|
||||
auto const gb = static_cast<int>(std::min<std::uint64_t>(cacheMemoryBudget() >> 30, 1024));
|
||||
|
||||
switch (item)
|
||||
{
|
||||
case SizedItem::SweepInterval:
|
||||
return 30;
|
||||
case SizedItem::TreeCacheSize:
|
||||
// Half the budget at an estimated 8 KiB per entry (the node plus
|
||||
// its weak-tracking entry, hash buckets, and control block):
|
||||
// 1 GiB / 2 / 8 KiB = 65536 entries per budget GB.
|
||||
return std::max(16384, gb * 65536);
|
||||
case SizedItem::TreeCacheAge:
|
||||
return treeCacheAge.value_or(300);
|
||||
case SizedItem::LedgerSize:
|
||||
return std::clamp(gb * 6, 32, 384);
|
||||
case SizedItem::LedgerAge:
|
||||
return ledgerCacheAge.value_or(180);
|
||||
case SizedItem::LedgerFetch:
|
||||
return ledgerFetchSize.value_or(4);
|
||||
case SizedItem::HashNodeDbCache:
|
||||
case SizedItem::TxnDbCache:
|
||||
case SizedItem::LgrDbCache:
|
||||
// HashNodeDbCache is consumed in MB (RocksDB cache_mb); the two
|
||||
// SQLite page caches are consumed in KB.
|
||||
return std::clamp(gb * 2, 4, 128);
|
||||
case SizedItem::BurstSize:
|
||||
return std::clamp(gb, 4, 48);
|
||||
case SizedItem::AccountIdCacheSize:
|
||||
// Fixed regardless of budget: ~22 MB at 72 bytes per slot, and
|
||||
// the value stays prime for hash distribution.
|
||||
return 300007;
|
||||
}
|
||||
|
||||
UNREACHABLE("xrpl::Config::getValueFor : invalid item");
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::uint64_t
|
||||
Config::cacheMemoryBudget() const
|
||||
{
|
||||
if (memoryLimit)
|
||||
return *memoryLimit;
|
||||
|
||||
// ramSize_ is in bytes; 0 when detection failed, which disables
|
||||
// enforcement.
|
||||
return ramSize_;
|
||||
}
|
||||
|
||||
FeeSetup
|
||||
|
||||
@@ -3,11 +3,13 @@
|
||||
#include <xrpld/rpc/Role.h>
|
||||
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/json/json_value.h>
|
||||
#include <xrpl/resource/Charge.h>
|
||||
#include <xrpl/resource/Consumer.h>
|
||||
#include <xrpl/server/InfoSub.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string_view>
|
||||
|
||||
namespace xrpl {
|
||||
@@ -30,6 +32,7 @@ struct Context
|
||||
LedgerMaster& ledgerMaster;
|
||||
resource::Consumer& consumer;
|
||||
Role role;
|
||||
std::shared_ptr<JobQueue::Coro> coro;
|
||||
InfoSub::pointer infoSub;
|
||||
unsigned int apiVersion;
|
||||
};
|
||||
|
||||
@@ -184,10 +184,13 @@ public:
|
||||
|
||||
private:
|
||||
json::Value
|
||||
processSession(std::shared_ptr<WSSession> const& session, json::Value const& jv);
|
||||
processSession(
|
||||
std::shared_ptr<WSSession> const& session,
|
||||
std::shared_ptr<JobQueue::Coro> const& coro,
|
||||
json::Value const& jv);
|
||||
|
||||
void
|
||||
processSession(std::shared_ptr<Session> const&);
|
||||
processSession(std::shared_ptr<Session> const&, std::shared_ptr<JobQueue::Coro> coro);
|
||||
|
||||
void
|
||||
processRequest(
|
||||
@@ -195,6 +198,7 @@ private:
|
||||
std::string const& request,
|
||||
beast::ip::Endpoint const& remoteIPAddress,
|
||||
Output const&,
|
||||
std::shared_ptr<JobQueue::Coro> coro,
|
||||
std::string_view forwardedFor,
|
||||
std::string_view user);
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
#include <xrpld/rpc/detail/PathRequest.h>
|
||||
|
||||
#include <xrpl/basics/Log.h>
|
||||
#include <xrpl/basics/scope.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/json/json_value.h>
|
||||
@@ -138,12 +137,9 @@ PathRequestManager::updateAll(std::shared_ptr<ReadView const> const& inLedger)
|
||||
}
|
||||
else if (request->hasCompletion())
|
||||
{
|
||||
// One-shot request with completion function. Fire the
|
||||
// completion even if doUpdate throws, so the RPC
|
||||
// handler blocked in doRipplePathFind is released
|
||||
// immediately instead of waiting out its timeout.
|
||||
ScopeExit const complete{[&request] { request->updateComplete(); }};
|
||||
// One-shot request with completion function
|
||||
request->doUpdate(cache, false);
|
||||
request->updateComplete();
|
||||
++processed;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
#include <xrpl/beast/rfc2616.h>
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
#include <xrpl/config/Constants.h>
|
||||
#include <xrpl/core/CoroTask.h>
|
||||
#include <xrpl/core/Job.h>
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/json/Output.h>
|
||||
@@ -318,23 +317,9 @@ ServerHandler::onRequest(Session& session)
|
||||
}
|
||||
|
||||
std::shared_ptr<Session> const detachedSession = session.detach();
|
||||
auto const postResult = jobQueue_.postCoroTask(
|
||||
JtClientRpc,
|
||||
"RPC-Client",
|
||||
// Safe capture: postCoroTask heap-allocates the lambda (FuncStore),
|
||||
// detachedSession is captured by value, and this (the ServerHandler)
|
||||
// outlives the JobQueue jobs.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
[this, detachedSession](auto) -> CoroTask<void> {
|
||||
try
|
||||
{
|
||||
processSession(detachedSession);
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
JLOG(journal_.error()) << "RPC-Client coroutine exception: " << e.what();
|
||||
}
|
||||
co_return;
|
||||
auto const postResult = jobQueue_.postCoro(
|
||||
JtClientRpc, "RPC-Client", [this, detachedSession](std::shared_ptr<JobQueue::Coro> coro) {
|
||||
processSession(detachedSession, coro);
|
||||
});
|
||||
if (postResult == nullptr)
|
||||
{
|
||||
@@ -370,30 +355,17 @@ ServerHandler::onWSMessage(
|
||||
|
||||
JLOG(journal_.trace()) << "Websocket received '" << jv << "'";
|
||||
|
||||
auto const postResult = jobQueue_.postCoroTask(
|
||||
auto const postResult = jobQueue_.postCoro(
|
||||
JtClientWebsocket,
|
||||
"WS-Client",
|
||||
// Safe capture: postCoroTask heap-allocates the lambda (FuncStore),
|
||||
// session and jv are captured by value, and this (the ServerHandler)
|
||||
// outlives the JobQueue jobs.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines)
|
||||
[this, session, jv = std::move(jv)](auto) -> CoroTask<void> {
|
||||
try
|
||||
{
|
||||
auto const jr = this->processSession(session, jv);
|
||||
auto const s = to_string(jr);
|
||||
auto const n = s.length();
|
||||
boost::beast::multi_buffer sb(n);
|
||||
sb.commit(
|
||||
boost::asio::buffer_copy(sb.prepare(n), boost::asio::buffer(s.c_str(), n)));
|
||||
session->send(std::make_shared<StreambufWSMsg<decltype(sb)>>(std::move(sb)));
|
||||
session->complete();
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
{
|
||||
JLOG(journal_.error()) << "WS-Client coroutine exception: " << e.what();
|
||||
}
|
||||
co_return;
|
||||
[this, session, jv = std::move(jv)](std::shared_ptr<JobQueue::Coro> const& coro) {
|
||||
auto const jr = this->processSession(session, coro, jv);
|
||||
auto const s = to_string(jr);
|
||||
auto const n = s.length();
|
||||
boost::beast::multi_buffer sb(n);
|
||||
sb.commit(boost::asio::buffer_copy(sb.prepare(n), boost::asio::buffer(s.c_str(), n)));
|
||||
session->send(std::make_shared<StreambufWSMsg<decltype(sb)>>(std::move(sb)));
|
||||
session->complete();
|
||||
});
|
||||
if (postResult == nullptr)
|
||||
{
|
||||
@@ -438,7 +410,10 @@ logDuration(json::Value const& request, T const& duration, beast::Journal& journ
|
||||
}
|
||||
|
||||
json::Value
|
||||
ServerHandler::processSession(std::shared_ptr<WSSession> const& session, json::Value const& jv)
|
||||
ServerHandler::processSession(
|
||||
std::shared_ptr<WSSession> const& session,
|
||||
std::shared_ptr<JobQueue::Coro> const& coro,
|
||||
json::Value const& jv)
|
||||
{
|
||||
auto is = std::static_pointer_cast<WSInfoSub>(session->appDefined);
|
||||
if (is->getConsumer().disconnect(journal_))
|
||||
@@ -505,6 +480,7 @@ ServerHandler::processSession(std::shared_ptr<WSSession> const& session, json::V
|
||||
.ledgerMaster = app_.getLedgerMaster(),
|
||||
.consumer = is->getConsumer(),
|
||||
.role = role,
|
||||
.coro = coro,
|
||||
.infoSub = is,
|
||||
.apiVersion = apiVersion},
|
||||
jv,
|
||||
@@ -573,14 +549,18 @@ ServerHandler::processSession(std::shared_ptr<WSSession> const& session, json::V
|
||||
return jr;
|
||||
}
|
||||
|
||||
// Run as a coroutine.
|
||||
void
|
||||
ServerHandler::processSession(std::shared_ptr<Session> const& session)
|
||||
ServerHandler::processSession(
|
||||
std::shared_ptr<Session> const& session,
|
||||
std::shared_ptr<JobQueue::Coro> coro)
|
||||
{
|
||||
processRequest(
|
||||
session->port(),
|
||||
buffersToString(session->request().body().data()),
|
||||
session->remoteAddress().atPort(0),
|
||||
makeOutput(*session),
|
||||
coro,
|
||||
forwardedFor(session->request()),
|
||||
[&] -> std::string_view {
|
||||
auto const iter = session->request().find("X-User");
|
||||
@@ -621,6 +601,7 @@ ServerHandler::processRequest(
|
||||
std::string const& request,
|
||||
beast::ip::Endpoint const& remoteIPAddress,
|
||||
Output const& output,
|
||||
std::shared_ptr<JobQueue::Coro> coro,
|
||||
std::string_view forwardedFor,
|
||||
std::string_view user)
|
||||
{
|
||||
@@ -880,6 +861,7 @@ ServerHandler::processRequest(
|
||||
.ledgerMaster = app_.getLedgerMaster(),
|
||||
.consumer = usage,
|
||||
.role = role,
|
||||
.coro = coro,
|
||||
.infoSub = InfoSub::pointer(),
|
||||
.apiVersion = apiVersion},
|
||||
params,
|
||||
|
||||
@@ -65,12 +65,6 @@ static constexpr int kDefaultAutoFillFeeMultiplier = 10;
|
||||
static constexpr int kDefaultAutoFillFeeDivisor = 1;
|
||||
static constexpr int kMaxPathfindsInProgress = 2;
|
||||
static constexpr int kMaxPathfindJobCount = 50;
|
||||
|
||||
/**
|
||||
* How long a blocking ripple_path_find call waits for the path-finding
|
||||
* continuation before giving up and returning an internal error.
|
||||
*/
|
||||
constexpr auto kPathfindCompletionTimeout = std::chrono::seconds{30};
|
||||
static constexpr int kMaxJobQueueClients = 500;
|
||||
constexpr auto kMaxValidatedLedgerAge = std::chrono::minutes{2};
|
||||
static constexpr int kMaxRequestSize = 1000000;
|
||||
|
||||
@@ -7,15 +7,14 @@
|
||||
#include <xrpld/rpc/detail/RPCLedgerHelpers.h>
|
||||
#include <xrpld/rpc/detail/Tuning.h>
|
||||
|
||||
#include <xrpl/core/JobQueue.h>
|
||||
#include <xrpl/json/json_value.h>
|
||||
#include <xrpl/protocol/ErrorCodes.h>
|
||||
#include <xrpl/protocol/RPCErr.h>
|
||||
#include <xrpl/protocol/jss.h>
|
||||
#include <xrpl/resource/Fees.h>
|
||||
|
||||
#include <condition_variable>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <utility>
|
||||
|
||||
namespace xrpl {
|
||||
@@ -48,62 +47,98 @@ doRipplePathFind(rpc::JsonContext& context)
|
||||
PathRequest::pointer request;
|
||||
lpLedger = context.ledgerMaster.getClosedLedger();
|
||||
|
||||
// The wait below parks this JobQueue worker thread until the
|
||||
// path-finding continuation fires. The continuation is fired by a
|
||||
// JtUpdatePf job, which itself needs a free worker to run. Bound
|
||||
// the number of concurrently parked workers (LegacyPathFind admits
|
||||
// at most kMaxPathfindsInProgress non-admin requests) so that
|
||||
// concurrent ripple_path_find calls cannot occupy every worker and
|
||||
// stall the whole JobQueue. The guard must stay in scope until the
|
||||
// wait completes. (The old Boost.Coroutine implementation did not
|
||||
// need this: it suspended and released the worker instead of
|
||||
// blocking it.)
|
||||
rpc::LegacyPathFind const lpf(isUnlimited(context.role), context.app);
|
||||
if (!lpf.isOk())
|
||||
return rpcError(RpcTooBusy);
|
||||
|
||||
// makeLegacyPathRequest enqueues a path-finding job that runs
|
||||
// asynchronously. We block this thread with a condition_variable
|
||||
// until the path-finding continuation signals completion.
|
||||
// If makeLegacyPathRequest cannot schedule the job (e.g. during
|
||||
// shutdown), it returns an empty request and we skip the wait.
|
||||
// Replaces the old Coro yield/resume pattern with synchronous
|
||||
// blocking, eliminating shutdown race conditions.
|
||||
// It doesn't look like there's much odd happening here, but you should
|
||||
// be aware this code runs in a JobQueue::Coro, which is a coroutine.
|
||||
// And we may be flipping around between threads. Here's an overview:
|
||||
//
|
||||
// The state is shared with the continuation so that it stays alive
|
||||
// even if we stop waiting before the continuation runs.
|
||||
struct PathDone
|
||||
{
|
||||
std::mutex mtx;
|
||||
std::condition_variable cv;
|
||||
bool done = false;
|
||||
};
|
||||
auto const state = std::make_shared<PathDone>();
|
||||
|
||||
// 1. We're running doRipplePathFind() due to a call to
|
||||
// ripple_path_find. doRipplePathFind() is currently running
|
||||
// inside of a JobQueue::Coro using a JobQueue thread.
|
||||
//
|
||||
// 2. doRipplePathFind's call to makeLegacyPathRequest() enqueues the
|
||||
// path-finding request. That request will (probably) run at some
|
||||
// indeterminate future time on a (probably different) JobQueue
|
||||
// thread.
|
||||
//
|
||||
// 3. As a continuation from that path-finding JobQueue thread, the
|
||||
// coroutine we're currently running in (!) is posted to the
|
||||
// JobQueue. Because it is a continuation, that post won't
|
||||
// happen until the path-finding request completes.
|
||||
//
|
||||
// 4. Once the continuation is enqueued, and we have reason to think
|
||||
// the path-finding job is likely to run, then the coroutine we're
|
||||
// running in yield()s. That means it surrenders its thread in
|
||||
// the JobQueue. The coroutine is suspended, but ready to run,
|
||||
// because it is kept resident by a shared_ptr in the
|
||||
// path-finding continuation.
|
||||
//
|
||||
// 5. If all goes well then path-finding runs on a JobQueue thread
|
||||
// and executes its continuation. The continuation posts this
|
||||
// same coroutine (!) to the JobQueue.
|
||||
//
|
||||
// 6. When the JobQueue calls this coroutine, this coroutine resumes
|
||||
// from the line below the coro->yield() and returns the
|
||||
// path-finding result.
|
||||
//
|
||||
// With so many moving parts, what could go wrong?
|
||||
//
|
||||
// Just in terms of the JobQueue refusing to add jobs at shutdown
|
||||
// there are two specific things that can go wrong.
|
||||
//
|
||||
// 1. The path-finding Job queued by makeLegacyPathRequest() might be
|
||||
// rejected (because we're shutting down).
|
||||
//
|
||||
// Fortunately this problem can be addressed by looking at the
|
||||
// return value of makeLegacyPathRequest(). If
|
||||
// makeLegacyPathRequest() cannot get a thread to run the path-find
|
||||
// on, then it returns an empty request.
|
||||
//
|
||||
// 2. The path-finding job might run, but the Coro::post() might be
|
||||
// rejected by the JobQueue (because we're shutting down).
|
||||
//
|
||||
// We handle this case by resuming (not posting) the Coro.
|
||||
// By resuming the Coro, we allow the Coro to run to completion
|
||||
// on the current thread instead of requiring that it run on a
|
||||
// new thread from the JobQueue.
|
||||
//
|
||||
// Both of these failure modes are hard to recreate in a unit test
|
||||
// because they are so dependent on inter-thread timing. However
|
||||
// the failure modes can be observed by synchronously (inside the
|
||||
// xrpld source code) shutting down the application. The code to
|
||||
// do so looks like this:
|
||||
//
|
||||
// context.app.signalStop();
|
||||
// while (! context.app.getJobQueue().jobCounter().joined()) { }
|
||||
//
|
||||
// The first line starts the process of shutting down the app.
|
||||
// The second line waits until no more jobs can be added to the
|
||||
// JobQueue before letting the thread continue.
|
||||
//
|
||||
// May 2017
|
||||
jvResult = context.app.getPathRequestManager().makeLegacyPathRequest(
|
||||
request,
|
||||
[state]() {
|
||||
[&context]() {
|
||||
// Copying the shared_ptr keeps the coroutine alive up
|
||||
// through the return. Otherwise the storage under the
|
||||
// captured reference could evaporate when we return from
|
||||
// coroCopy->resume(). This is not strictly necessary, but
|
||||
// will make maintenance easier.
|
||||
std::shared_ptr<JobQueue::Coro> const coroCopy{context.coro};
|
||||
if (!coroCopy->post())
|
||||
{
|
||||
std::scoped_lock const lk(state->mtx);
|
||||
state->done = true;
|
||||
// The post() failed, so we won't get a thread to let
|
||||
// the Coro finish. We'll call Coro::resume() so the
|
||||
// Coro can finish on our thread. Otherwise the
|
||||
// application will hang on shutdown.
|
||||
coroCopy->resume();
|
||||
}
|
||||
state->cv.notify_one();
|
||||
},
|
||||
context.consumer,
|
||||
lpLedger,
|
||||
context.params);
|
||||
if (request)
|
||||
{
|
||||
std::unique_lock lk(state->mtx);
|
||||
if (!state->cv.wait_for(
|
||||
lk, rpc::tuning::kPathfindCompletionTimeout, [&state] { return state->done; }))
|
||||
{
|
||||
// Path-finding continuation never fired (e.g. shutdown
|
||||
// race or unexpected failure). Return an internal error
|
||||
// rather than blocking the RPC thread indefinitely.
|
||||
return rpcError(RpcInternal);
|
||||
}
|
||||
lk.unlock();
|
||||
context.coro->yield();
|
||||
jvResult = request->doStatus(context.params);
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include <xrpl/basics/Log.h>
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/basics/chrono.h>
|
||||
#include <xrpl/beast/insight/NullCollector.h>
|
||||
#include <xrpl/shamap/FullBelowCache.h>
|
||||
#include <xrpl/shamap/TreeNodeCache.h>
|
||||
|
||||
@@ -39,8 +40,19 @@ NodeFamily::NodeFamily(Application& app, CollectorManager& cm)
|
||||
app.config().getValueFor(SizedItem::TreeCacheSize),
|
||||
std::chrono::seconds(app.config().getValueFor(SizedItem::TreeCacheAge)),
|
||||
stopwatch(),
|
||||
j_))
|
||||
j_,
|
||||
beast::insight::NullCollector::make(),
|
||||
// Hard cap: the clamped target, enforced on insert; 0 = off.
|
||||
app.config().cacheMemoryBudget() != 0
|
||||
? app.config().getValueFor(SizedItem::TreeCacheSize)
|
||||
: 0))
|
||||
{
|
||||
// Bytes, not whole GB: a cgroup limit under 1 GiB is a nonzero budget
|
||||
// with the cap on, and must not read as the disabled value.
|
||||
auto const budget = app.config().cacheMemoryBudget();
|
||||
JLOG(j_.info()) << "TreeNodeCache sizing: target="
|
||||
<< app.config().getValueFor(SizedItem::TreeCacheSize) << " entries, budget "
|
||||
<< budget << " bytes" << (budget == 0 ? " (enforcement disabled)" : "");
|
||||
}
|
||||
|
||||
void
|
||||
|
||||
Reference in new issue
Block a user