Compare commits

...

76 Commits

Author SHA1 Message Date
Ed Hennis
81e06a9ba8 Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop:
  fix: Update PermissionedDEX invariant domain tracking for valid offer replacement (7387)
  feat: Check default fields are not default when serializing (6267)
  test: Migrate csf and xrpld-consensus Beast non-JTx tests to GTest (7046)
  test: Modularize Peerfinder component and migrate Peerfinder tests from Beast to GTest and GMock (7054)
  test: Add an RAII class to manage the env.parseFailureExpected flag (7669)
  build: Use custom libc in a devshell by default (7852)
  feat: Add fixCleanup3_4_0 amendment (no functionality yet) (7854)
  ci: Add llvm-tools-preview to rust toolchain (7853)
  ci: Update CI image (7850)
  ci: Update XRPLF/actions (7849)
  chore: Clean up grammar in PR template (7846)
  build: Create versioned compiler/tooling symlinks in nix environments (7844)
  ci: Use rust-overlay to bring Rust into nix (7837)
2026-07-23 21:03:45 -04:00
Kassaking7
9afa1cf4d1 fix: Update PermissionedDEX invariant domain tracking for valid offer replacement (#7387)
Co-authored-by: Bart <bthomee@users.noreply.github.com>
2026-07-23 21:40:21 +00:00
Ed Hennis
7908aec2ec feat: Check default fields are not default when serializing (#6267)
Co-authored-by: Vito <5780819+Tapanito@users.noreply.github.com>
Co-authored-by: Ayaz Salikhov <mathbunnyru@users.noreply.github.com>
Co-authored-by: Bart <bthomee@users.noreply.github.com>
2026-07-23 21:39:11 +00:00
Marek Foss
4c0180b3db test: Migrate csf and xrpld-consensus Beast non-JTx tests to GTest (#7046)
Co-authored-by: Alex Kremer <akremer@ripple.com>
2026-07-23 21:38:21 +00:00
Marek Foss
4acccfeda8 test: Modularize Peerfinder component and migrate Peerfinder tests from Beast to GTest and GMock (#7054)
Co-authored-by: Alex Kremer <akremer@ripple.com>
2026-07-23 21:00:06 +00:00
Ed Hennis
b89d75a2d5 test: Add an RAII class to manage the env.parseFailureExpected flag (#7669) 2026-07-23 20:57:00 +00:00
Ayaz Salikhov
40cdf49d15 build: Use custom libc in a devshell by default (#7852) 2026-07-23 19:05:24 +00:00
Mayukha Vadari
38c54c3f36 feat: Add fixCleanup3_4_0 amendment (no functionality yet) (#7854) 2026-07-23 18:50:59 +00:00
Sergey Kuznetsov
95e1ffea6e ci: Add llvm-tools-preview to rust toolchain (#7853) 2026-07-23 16:49:44 +00:00
Sergey Kuznetsov
74cfb3586a ci: Update CI image (#7850) 2026-07-23 11:21:49 +00:00
Ayaz Salikhov
0072ced94c ci: Update XRPLF/actions (#7849) 2026-07-23 10:22:18 +00:00
Mayukha Vadari
12ed506565 chore: Clean up grammar in PR template (#7846) 2026-07-22 18:50:10 +00:00
Ayaz Salikhov
3122de86bf build: Create versioned compiler/tooling symlinks in nix environments (#7844) 2026-07-22 17:04:02 +00:00
Sergey Kuznetsov
7edf39e622 ci: Use rust-overlay to bring Rust into nix (#7837) 2026-07-22 14:00:17 +00:00
Ed Hennis
b3141d91fa Merge branch 'develop' into ximinez/directory 2026-07-21 19:29:00 -04:00
dependabot[bot]
ba01b05f33 ci: [DEPENDABOT] bump actions/setup-python from 6.3.0 to 7.0.0 (#7830)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-07-21 16:04:45 +00:00
Sergey Kuznetsov
c414e1877d ci: Update pre-commit image (#7838) 2026-07-21 15:37:47 +00:00
Sergey Kuznetsov
f56b79f00d ci: Add cargo to pre-commit image (#7835) 2026-07-21 13:39:37 +00:00
Ayaz Salikhov
4918f3bb20 chore: Add doxygen in pre-commit image (#7836) 2026-07-21 13:38:26 +00:00
Mayukha Vadari
ee0a3dfad7 ci: Improve test debuggability (#7619)
Co-authored-by: Ayaz Salikhov <mathbunnyru@users.noreply.github.com>
2026-07-21 13:14:40 +00:00
Bart
042c9660cd chore: Add CODEOWNERS for CI-related changes (#7832)
Co-authored-by: Bart <11445373+bthomee@users.noreply.github.com>
2026-07-20 21:06:24 +00:00
Ayaz Salikhov
60b49f3fe0 ci: Use in-house image for pre-commit (#7610) 2026-07-20 19:11:09 +00:00
Ayaz Salikhov
4c869ee16c ci: Build separate pre-commit docker image (#7831) 2026-07-20 17:40:38 +00:00
Ayaz Salikhov
b2f45813e0 chore: Workaround boost compiler resolution inside Nix environment (#7826) 2026-07-20 14:04:08 +00:00
Vito Tumas
5ce0b1c2c7 refactor: Restructure LendingHelpers to improve readability (#7807) 2026-07-16 15:47:56 +00:00
Andrzej Budzanowski
701311f27e test: Add google benchmark dependency and migrate nodestore timing test as a benchmark (#7317)
Co-authored-by: Marek Foss <marek.foss@neti-soft.com>
Co-authored-by: Alex Kremer <akremer@ripple.com>
2026-07-16 15:44:43 +00:00
Sergey Kuznetsov
b1a670c46e ci: Add Rust to CI (#7808)
Co-authored-by: Ayaz Salikhov <mathbunnyru@users.noreply.github.com>
2026-07-16 15:28:04 +00:00
Bart
18e311e1e2 chore: Bump version to 3.3.0-rc1 (#7806)
Co-authored-by: Bart <11445373+bthomee@users.noreply.github.com>
Co-authored-by: Ayaz Salikhov <mathbunnyru@users.noreply.github.com>
2026-07-16 13:54:12 +00:00
Denis Angell
69b70d7a0d fix: Refactor Batch Transaction IDs (#7736)
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Mayukha Vadari <mvadari@ripple.com>
2026-07-16 13:15:59 +00:00
Mayukha Vadari
b42cde3e85 refactor: Remove redundant enable checks in ConfidentialMPT txs (#7809) 2026-07-16 09:51:19 +00:00
Peter Chen
cd38c0e800 chore: Update mpt-crypto-lib to 0.4.0-rc4 (#7813) 2026-07-15 23:47:22 +00:00
Vlad
433e5f6896 fix: Reject zero CheckID in CheckCancel and CheckCash (#7685) 2026-07-15 22:08:45 +00:00
Ayaz Salikhov
781ab723af ci: Fix workflow launch on matrix-unrelated labels (#7812) 2026-07-15 18:24:31 +00:00
Ed Hennis
a24e543af3 fix: Allocate TaggedCache::getKeys() memory outside of lock (#7567)
Co-authored-by: xrplf-ai-reviewer[bot] <266832837+xrplf-ai-reviewer[bot]@users.noreply.github.com>
2026-07-15 13:30:20 +00:00
Ed Hennis
1b1cd4c260 Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop: (21 commits)
  fix: Re-store nodes missing from both backends during online_delete rotation (7763)
  fix: Add amendment sponsor for AccountRootsDeletedClean (7801)
  fix: Update base_uint and test changes released in 3.1.3 (7570)
  fix: Handle rounding just above kMaxRep more accurately (7389)
  fix: Document and assert "after" is never null in invariants (7354)
  ci: Run full matrix only on `Ready to merge` or `Full CI build` labeled PRs (7689)
  fix: Strengthen Clawback invariant checks for MPT balances (7285)
  test: Add unit tests for IP address related functions (7744)
  ci: Add Rust to Nix docker image (7571)
  docs: Add more information about pre-commit hooks and how to set them up (7802)
  test: Add JSON array size tests (7592)
  chore: Enable most readability checks (7772)
  ci: Do not run conflict checker when label is applied (7774)
  chore: Run clang_tidy_check with `pass_filenames: false` from pre-commit (7800)
  feat: Add delegate filter param for account_tx RPC (6126)
  refactor: Move `jss.h` `include` out of `Indexes.h` (7799)
  test: Add tests for check doxygen style (7795)
  style: Add pre-commit hook to check doxygen style (7794)
  style: Unify style for all Doxygen comments (7776)
  fix: Improve Number addition/subtraction rounding (7369)
  ...
2026-07-14 20:37:19 -04:00
Sophia Xie
a0fd1cce54 fix: Re-store nodes missing from both backends during online_delete rotation (#7763)
Co-authored-by: Valentin Balaschenko <13349202+vlntb@users.noreply.github.com>
2026-07-14 23:42:40 +00:00
Kassaking7
cda63d00a2 fix: Add amendment sponsor for AccountRootsDeletedClean (#7801) 2026-07-14 19:41:53 +00:00
Ed Hennis
a482d6aced Merge branch 'develop' into ximinez/directory 2026-07-09 19:50:00 -04:00
Ed Hennis
299627298f Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop:
  chore: Enable most cppcoreguidelines checks (7660)
  test: Migrate basics Beast tests to GTest (7136)
  test: Migrate resource, shamap Beast tests to GTest (7133)
  chore: Enable most misc checks (7663)
  chore: Fix unity build (7730)
  chore: Enable most modernize checks (7664)
  chore: Delete dead code (7718)
  chore: Enable modernize-use-constraints (7715)
  chore: Enable modernize-avoid-bind (7711)
  chore: Enable modernize-use-auto (7707)
  build: Add protobuf dependencies to Nix (7706)
  feat: Enable ConfidentialTransfer and BatchV1_1 (7698)
  chore: Enable modernize-unary-static-assert (7705)
  chore: Make clang-tidy happy on macOS (7701)
  chore: Improve pre-commit hooks (7702)
  refactor: Retire DisallowIncomingV1 fix (7364)
  build: Add verify-headers target to cleanup headers (7670)
2026-07-07 18:17:50 -04:00
Ed Hennis
640798696b Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop:
  refactor: Retire InnerObjTemplate fix (7368)
  fix: Disable AMM creation with Vault shares (7666)
  test: Add tests for TMProofPathResponse and TMReplayDeltaResponse invalid hash/key sizes (7593)
  ci: [DEPENDABOT] bump actions/setup-python from 6.2.0 to 6.3.0 (7657)
  build: Don't reuse binaries between different C++ versions (7681)
  chore: Update pre-commit hooks && actions (7686)
  feat: Add an invariant to ensure object deletion also deletes its pseudo-account (7445)
  feat: Add Batch (XLS-56) V1_1 (6446)
  feat: Introduce lending 1.1 amendment and add `MemoData` field to `VaultDelete` transaction (6324)
2026-07-01 16:10:38 -04:00
Ed Hennis
47512afad5 Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop: (69 commits)
  chore: Use std::ranges where possible (7634)
  ci: Use macOS 26 Tahoe with apple-clang 21 (7601)
  build: Mark sec256k1 and mpt-crypto as transitive headers (7658)
  chore: Add a script to nicely format clang-tidy output (7650)
  chore: Enable most bugprone checks (7643)
  feat: Confidential Transfer for MPT (5860)
  fix: Use trustline balance direction to validate IOU PaymentMint/PaymentBurn (7584)
  fix: Unify freeze checks for pseudo-account deposit/withdraw (7382)
  fix: Block delegate tx from being queued (7640)
  chore: Enable groups of clang-tidy checks by default (7637)
  ci: Better determine when we need to run full clang-tidy (7635)
  refactor: Retire NFTokenReserve fix (7367)
  refactor: Retire Clawback amendment (7353)
  refactor: Rename (mostly keylet) functions to more closely match the docs (7059)
  build: Switch to a new conan XRPLF remote, again (7638)
  chore: Revert "build: Switch to a new conan XRPLF remote (7622)" (7623)
  build: Switch to a new conan XRPLF remote (7622)
  build: Align xrpld RPM packaging with DEB package (7529)
  chore: Use clang-tidy v22 new features (7427)
  build: Patch nix binaries in CMake (7539)
  ...
2026-06-30 17:16:29 -04:00
Ed Hennis
ff3708a757 Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop: (48 commits)
  test: Add null check unit test for `Oracle::aggregatePrice` (7306)
  ci: Patch conan recipe for Nix to be able to use on macOS (7532)
  ci: Run sanitizers on release builds too (7527)
  fix: Correct hybrid offer deletion on credential expiry (6843)
  ci: Make sanitizer flags lists in the profile, not a string (7449)
  ci: Make configurations launch on certain event types (7447)
  fix: Add [[maybe_unused]] to fix320Enabled for assert=OFF builds (7446)
  ci: Add `gh` and `file` to nix packages (7444)
  fix: Disable transaction invariants (7409)
  perf: Dispatch "hasInvalidAmount()" on type tag instead of dynamic_cast (7402)
  refactor: Retire fixUniversalNumber amendment (5962)
  test: Do not create data directory for memory databases (7323)
  ci: Launch upload-conan-deps on profile change (7442)
  fix: Fix Number comparison operator (7406)
  feat: Use C++ 23 standard (7431)
  refactor: Introduce XRPL_ASSERT_IF for amendment-gated assertions (7378)
  refactor: Change config section and key string literals into constants (7095)
  refactor: Use `std::move` and `std::string_view` where possible (7424)
  refactor: Use const function arguments where possible (7423)
  ci: Use XRPLF/actions build-multiarch-image workflow (7428)
  ...
2026-06-12 11:50:43 -04:00
Ed Hennis
391a1e442c Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop: (41 commits)
  release: Bump version to 3.2.0-rc2 (7348)
  refactor: Enable support for `fixCleanup3_2_0` amendment (7347)
  release: Bump version to 3.2.0-rc1 (7335)
  fix: Fix a rounding error at the `Number::maxRep` cusp (7051)
  ci: Only push docker images in XRPLF/rippled (7330)
  ci: [DEPENDABOT] bump docker/setup-buildx-action from 4.0.0 to 4.1.0 (7322)
  ci: [DEPENDABOT] bump codecov/codecov-action from 6.0.0 to 6.0.1 (7321)
  ci: [DEPENDABOT] bump docker/build-push-action from 7.1.0 to 7.2.0 (7320)
  ci: [DEPENDABOT] bump docker/metadata-action from 6.0.0 to 6.1.0 (7319)
  ci: [DEPENDABOT] bump docker/login-action from 4.1.0 to 4.2.0 (7318)
  fix: Update `clang-tidy` to include `src/tests` directory header check (7307)
  chore: Pin Python packages for codegen using uv (7329)
  style: Use shfmt instead of bashate (7326)
  fix: Fix edge-case where vault-depositor may get stuck (7139)
  fix: Fix `VaultInvariant` and `VaultDeposit` precision bugs at IOU scale boundaries (7272)
  ci: Add clang to nix images (7308)
  fix: Include management-fee delta in doOverpayment assertion (7039)
  fix: Fix clang-tidy pre-commit hook to locate compile_commands.json from repo root (7325)
  fix: Use consistent scale for `debtTotal` (7093)
  fix: Skip deleted book directories and non-root modifications in `ValidBookDirectory` invariant (7312)
  ...
2026-05-27 15:33:45 -04:00
Ed Hennis
9e77212900 Merge branch 'develop' into ximinez/directory 2026-05-19 16:53:55 -04:00
Ed Hennis
93f5a0e217 Merge branch 'develop' into ximinez/directory 2026-05-19 10:15:35 -04:00
Ed Hennis
71367f361c Merge branch 'develop' into ximinez/directory 2026-05-19 05:16:12 -04:00
Ed Hennis
931d21b2a7 Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop:
  refactor: Clean up comments post-clang-tidy changes (7283)
  release: Set version to 3.3.0-b0 (7280)
  refactor: Rename static constants (7120)
  refactor: Use `isFlag` where possible instead of bitwise math (7278)
  ci: Update XRPLF/actions (7281)
2026-05-16 09:49:20 -04:00
Ed Hennis
c99feb82e7 Merge branch 'develop' into ximinez/directory 2026-05-14 20:39:26 -04:00
Ed Hennis
7b53a5e0c5 Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop:
  chore: Consolidate fix amendments (7134)
  ci: Add Conan retry (7147)
  fix: Backport Permissioned Domains fixes (7016)
  refactor: Move unhex lookup table out of function (7104)
  refactor: Improve Forwarded header field parsing (7126)
  fix: Check network ID in `transactionSignFor` (7102)
  feat: Implement nix-based Dockerfile for CI (7083)
2026-05-14 10:46:04 -04:00
Ed Hennis
4a02518497 Merge branch 'develop' into ximinez/directory 2026-05-13 12:03:59 -04:00
Ed Hennis
0b6c3630cc Merge branch 'develop' into ximinez/directory 2026-05-12 19:19:05 -04:00
Ed Hennis
4d04ba5be5 Merge branch 'develop' into ximinez/directory 2026-05-12 16:26:28 -04:00
Ed Hennis
ed53557d41 Merge branch 'develop' into ximinez/directory 2026-05-11 13:34:31 -04:00
Ed Hennis
e4d10393f3 Merge branch 'develop' into ximinez/directory 2026-05-07 18:10:21 -04:00
Ed Hennis
fcc7f57e82 Merge branch 'develop' into ximinez/directory 2026-05-07 14:18:48 -04:00
Ed Hennis
a25229f154 Merge branch 'develop' into ximinez/directory 2026-05-07 13:28:50 -04:00
Ed Hennis
eb6eaf2532 Merge branch 'develop' into ximinez/directory 2026-05-06 22:34:41 -04:00
Ed Hennis
ff987fc7c6 Merge branch 'develop' into ximinez/directory 2026-05-06 14:18:20 -04:00
Ed Hennis
d21137c4c1 Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop:
  fix: Fix regressions in `server_definitions` (7008)
  chore: Do not duplicate sanitizer flags (7058)
  ci: Run pre-commit on diff in clang-tidy workflow (7078)
  ci: Use XRPLF/create-issue (7076)
  ci: Rewrite clang-tidy workflow(s) in a reusable manner (7062)
  chore: Ignore identifier-naming update in git blame (7066)
  refactor: Enable clang-tidy `readability-identifier-naming` check (6571)
2026-05-05 16:45:19 -04:00
Ed Hennis
ac7db2e621 Remove orphaned code that was introduced by a bad merge or rebase 2026-05-05 15:48:00 -04:00
Ed Hennis
42005a8080 Merge branch 'develop' into ximinez/directory 2026-05-01 13:59:22 -04:00
Ed Hennis
1f7b1b3a78 Merge remote-tracking branch 'XRPLF/develop' into ximinez/directory
* XRPLF/develop:
  feat: Create new transaction testing framework `TxTest` (6537)
  feat: Add cleanup amendment for 3.2.0 (7037)
  fix: Fix ubsan flagged issues (6151)
2026-04-28 15:28:15 -05:00
Ed Hennis
80b90544c5 Merge branch 'develop' into ximinez/directory 2026-04-25 14:46:02 -04:00
Ed Hennis
00b9a8cd67 Merge branch 'develop' into ximinez/directory 2026-04-23 15:56:20 -04:00
Ed Hennis
3be49f814a Merge branch 'develop' into ximinez/directory 2026-04-22 23:40:54 -04:00
Ed Hennis
1674fabe81 Merge branch 'develop' into ximinez/directory 2026-04-22 14:49:21 -04:00
Ed Hennis
6dfa47ce7a Merge branch 'develop' into ximinez/directory 2026-04-22 13:10:52 -04:00
Ed Hennis
bef095be65 Merge branch 'develop' into ximinez/directory 2026-04-21 18:58:08 -04:00
Ed Hennis
8e5d774c36 Merge branch 'develop' into ximinez/directory 2026-04-20 17:49:55 -04:00
Ed Hennis
fb8fb30f6c Merge branch 'develop' into ximinez/directory 2026-04-20 15:45:12 -04:00
Ed Hennis
a553001125 Merge branch 'develop' into ximinez/directory 2026-04-20 11:39:16 -04:00
Ed Hennis
57782e84ee Merge branch 'develop' into ximinez/directory 2026-04-17 18:14:35 -04:00
Ed Hennis
9d5076c8a9 Merge branch 'develop' into ximinez/directory 2026-04-16 13:44:45 -04:00
Ed Hennis
1af379e09f Merge branch 'develop' into ximinez/directory 2026-04-15 19:06:37 -04:00
Ed Hennis
1ced0875ae Merge branch 'develop' into ximinez/directory 2026-04-15 14:29:04 -04:00
Ed Hennis
53e6d7580a rabbit hole: refactor dirAdd to find gaps in "full" directories.
- This would potentially be very expensive to implement, so don't.
- However, it might be a good start for a ledger fix option.
2026-04-13 19:50:28 -04:00
203 changed files with 9319 additions and 7846 deletions

View File

@@ -30,7 +30,9 @@ ignoreRegExpList:
- ABCDEFGHIJKLMNOPQRSTUVWXYZ
- ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz
overrides:
- filename: "**/*_test.cpp" # all test files
- filename:
- "**/*_test.cpp" # legacy boost.test files
- "src/tests/**/*.cpp" # gtest test files
ignoreRegExpList:
- /"[^"]*"/g # double-quoted strings
- /'[^']*'/g # single-quoted strings
@@ -216,6 +218,7 @@ words:
- Nyffenegger
- onlatest
- ostr
- oxalica
- pargs
- partitioner
- paychan
@@ -262,6 +265,8 @@ words:
- Rohrs
- roundings
- rustc
- rustfmt
- rustup
- sahyadri
- Satoshi
- scons
@@ -305,6 +310,7 @@ words:
- takerpays
- ters
- TMEndpointv2
- toolchain
- tparam
- trixie
- tx
@@ -337,6 +343,7 @@ words:
- unsquelch
- unsquelched
- unsquelching
- unsuffixed
- unvalidated
- unveto
- unvetoed

View File

@@ -11,6 +11,9 @@ endfunction()
function(create_symbolic_link target link)
endfunction()
function(xrpl_add_benchmark name)
endfunction()
macro(exclude_from_default target_)
endmacro()

View File

@@ -1,10 +1,10 @@
<!--
This PR template helps you to write a good pull request description.
This PR template helps you write a good pull request description.
Please feel free to include additional useful information even beyond what is requested below.
If your branch is on a personal fork and has a name that allows it to
run CI build/test jobs (e.g. "ci/foo"), remember to rename it BEFORE
opening the PR. This avoids unnecessary redundant test runs. Renaming
opening the PR. This avoids redundant test runs. Renaming
the branch after opening the PR will close the PR.
https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-branches-in-your-repository/renaming-a-branch
-->
@@ -15,7 +15,7 @@ https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-
Please include a summary of the changes.
This may be a direct input to the release notes.
If too broad, please consider splitting into multiple PRs.
If a relevant task or issue, please link it here.
If there is a relevant task or issue, please link it here.
-->
### Context of Change
@@ -65,5 +65,5 @@ This section may not be needed if your change includes thoroughly commented unit
<!--
## Future Tasks
For future tasks related to PR.
For future tasks related to this PR.
-->

View File

@@ -40,18 +40,18 @@ listed later.
| 04 | xrpl/protocol |
| 05 | xrpl/core xrpl/resource xrpl/server |
| 06 | xrpl/ledger xrpl/nodestore xrpl/net |
| 07 | xrpl/shamap |
| 07 | xrpl/shamap xrpl/consensus |
## xrpld Modules (Application Implementation)
| Level / Tier | Module(s) |
| ------------ | -------------------------------- |
| 05 | xrpld/conditions xrpld/consensus |
| 06 | xrpld/core xrpld/peerfinder |
| 07 | xrpld/shamap xrpld/overlay |
| 08 | xrpld/app |
| 09 | xrpld/rpc |
| 10 | xrpld/perflog |
| Level / Tier | Module(s) |
| ------------ | --------------------------- |
| 05 | xrpld/conditions |
| 06 | xrpld/core xrpld/peerfinder |
| 07 | xrpld/shamap xrpld/overlay |
| 08 | xrpld/app |
| 09 | xrpld/rpc |
| 10 | xrpld/perflog |
## Test Modules

View File

@@ -1,9 +1,6 @@
Loop: xrpld.app xrpld.overlay
xrpld.app > xrpld.overlay
Loop: xrpld.app xrpld.peerfinder
xrpld.peerfinder ~= xrpld.app
Loop: xrpld.app xrpld.rpc
xrpld.rpc > xrpld.app

View File

@@ -1,8 +1,13 @@
benchmarks.libxrpl > xrpl.basics
benchmarks.libxrpl > xrpl.config
benchmarks.libxrpl > xrpl.nodestore
libxrpl.basics > xrpl.basics
libxrpl.conditions > xrpl.basics
libxrpl.conditions > xrpl.conditions
libxrpl.config > xrpl.basics
libxrpl.config > xrpl.config
libxrpl.consensus > xrpl.basics
libxrpl.consensus > xrpl.consensus
libxrpl.core > xrpl.basics
libxrpl.core > xrpl.core
libxrpl.core > xrpl.json
@@ -22,6 +27,9 @@ libxrpl.nodestore > xrpl.config
libxrpl.nodestore > xrpl.json
libxrpl.nodestore > xrpl.nodestore
libxrpl.nodestore > xrpl.protocol
libxrpl.peerfinder > xrpl.basics
libxrpl.peerfinder > xrpl.peerfinder
libxrpl.peerfinder > xrpl.protocol
libxrpl.protocol > xrpl.basics
libxrpl.protocol > xrpl.json
libxrpl.protocol > xrpl.protocol
@@ -57,9 +65,9 @@ test.app > test.jtx
test.app > test.unit_test
test.app > xrpl.basics
test.app > xrpl.config
test.app > xrpl.consensus
test.app > xrpl.core
test.app > xrpld.app
test.app > xrpld.consensus
test.app > xrpld.core
test.app > xrpld.overlay
test.app > xrpld.rpc
@@ -80,12 +88,9 @@ test.basics > xrpl.protocol
test.beast > xrpl.basics
test.conditions > xrpl.basics
test.conditions > xrpl.conditions
test.consensus > test.csf
test.consensus > test.jtx
test.consensus > test.unit_test
test.consensus > xrpl.basics
test.consensus > xrpld.app
test.consensus > xrpld.consensus
test.consensus > xrpl.ledger
test.consensus > xrpl.protocol
test.consensus > xrpl.shamap
@@ -100,10 +105,6 @@ test.core > xrpl.json
test.core > xrpl.protocol
test.core > xrpl.rdb
test.core > xrpl.server
test.csf > xrpl.basics
test.csf > xrpld.consensus
test.csf > xrpl.json
test.csf > xrpl.ledger
test.json > test.jtx
test.json > xrpl.json
test.jtx > test.unit_test
@@ -143,19 +144,13 @@ test.overlay > xrpl.config
test.overlay > xrpld.app
test.overlay > xrpld.core
test.overlay > xrpld.overlay
test.overlay > xrpld.peerfinder
test.overlay > xrpl.json
test.overlay > xrpl.nodestore
test.overlay > xrpl.peerfinder
test.overlay > xrpl.protocol
test.overlay > xrpl.resource
test.overlay > xrpl.server
test.overlay > xrpl.shamap
test.peerfinder > test.beast
test.peerfinder > test.unit_test
test.peerfinder > xrpl.basics
test.peerfinder > xrpld.core
test.peerfinder > xrpld.peerfinder
test.peerfinder > xrpl.protocol
test.protocol > test.jtx
test.protocol > test.unit_test
test.protocol > xrpl.basics
@@ -189,11 +184,13 @@ test.unit_test > xrpl.basics
test.unit_test > xrpl.protocol
tests.libxrpl > xrpl.basics
tests.libxrpl > xrpl.config
tests.libxrpl > xrpl.consensus
tests.libxrpl > xrpl.core
tests.libxrpl > xrpl.json
tests.libxrpl > xrpl.ledger
tests.libxrpl > xrpl.net
tests.libxrpl > xrpl.nodestore
tests.libxrpl > xrpl.peerfinder
tests.libxrpl > xrpl.protocol
tests.libxrpl > xrpl.protocol_autogen
tests.libxrpl > xrpl.resource
@@ -203,6 +200,10 @@ tests.libxrpl > xrpl.tx
xrpl.conditions > xrpl.basics
xrpl.conditions > xrpl.protocol
xrpl.config > xrpl.basics
xrpl.consensus > xrpl.basics
xrpl.consensus > xrpl.json
xrpl.consensus > xrpl.ledger
xrpl.consensus > xrpl.protocol
xrpl.core > xrpl.basics
xrpl.core > xrpl.json
xrpl.core > xrpl.protocol
@@ -217,6 +218,8 @@ xrpl.nodestore > xrpl.basics
xrpl.nodestore > xrpl.config
xrpl.nodestore > xrpl.json
xrpl.nodestore > xrpl.protocol
xrpl.peerfinder > xrpl.basics
xrpl.peerfinder > xrpl.protocol
xrpl.protocol > xrpl.basics
xrpl.protocol > xrpl.json
xrpl.protocol_autogen > xrpl.json
@@ -243,23 +246,20 @@ xrpl.tx > xrpl.protocol
xrpld.app > test.unit_test
xrpld.app > xrpl.basics
xrpld.app > xrpl.config
xrpld.app > xrpl.consensus
xrpld.app > xrpl.core
xrpld.app > xrpld.consensus
xrpld.app > xrpld.core
xrpld.app > xrpl.json
xrpld.app > xrpl.ledger
xrpld.app > xrpl.net
xrpld.app > xrpl.nodestore
xrpld.app > xrpl.peerfinder
xrpld.app > xrpl.protocol
xrpld.app > xrpl.rdb
xrpld.app > xrpl.resource
xrpld.app > xrpl.server
xrpld.app > xrpl.shamap
xrpld.app > xrpl.tx
xrpld.consensus > xrpl.basics
xrpld.consensus > xrpl.json
xrpld.consensus > xrpl.ledger
xrpld.consensus > xrpl.protocol
xrpld.core > xrpl.basics
xrpld.core > xrpl.config
xrpld.core > xrpl.core
@@ -268,21 +268,22 @@ xrpld.core > xrpl.protocol
xrpld.core > xrpl.rdb
xrpld.overlay > xrpl.basics
xrpld.overlay > xrpl.config
xrpld.overlay > xrpl.consensus
xrpld.overlay > xrpl.core
xrpld.overlay > xrpld.consensus
xrpld.overlay > xrpld.core
xrpld.overlay > xrpld.peerfinder
xrpld.overlay > xrpl.json
xrpld.overlay > xrpl.ledger
xrpld.overlay > xrpl.peerfinder
xrpld.overlay > xrpl.protocol
xrpld.overlay > xrpl.resource
xrpld.overlay > xrpl.server
xrpld.overlay > xrpl.shamap
xrpld.overlay > xrpl.tx
xrpld.peerfinder > xrpl.basics
xrpld.peerfinder > xrpl.config
xrpld.peerfinder > xrpld.app
xrpld.peerfinder > xrpld.core
xrpld.peerfinder > xrpl.protocol
xrpld.peerfinder > xrpl.peerfinder
xrpld.peerfinder > xrpl.rdb
xrpld.perflog > xrpl.basics
xrpld.perflog > xrpl.config

View File

@@ -1,5 +1,5 @@
{
"image_tag": "sha-e29b523",
"image_tag": "sha-3122de8",
"configs": {
"ubuntu": [
{

View File

@@ -1,4 +1,4 @@
name: Build Nix Docker images
name: Build `nix` Docker images
on:
push:
@@ -8,6 +8,7 @@ on:
- ".github/workflows/build-nix-images.yml"
- "flake.nix"
- "flake.lock"
- "rust-toolchain.toml"
- "nix/**"
- "!nix/docker/README.md"
- "!nix/devshell.nix"
@@ -18,6 +19,7 @@ on:
- ".github/workflows/build-nix-images.yml"
- "flake.nix"
- "flake.lock"
- "rust-toolchain.toml"
- "nix/**"
- "!nix/docker/README.md"
- "!nix/devshell.nix"
@@ -36,7 +38,7 @@ defaults:
jobs:
build-merge:
name: Build and push nix-${{ matrix.distro.name }}
name: Build and push `nix-${{ matrix.distro.name }}` image
permissions:
contents: read
packages: write
@@ -54,7 +56,7 @@ jobs:
base_image: debian:bookworm
- name: rhel
base_image: registry.access.redhat.com/ubi9/ubi:latest
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@ee03d31bcc4501d7599dc1b1ecd7a34af582ad1c
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32
with:
image_name: xrpld/nix-${{ matrix.distro.name }}
dockerfile: nix/docker/Dockerfile

View File

@@ -1,4 +1,4 @@
name: Build packaging Docker images
name: Build `packaging` Docker images
on:
push:
@@ -26,7 +26,7 @@ defaults:
jobs:
build-merge:
name: Build and push packaging-${{ matrix.distro.name }}
name: Build and push `packaging-${{ matrix.distro.name }}` image
permissions:
contents: read
packages: write
@@ -38,7 +38,7 @@ jobs:
base_image: debian:bookworm
- name: rhel
base_image: registry.access.redhat.com/ubi9/ubi:latest
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@ee03d31bcc4501d7599dc1b1ecd7a34af582ad1c
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32
with:
image_name: xrpld/packaging-${{ matrix.distro.name }}
dockerfile: package/Dockerfile

View File

@@ -0,0 +1,38 @@
name: Build `pre-commit` Docker image
on:
push:
branches:
- develop
paths:
- ".github/workflows/build-pre-commit-image.yml"
- "bin/pre-commit/Dockerfile"
- "rust-toolchain.toml"
pull_request:
paths:
- ".github/workflows/build-pre-commit-image.yml"
- "bin/pre-commit/Dockerfile"
- "rust-toolchain.toml"
workflow_dispatch:
concurrency:
# Read `on-trigger.yml` for the rationale behind this concurrency group name.
group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' && github.sha || github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash
jobs:
build-merge:
name: Build and push `pre-commit` image
permissions:
contents: read
packages: write
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32
with:
image_name: xrpld/pre-commit
dockerfile: bin/pre-commit/Dockerfile
base_image: ubuntu:26.04
push: ${{ github.event_name == 'push' }}

View File

@@ -20,4 +20,4 @@ on:
jobs:
check_title:
if: ${{ github.event.pull_request.draft != true }}
uses: XRPLF/actions/.github/workflows/check-pr-title.yml@cba1f0891650baf1a9c88624dc2d72573be2eb81
uses: XRPLF/actions/.github/workflows/check-pr-title.yml@d7c65e49225a38f6d8010eacf017bb5a98d7476c

View File

@@ -25,18 +25,11 @@ on:
- unlabeled
concurrency:
# Use a per-ref group so a newer run (a push, or a change to a label below)
# supersedes the in-progress one for that ref. Label events we don't act on get
# their own unique group (per run id) instead, keeping them out of the shared
# group so real builds keep running. Keep this list in sync with `should-run`.
group: >-
${{ github.workflow }}-${{ github.ref }}${{
((github.event.action == 'labeled' || github.event.action == 'unlabeled')
&& github.event.label.name != 'Ready to merge'
&& github.event.label.name != 'DraftRunCI'
&& github.event.label.name != 'Full CI build')
&& format('-{0}', github.run_id) || ''
}}
# A single per-ref group with cancel-in-progress means any newer run (a push
# or a label change) supersedes the in-progress one for that ref. Keeping
# exactly one authoritative run per ref ensures a fast do-nothing run can never
# mask a real build's checks.
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
defaults:
@@ -44,21 +37,17 @@ defaults:
shell: bash
jobs:
# This job determines whether the rest of the workflow should run. It runs
# when the PR is not a draft (which should also cover merge-group) or has the
# 'DraftRunCI' or 'Full CI build' label. For label events it only runs when the
# label added or removed is one we act on ('Ready to merge', 'DraftRunCI' or
# 'Full CI build'), so unrelated label changes do not trigger a redundant run.
# This job determines whether the rest of the workflow should run at all,
# based on the current set of labels: it runs when the PR is not a draft
# (which should also cover merge-group) or has the 'DraftRunCI' or
# 'Full CI build' label. Whether a build then happens, and whether it is the
# minimal or full matrix, is decided further below and in the strategy matrix.
should-run:
if: >-
${{
((github.event.action != 'labeled' && github.event.action != 'unlabeled')
|| github.event.label.name == 'Ready to merge'
|| github.event.label.name == 'DraftRunCI'
|| github.event.label.name == 'Full CI build')
&& (!github.event.pull_request.draft
|| contains(github.event.pull_request.labels.*.name, 'DraftRunCI')
|| contains(github.event.pull_request.labels.*.name, 'Full CI build'))
!github.event.pull_request.draft
|| contains(github.event.pull_request.labels.*.name, 'DraftRunCI')
|| contains(github.event.pull_request.labels.*.name, 'Full CI build')
}}
runs-on: ubuntu-latest
steps:

View File

@@ -14,7 +14,7 @@ on:
jobs:
# Call the workflow in the XRPLF/actions repo that runs the pre-commit hooks.
run-hooks:
uses: XRPLF/actions/.github/workflows/pre-commit.yml@1bde119a1ab71305ba5d3716e7a82cea1c7bdede
uses: XRPLF/actions/.github/workflows/pre-commit.yml@3ba08d6ddf114092891d48491fc2e26c3ba15552
with:
runs_on: ubuntu-latest
container: '{ "image": "ghcr.io/xrplf/ci/tools-rippled-pre-commit:sha-41ec7c1" }'
container: '{ "image": "ghcr.io/xrplf/xrpld/pre-commit:sha-f56b79f" }'

View File

@@ -41,13 +41,13 @@ env:
jobs:
build:
runs-on: ubuntu-latest
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-e29b523
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-3122de8
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Prepare runner
uses: XRPLF/actions/prepare-runner@64ec3cf3b152b4444638f470bbd6df7a7a30c81c
uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606
with:
enable_ccache: false

View File

@@ -113,7 +113,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Prepare runner
uses: XRPLF/actions/prepare-runner@64ec3cf3b152b4444638f470bbd6df7a7a30c81c
uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606
with:
enable_ccache: ${{ inputs.ccache_enabled }}
@@ -223,11 +223,13 @@ jobs:
BUILD_TYPE: ${{ inputs.build_type }}
CMAKE_TARGET: ${{ inputs.cmake_target }}
run: |
set -o pipefail
cmake \
--build . \
--config "${BUILD_TYPE}" \
--parallel "${BUILD_NPROC}" \
--target "${CMAKE_TARGET}"
--target "${CMAKE_TARGET}" \
2>&1 | tee "${GITHUB_WORKSPACE}/build.log"
- name: Show ccache statistics
if: ${{ inputs.ccache_enabled }}
@@ -322,27 +324,46 @@ jobs:
PRELOAD=""
fi
LD_PRELOAD="$PRELOAD" ./xrpld --unittest --unittest-jobs "${BUILD_NPROC}" 2>&1 | tee unittest.log
LD_PRELOAD="$PRELOAD" ./xrpld --unittest --unittest-jobs "${BUILD_NPROC}" 2>&1 | tee "${GITHUB_WORKSPACE}/unittest.log"
- name: Show test failure summary
if: ${{ failure() && !inputs.build_only }}
env:
WORKING_DIR: ${{ runner.os == 'Windows' && format('{0}\{1}', env.BUILD_DIR, inputs.build_type) || env.BUILD_DIR }}
# Smoke-run every benchmark module with a single repetition to confirm the
# benchmarks still build and execute. This is a correctness check, not a
# performance measurement, so it is skipped for instrumented builds
# (sanitizers/coverage/voidstar), where it would be slow and meaningless,
# and on Windows, where the `install` target does not build them.
- name: Run the benchmarks
if: ${{ !inputs.build_only && runner.os != 'Windows' && env.SANITIZERS_ENABLED == 'false' && env.COVERAGE_ENABLED != 'true' && env.VOIDSTAR_ENABLED != 'true' }}
working-directory: ${{ env.BUILD_DIR }}
run: |
if [ ! -d "${WORKING_DIR}" ]; then
echo "Working directory '${WORKING_DIR}' does not exist."
exit 0
fi
rc=0
while IFS= read -r bench; do
echo "::group::${bench}"
"./${bench}" --benchmark_repetitions=1 || rc=1
echo "::endgroup::"
done < <(find src/benchmarks -type f -perm -u+x -name 'xrpl.bench.*')
exit "${rc}"
cd "${WORKING_DIR}"
- name: Show build/test failure summary
if: ${{ failure() }}
run: |
cd "${GITHUB_WORKSPACE}"
if [ ! -f unittest.log ]; then
echo "unittest.log not found; embedded tests may not have run."
exit 0
fi
if ! grep -E "failed" unittest.log; then
echo "Log present but no failure lines found in unittest.log."
if [ -f unittest.log ]; then
if ! grep -E "failed" unittest.log | grep -vE "^I[0-9]|^[0-9]+> (ERR:|FTL:)"; then
echo "unittest.log present but no failure lines found."
fi
elif [ -f build.log ]; then
# GCC/Clang emit "error:" (covers "fatal error:"); MSVC emits
# "error C####:", "error LNK####:", and "fatal error LNK####:".
# -A6 prints the lines that follow each match (source line, caret,
# notes, and the "N errors generated" tally) to capture the whole
# diagnostic block.
if ! grep -E -A6 "error:|error C[0-9]{4}|error LNK[0-9]{4}|fatal error" build.log; then
echo "build.log present but no compile errors found."
fi
else
echo "unittest.log/build.log not found; something went wrong."
exit 1
fi
- name: Debug failure (Linux)
if: ${{ failure() && runner.os == 'Linux' && !inputs.build_only }}

View File

@@ -34,7 +34,7 @@ jobs:
needs: [determine-files]
if: ${{ needs.determine-files.outputs.cpp_changed_files != '' || needs.determine-files.outputs.need_full_run == 'true' }}
runs-on: ["self-hosted", "Linux", "X64", "heavy"]
container: "ghcr.io/xrplf/xrpld/nix-debian:sha-e29b523"
container: "ghcr.io/xrplf/xrpld/nix-debian:sha-3122de8"
permissions:
contents: read
issues: write
@@ -43,7 +43,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Prepare runner
uses: XRPLF/actions/prepare-runner@64ec3cf3b152b4444638f470bbd6df7a7a30c81c
uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606
with:
enable_ccache: false

View File

@@ -30,7 +30,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.13"

View File

@@ -26,7 +26,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Set up Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.13"

View File

@@ -40,7 +40,7 @@ defaults:
jobs:
upload:
runs-on: ubuntu-latest
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-e29b523
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-3122de8
env:
REMOTE_NAME: ${{ inputs.remote_name }}
CONAN_LOGIN_USERNAME_XRPLF: ${{ secrets.remote_username }}

View File

@@ -68,7 +68,7 @@ jobs:
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Prepare runner
uses: XRPLF/actions/prepare-runner@64ec3cf3b152b4444638f470bbd6df7a7a30c81c
uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606
with:
enable_ccache: false

3
.gitignore vendored
View File

@@ -81,6 +81,9 @@ DerivedData
# Python
__pycache__
# Rust build artifacts.
target/
# Direnv's directory
/.direnv

View File

@@ -55,7 +55,7 @@ repos:
types_or: [c++, c]
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: dd18dad857d6133e90bbe478f4f2f22ec0030269 # frozen: v22.1.5
rev: f4d7745e17a28aad7eed2f4874ca8d1568c11c4c # frozen: v22.1.8
hooks:
- id: clang-format
args: [--style=file]
@@ -68,7 +68,7 @@ repos:
- id: gersemi
- repo: https://github.com/rbubley/mirrors-prettier
rev: 39e2973981e6d2f9b6c543b0086a2d2393abdc89 # frozen: v3.9.4
rev: 9337a74165b178ae2c766f60bee7252a0f06f3e8 # frozen: v3.9.5
hooks:
- id: prettier
args: [--end-of-line=auto]

View File

@@ -131,6 +131,10 @@ else()
endif()
target_link_libraries(xrpl_libs INTERFACE ${nudb})
if(benchmark)
find_package(benchmark REQUIRED)
endif()
if(coverage)
include(XrplCov)
endif()
@@ -145,3 +149,7 @@ if(tests)
include(CTest)
add_subdirectory(src/tests/libxrpl)
endif()
if(benchmark)
add_subdirectory(src/benchmarks/libxrpl)
endif()

4
CODEOWNERS Normal file
View File

@@ -0,0 +1,4 @@
# By default, anyone can review changes.
# The CI tooling team should review changes to the CI configuration.
/.github/ @XRPLF/ci-tooling

View File

@@ -83,6 +83,7 @@ If you create new source files, they must be organized as follows:
`src/libxrpl`.
- All other non-test files must go under `src/xrpld`.
- All test source files must go under `src/test`.
- All benchmark source files must go under `src/benchmarks`.
The source must be formatted according to the style guide below. The easiest
way to satisfy this is to install the [`pre-commit`](#pre-commit-hooks) hooks,

55
bin/pre-commit/Dockerfile Normal file
View File

@@ -0,0 +1,55 @@
ARG BASE_IMAGE=ubuntu:26.04
FROM ${BASE_IMAGE}
SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"]
ENTRYPOINT ["/bin/bash"]
ARG DEBIAN_FRONTEND=noninteractive
RUN <<EOF
pkgs=()
pkgs+=(curl) # Required to install nix.
pkgs+=(doxygen) # Needed for Clio's check-doxygen-docs.sh.
pkgs+=(git) # Required for prepare-runner.
pkgs+=(libatomic1) # Required to run pre-commit provided `node`.
pkgs+=(python3) # Python 3 interpreter.
pkgs+=(python3-pip) # Package manager for Python applications.
pkgs+=(xz-utils) # Required to install nix
apt-get update
apt-get install -y --no-install-recommends "${pkgs[@]}"
apt-get clean
rm -rf /var/lib/apt/lists/*
EOF
ARG PRE_COMMIT_VERSION=4.6.0
RUN pip install --no-cache --break-system-packages \
pre-commit==${PRE_COMMIT_VERSION}
RUN sh <(curl --proto '=https' --tlsv1.2 -L https://nixos.org/nix/install) --daemon --yes
# Add nix to PATH and set NIX environment variables,
# so nix is available in all shells including non-interactive shells (e.g., GitHub Actions).
ENV PATH="/nix/var/nix/profiles/default/bin:${PATH}"
ENV NIX_PROFILES="/nix/var/nix/profiles/default"
ENV NIX_SSL_CERT_FILE="/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"
# Verify nix installation
RUN nix --version
ENV RUSTUP_HOME="/opt/rust/rustup"
ENV CARGO_HOME="/opt/rust/cargo"
ENV PATH="/opt/rust/cargo/bin:${PATH}"
WORKDIR /tmp
COPY rust-toolchain.toml /tmp/rust-toolchain.toml
RUN <<EOF
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
| sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
rustup toolchain install
rustup show
cargo fmt --version
EOF
WORKDIR /

View File

@@ -0,0 +1,36 @@
include(isolate_headers)
# Define a benchmark executable for the module `name`.
#
# This follows the same general pattern as other build helpers in this repo
# (e.g. `add_module`): create a target and isolate headers, but here the target
# is a benchmark executable and no `add_test(...)` is registered.
#
# `isolate_headers` exposes only `${CMAKE_CURRENT_SOURCE_DIR}/${name}` on the
# include path, rooted at `src`, so a benchmark's own headers are reached as
# `<benchmarks/.../${name}/...>` and nothing else in the tree leaks in.
function(xrpl_add_benchmark name)
set(target ${PROJECT_NAME}.bench.${name})
file(
GLOB_RECURSE sources
CONFIGURE_DEPENDS
"${CMAKE_CURRENT_SOURCE_DIR}/${name}/*.cpp"
"${CMAKE_CURRENT_SOURCE_DIR}/${name}.cpp"
)
add_executable(${target} ${ARGN} ${sources})
# Benchmark sources register cases through Google Benchmark's static
# registrars (anonymous-namespace lambdas). Merging several such files into
# one unity translation unit collides those internal-linkage entities, so
# keep benchmarks out of the unity build - mirroring xrpl.libpb in
# XrplCore.cmake. Each file compiles fine on its own.
set_target_properties(${target} PROPERTIES UNITY_BUILD OFF)
isolate_headers(
${target}
"${CMAKE_SOURCE_DIR}/src"
"${CMAKE_CURRENT_SOURCE_DIR}/${name}"
PRIVATE
)
endfunction()

View File

@@ -133,6 +133,12 @@ target_link_libraries(
add_module(xrpl resource)
target_link_libraries(xrpl.libxrpl.resource PUBLIC xrpl.libxrpl.protocol)
add_module(xrpl peerfinder)
target_link_libraries(
xrpl.libxrpl.peerfinder
PUBLIC xrpl.libxrpl.basics xrpl.libxrpl.protocol
)
# Level 08
add_module(xrpl net)
target_link_libraries(
@@ -201,6 +207,16 @@ target_link_libraries(
add_module(xrpl tx)
target_link_libraries(xrpl.libxrpl.tx PUBLIC xrpl.libxrpl.ledger)
add_module(xrpl consensus)
target_link_libraries(
xrpl.libxrpl.consensus
PUBLIC
xrpl.libxrpl.basics
xrpl.libxrpl.json
xrpl.libxrpl.protocol
xrpl.libxrpl.ledger
)
add_library(xrpl.libxrpl)
set_target_properties(xrpl.libxrpl PROPERTIES OUTPUT_NAME xrpl)
@@ -220,6 +236,7 @@ target_link_modules(
beast
conditions
config
consensus
core
crypto
git
@@ -227,6 +244,7 @@ target_link_modules(
ledger
net
nodestore
peerfinder
protocol
protocol_autogen
rdb

View File

@@ -30,6 +30,8 @@ if(tests)
endif()
endif()
option(benchmark "Build benchmarks" ON)
# Enabled by default so every header is compiled on its own as the main file of
# its own compile_commands.json entry - this is what lets clang-tidy (and clangd
# and IDEs) analyse a header's own includes directly. The per-header objects are

View File

@@ -10,22 +10,23 @@
"rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1782392413.075713",
"re2/20251105#8579cfd0bda4daf0683f9e3898f964b4%1782392402.431897",
"protobuf/6.33.5#ff253ead763bd8d9904a52979cd21e81%1782392410.233933",
"openssl/3.6.3#1163d4ddc603907084d08a6a0c6e580f%1782307150.583886",
"openssl/3.6.3#f806de8933e3bf6f01016c6a888cee2e%1783945160.863288",
"nudb/2.0.9#11149c73f8f2baff9a0198fe25971fc7%1782392402.297166",
"mpt-crypto/0.4.0-rc2#a580f2f9ad0e795de696aa62d54fb9af%1782425834.488828",
"mpt-crypto/0.4.0-rc4#ffdba12f2332357f0d8b0ae944cfff52%1784138702.932355",
"lz4/1.10.0#982d9b673900f665a1da109e09c17cab%1782392402.164188",
"libiconv/1.17#9923bc6dc6f106646d6967e0039a5ada%1782392792.775744",
"libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1782392402.420732",
"libarchive/3.8.7#c446109bd1f1d8ba7936c94189bc50e6%1782392403.066892",
"jemalloc/5.3.1#1fc58d55316041f10fbc1e8a2eae632a%1776700028.228",
"gtest/1.17.0#5224b3b3ff3b4ce1133cbdd27d53ee7d%1782392402.791979",
"grpc/1.81.1#5217e6ef0544c42b46f4af35d5e7f649%1782307148.845616",
"grpc/1.81.1#f729f6d75992d20f9c72828e9142d62f%1783945160.094135",
"ed25519/2015.03#ae761bdc52730a843f0809bdf6c1b1f6%1782307148.15562",
"date/3.0.4#862e11e80030356b53c2c38599ceb32b%1782392402.538492",
"c-ares/1.34.6#545240bb1c40e2cacd4362d6b8967650%1782392402.681654",
"bzip2/1.0.8#c470882369c2d95c5c77e970c0c7e321%1782392402.296732",
"boost/1.91.0#ea540ca2133d831b560036aa24dece3c%1782392419.475605",
"abseil/20250127.0#bb0baf1f362bc4a725a24eddd419b8f7%1782307147.395833"
"benchmark/1.9.5#b885dc73ad67b40a55d45684d1c88ad1%1782736613.864841",
"abseil/20250127.0#9ef01c1451a8340f9022e46238c0fbb6%1783945159.651047"
],
"build_requires": [
"zlib/1.3.2#1cb806da49011867778ffb6ac7190fcb%1782392402.122708",
@@ -38,7 +39,7 @@
"b2/5.4.2#ffd6084a119587e70f11cd45d1a386e2%1782392402.624226",
"automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56",
"autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86",
"abseil/20250127.0#bb0baf1f362bc4a725a24eddd419b8f7%1782307147.395833"
"abseil/20250127.0#9ef01c1451a8340f9022e46238c0fbb6%1783945159.651047"
],
"python_requires": [],
"overrides": {

View File

@@ -20,6 +20,22 @@ compiler.libcxx={{ detect_api.detect_libcxx(compiler, version, compiler_exe) }}
{% endif %}
[conf]
{# The Boost recipe builds with b2, which doesn't use Conan's toolchain files. #}
{# Instead it hand-rolls the compiler for user-config.jam, #}
{# and its fallback probes a version-suffixed binary (e.g. `g++-15`) before plain `g++`. #}
{# Inside the Nix shell the wrapper only provides `g++`/`gcc` (no `-15` suffix), #}
{# so on a host that also has a system `g++-15` the probe escapes Nix #}
{# and picks the system compiler, which is mismatched with the Nix libraries #}
{# and breaks the build (e.g. Boost.Stacktrace link checks fail). #}
{# Pinning the executables here short-circuits that probe so Boost (and the rest of the toolchain) #}
{# resolve the same compiler. #}
{# Not part of the package ID, so binaries stay shareable. #}
{% if os != "Windows" %}
{% set cc_exe = {"gcc": "gcc", "clang": "clang", "apple-clang": "clang"}.get(compiler) %}
{% set cxx_exe = {"gcc": "g++", "clang": "clang++", "apple-clang": "clang++"}.get(compiler) %}
tools.build:compiler_executables={'c':'{{ cc_exe }}','cpp':'{{ cxx_exe }}'}
{% endif %}
{# By default, Conan tries to reuse binaries built with different cppstd versions. #}
{# We want to avoid that to improve reproduceability, so we add the cppstd version to the package ID. #}
{# More info: https://docs.conan.io/2/reference/extensions/binary_compatibility.html #}

View File

@@ -15,6 +15,7 @@ class Xrpl(ConanFile):
settings = "os", "compiler", "build_type", "arch"
options = {
"assertions": [True, False],
"benchmark": [True, False],
"coverage": [True, False],
"fPIC": [True, False],
"jemalloc": [True, False],
@@ -46,6 +47,7 @@ class Xrpl(ConanFile):
default_options = {
"assertions": False,
"benchmark": True,
"coverage": False,
"fPIC": True,
"jemalloc": False,
@@ -129,12 +131,14 @@ class Xrpl(ConanFile):
self.options["boost"].without_cobalt = True
def requirements(self):
if self.options.benchmark:
self.requires("benchmark/1.9.5")
self.requires("boost/1.91.0", force=True, transitive_headers=True)
self.requires("date/3.0.4", transitive_headers=True)
if self.options.jemalloc:
self.requires("jemalloc/5.3.1")
self.requires("lz4/1.10.0", force=True)
self.requires("mpt-crypto/0.4.0-rc2", transitive_headers=True)
self.requires("mpt-crypto/0.4.0-rc4", transitive_headers=True)
self.requires("protobuf/6.33.5", force=True)
if self.options.rocksdb:
self.requires("rocksdb/10.5.1")
@@ -162,6 +166,7 @@ class Xrpl(ConanFile):
def generate(self):
tc = CMakeToolchain(self)
tc.variables["tests"] = self.options.tests
tc.variables["benchmark"] = self.options.benchmark
tc.variables["assert"] = self.options.assertions
tc.variables["coverage"] = self.options.coverage
tc.variables["jemalloc"] = self.options.jemalloc

30
docs/build/nix.md vendored
View File

@@ -38,8 +38,10 @@ The first time you run this command, it will take a few minutes to download and
### Platform notes
- **Linux**: `nix develop` gives you a shell with all the tooling necessary to
develop xrpld and with GCC 15.2 (also provided by Nix). There are no caveats.
- **Linux**: `nix develop` gives you a shell with all the tooling necessary to develop xrpld
and with the same GCC/glibc toolchain that Nix builds for CI.
See [Choosing a different compiler](#choosing-a-different-compiler)
for the custom-vs-plain toolchain trade-off.
- **macOS**: `nix develop` gives you a full environment too, with Clang (and
every other tool, including Conan) provided by Nix. To use your system-wide
Apple Clang instead, enter `nix develop .#apple-clang`. Conan has no binary in
@@ -63,8 +65,16 @@ The first time you run this command, it will take a few minutes to download and
### Choosing a different compiler
A compiler can be chosen by providing its name with the `.#` prefix, e.g. `nix develop .#clang`.
The `.#gcc` and `.#clang` shells provide the same GCC and Clang versions used in CI
(pinned in [`nix/packages.nix`](../../nix/packages.nix)).
On Linux, `.#gcc` and `.#clang` provide the exact toolchain CI uses:
the compiler (pinned in [`nix/packages.nix`](../../nix/packages.nix))
rebuilt against the pinned custom glibc (see [`nix/compilers.nix`](../../nix/compilers.nix)).
Building that toolchain the first time is slow unless it is fetched from a Nix binary cache.
If you don't need the custom glibc, the Linux-only `.#gcc-plain` and `.#clang-plain`
give you the stock nixpkgs compilers of the same versions.
On macOS there is no custom glibc, so `.#gcc` and `.#clang` are already the plain nixpkgs toolchain,
and the `-plain` variants do not exist.
Use `nix flake show` to see all the available development shells.
Use `nix develop .#no-compiler` to use the compiler from your system.
@@ -72,14 +82,18 @@ Use `nix develop .#no-compiler` to use the compiler from your system.
### Example Usage
```bash
# Use GCC (same version as CI)
# Use GCC — same toolchain as CI (custom glibc on Linux)
nix develop .#gcc
# Use Clang (same version as CI)
# Use Clang — same toolchain as CI (custom glibc on Linux)
nix develop .#clang
# Use default for your platform
nix develop
# Stock nixpkgs GCC/Clang, Linux only — skips the custom-glibc build, but does not match CI
nix develop .#gcc-plain
nix develop .#clang-plain
```
### Using a different shell
@@ -110,6 +124,10 @@ nix develop -c "$SHELL"
Once inside the Nix development shell, follow the standard [build instructions](../../BUILD.md#steps). The Nix shell provides all necessary tools (CMake, Ninja, Conan, etc.).
Coverage builds (`-Dcoverage=ON`) work in the `gcc` shell (and `gcc-plain` on Linux):
each ships a `gcov` matching its compiler, since Nix's cc-wrapper does not expose one.
The `clang` shells do not include `llvm-cov`, so use a `gcc` shell for coverage.
## Automatic Activation with direnv
[direnv](https://direnv.net/) or [nix-direnv](https://github.com/nix-community/nix-direnv) can automatically activate the Nix development shell when you enter the repository directory.

23
flake.lock generated
View File

@@ -36,7 +36,28 @@
"root": {
"inputs": {
"nixpkgs": "nixpkgs",
"nixpkgs-custom-glibc": "nixpkgs-custom-glibc"
"nixpkgs-custom-glibc": "nixpkgs-custom-glibc",
"rust-overlay": "rust-overlay"
}
},
"rust-overlay": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1784611586,
"narHash": "sha256-OfqgY+0hp/zseZB7uyH0U8kIDPS4scZZCyAurEplvG0=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "14f58845249f3552a89b07772626b8d3c632fa86",
"type": "github"
},
"original": {
"owner": "oxalica",
"repo": "rust-overlay",
"type": "github"
}
}
},

View File

@@ -10,12 +10,25 @@
url = "github:NixOS/nixpkgs/9cd98386a38891d1074fc18036b842dc4416f562";
flake = false;
};
# Pinned Rust toolchains, delivered from the Nix store. Lets the Nix CI
# image and dev shell honour the single `rust-toolchain.toml` pin (shared
# with the rustup-based non-Nix runners) while staying hermetic — the
# toolchain lands in the image's Nix closure and is locked by flake.lock.
rust-overlay = {
url = "github:oxalica/rust-overlay";
inputs.nixpkgs.follows = "nixpkgs";
};
};
outputs =
{ nixpkgs, nixpkgs-custom-glibc, ... }:
{
nixpkgs,
nixpkgs-custom-glibc,
rust-overlay,
...
}:
let
forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc; };
forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc rust-overlay; };
in
{
devShells = forEachSystem (import ./nix/devshell.nix);

View File

@@ -3,6 +3,9 @@
#include <xrpl/basics/IntrusivePointer.ipp>
#include <xrpl/basics/Log.h> // IWYU pragma: keep
#include <xrpl/basics/TaggedCache.h>
#include <xrpl/basics/scope.h>
#include <algorithm>
namespace xrpl {
@@ -601,8 +604,42 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
std::vector<key_type> v;
{
std::scoped_lock const lock(mutex_);
v.reserve(cache_.size());
// Keep track of how many iterations are needed. Exit the loop if the number of retries gets
// absurd. (Note that if this somehow ever happens, one more allocation will be done under
// lock, which is undesirable, but really should be almost impossible.)
std::size_t allocationIterations = 0;
std::unique_lock lock(mutex_);
for (auto size = cache_.size(); v.capacity() < size && allocationIterations < 20;
size = cache_.size())
{
ScopeUnlock const unlock(lock);
if (allocationIterations > 0)
{
JLOG(journal_.info())
<< "getKeys(): Cache grew beyond allocated capacity after "
<< allocationIterations << " prior attempt(s). Have " << v.capacity()
<< ", need " << size << ". Retrying allocation";
}
// Allocate the current size plus a little extra, in case the cache grows while
// allocating. Each time another allocation is needed, the extra also gets bigger until
// it ultimately doubles the size + 1.
constexpr std::size_t baseShift = 5;
auto const bufferOffset = std::min(allocationIterations, std::size_t{baseShift});
auto const bufferShift = baseShift - bufferOffset;
size += (size >> bufferShift) + 1;
v.reserve(size);
++allocationIterations;
}
if (v.capacity() < cache_.size())
{
// LCOV_EXCL_START
UNREACHABLE("xrpl::TaggedCache::getKeys(): failed to allocate sufficient capacity");
v.reserve(cache_.size());
// LCOV_EXCL_STOP
}
XRPL_ASSERT(lock.owns_lock(), "xrpl::TaggedCache::getKeys(): owns lock");
XRPL_ASSERT(
v.capacity() >= cache_.size(), "xrpl::TaggedCache::getKeys(): sufficient capacity");
for (auto const& _ : cache_)
v.push_back(_.first);
}

View File

@@ -5,6 +5,7 @@
#include <xrpl/beast/container/detail/aged_associative_container.h>
#include <xrpl/beast/container/detail/aged_container_iterator.h>
#include <xrpl/beast/container/detail/empty_base_optimization.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <boost/intrusive/list.hpp>
#include <boost/intrusive/unordered_set.hpp>

View File

@@ -10,7 +10,7 @@
namespace xrpl {
template <class TxID, class Sequence>
class RCLCensorshipDetector
class CensorshipDetector
{
public:
struct TxIDSeq
@@ -49,7 +49,7 @@ private:
TxIDSeqVec tracker_;
public:
RCLCensorshipDetector() = default;
CensorshipDetector() = default;
/**
* Add transactions being proposed for the current consensus round.

View File

@@ -1,15 +1,14 @@
#pragma once
#include <xrpld/consensus/ConsensusParms.h>
#include <xrpld/consensus/ConsensusProposal.h>
#include <xrpld/consensus/ConsensusTypes.h>
#include <xrpl/basics/Log.h>
#include <xrpl/basics/UnorderedContainers.h>
#include <xrpl/basics/chrono.h>
#include <xrpl/beast/clock/abstract_clock.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/consensus/ConsensusParms.h>
#include <xrpl/consensus/ConsensusProposal.h>
#include <xrpl/consensus/ConsensusTypes.h>
#include <xrpl/json/json_value.h>
#include <xrpl/json/json_writer.h>
#include <xrpl/ledger/LedgerTiming.h>

View File

@@ -1,11 +1,10 @@
#pragma once
#include <xrpld/consensus/ConsensusProposal.h>
#include <xrpld/consensus/DisputedTx.h>
#include <xrpl/basics/UnorderedContainers.h>
#include <xrpl/basics/chrono.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/consensus/ConsensusProposal.h>
#include <xrpl/consensus/DisputedTx.h>
#include <chrono>
#include <cstddef>

View File

@@ -1,9 +1,8 @@
#pragma once
#include <xrpld/consensus/ConsensusParms.h>
#include <xrpl/basics/Log.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/consensus/ConsensusParms.h>
#include <xrpl/json/json_value.h>
#include <xrpl/json/json_writer.h>

View File

@@ -1,7 +1,5 @@
#pragma once
#include <xrpld/consensus/LedgerTrie.h>
#include <xrpl/basics/Log.h>
#include <xrpl/basics/UnorderedContainers.h>
#include <xrpl/basics/chrono.h>
@@ -11,6 +9,7 @@
#include <xrpl/beast/hash/uhash.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/consensus/LedgerTrie.h>
#include <xrpl/json/json_value.h>
#include <algorithm>

View File

@@ -263,10 +263,11 @@ constructLoanState(
Number const& principalOutstanding,
Number const& managementFeeOutstanding);
// Constructs a valid LoanState object from a Loan object, which always has
// rounded values
// Overload of constructLoanState() that reads the three tracked fields
// directly from a Loan ledger object, which always holds rounded values,
// rather than taking them as separate Number arguments.
LoanState
constructRoundedLoanState(SLE::const_ref loan);
constructLoanState(SLE::const_ref loan);
Number
computeManagementFee(

View File

@@ -41,6 +41,19 @@ public:
std::unique_ptr<NodeStore::Backend>&& newBackend,
std::function<void(std::string const& writableName, std::string const& archiveName)> const&
f) = 0;
/**
* Marks an online-delete rotation as in progress (or completed).
*
* While in flight, a read served by the archive backend is copied
* forward into the writable backend even for ordinary
* (duplicate == false) fetches: the archive is about to be deleted,
* and a node body canonicalized into caches during the rotation
* window would otherwise survive only in RAM once the archive is
* dropped.
*/
virtual void
setRotationInFlight(bool inFlight) = 0;
};
} // namespace xrpl::NodeStore

View File

@@ -9,6 +9,7 @@
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Scheduler.h>
#include <atomic>
#include <cstdint>
#include <functional>
#include <memory>
@@ -69,11 +70,22 @@ public:
void
sweep() override;
void
setRotationInFlight(bool inFlight) override;
private:
std::shared_ptr<Backend> writableBackend_;
std::shared_ptr<Backend> archiveBackend_;
mutable std::mutex mutex_;
// True between SHAMapStore starting the cache-freshen phase and the
// completion of rotate(). While true, archive hits on ordinary
// (duplicate == false) fetches are copied forward into the writable
// backend; copyForwardCount_ tallies them per rotation for the
// summary line logged at swap.
std::atomic<bool> rotationInFlight_{false};
std::atomic<std::uint64_t> copyForwardCount_{0};
std::shared_ptr<NodeObject>
fetchNodeObject(uint256 const& hash, std::uint32_t, FetchReport& fetchReport, bool duplicate)
override;

View File

@@ -0,0 +1,163 @@
#pragma once
#include <xrpl/beast/utility/PropertyStream.h>
#include <xrpl/peerfinder/detail/Tuning.h>
#include <cstddef>
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
namespace xrpl::PeerFinder {
struct PeerLimitConfig
{
std::optional<std::size_t> maxPeers;
std::optional<std::size_t> inPeers;
std::optional<std::size_t> outPeers;
};
/**
* PeerFinder configuration settings.
*/
struct Config
{
/**
* The largest number of public peer slots to allow.
* This includes both inbound and outbound, but does not include
* fixed peers.
*/
std::size_t maxPeers{Tuning::kDefaultMaxPeers};
/**
* The number of automatic outbound connections to maintain.
* Outbound connections are only maintained if autoConnect
* is `true`.
*/
std::size_t outPeers = calcOutPeers(); // Note: relies on `maxPeers` being initialized
/**
* The number of automatic inbound connections to maintain.
* Inbound connections are only maintained if wantIncoming
* is `true`.
*/
std::size_t inPeers{0};
/**
* `true` if we want our IP address kept private.
*/
bool peerPrivate = true;
/**
* `true` if we want to accept incoming connections.
*/
bool wantIncoming{true};
/**
* `true` if we want to establish connections automatically
*/
bool autoConnect{true};
/**
* The listening port number.
*/
std::uint16_t listeningPort{0};
/**
* The set of features we advertise.
*/
std::string features;
/**
* Limit how many incoming connections we allow per IP
*/
int ipLimit{0};
/**
* `true` if we want to verify endpoints in TMEndpoints messages
*/
bool verifyEndpoints = true;
//--------------------------------------------------------------------------
/**
* Returns a suitable value for outPeers according to the rules.
*/
[[nodiscard]] std::size_t
calcOutPeers() const;
/**
* Adjusts the values so they follow the business rules.
*/
void
applyTuning();
/**
* Write the configuration into a property stream
*/
void
onWrite(beast::PropertyStream::Map& map) const;
/**
* Make PeerFinder::Config from peer limit and server mode parameters.
*/
static Config
makeConfig(
bool peerPrivate,
bool standalone,
PeerLimitConfig const& limits,
std::uint16_t port,
bool validationPublicKey,
int ipLimit,
bool verifyEndpoints);
/**
* Compares two configurations for equality field by field.
*/
friend bool
operator==(Config const& lhs, Config const& rhs) = default;
};
//------------------------------------------------------------------------------
/**
* Possible results from activating a slot.
*/
enum class Result { InboundDisabled, DuplicatePeer, IpLimitExceeded, Full, Success };
/**
* @brief Converts a `Result` enum value to its string representation.
*
* This function provides a human-readable string for a given `Result` enum,
* which is useful for logging, debugging, or displaying status messages.
*
* @param result The `Result` enum value to convert.
* @return A `std::string_view` representing the enum value. Returns "unknown"
* if the enum value is not explicitly handled.
*
* @note This function returns a `std::string_view` for performance.
* A `std::string` would need to allocate memory on the heap and copy the
* string literal into it every time the function is called.
*/
inline std::string_view
to_string(Result result) noexcept
{
switch (result)
{
case Result::InboundDisabled:
return "inbound disabled";
case Result::DuplicatePeer:
return "peer already connected";
case Result::IpLimitExceeded:
return "ip limit exceeded";
case Result::Full:
return "slots full";
case Result::Success:
return "success";
}
return "unknown";
}
} // namespace xrpl::PeerFinder

View File

@@ -0,0 +1,179 @@
#pragma once
#include <xrpl/beast/net/IPEndpoint.h>
#include <xrpl/beast/utility/PropertyStream.h>
#include <xrpl/peerfinder/Config.h>
#include <xrpl/peerfinder/Slot.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/protocol/PublicKey.h>
#include <boost/asio/ip/tcp.hpp>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace xrpl::PeerFinder {
/**
* Maintains a set of IP addresses used for getting into the network.
*/
class Manager : public beast::PropertyStream::Source
{
protected:
Manager() noexcept;
public:
/**
* Destroy the object.
* Any pending source fetch operations are aborted.
* There may be some listener calls made before the
* destructor returns.
*/
~Manager() override = default;
/**
* Set the configuration for the manager.
* The new settings will be applied asynchronously.
* Thread safety:
* Can be called from any threads at any time.
*/
virtual void
setConfig(Config const& config) = 0;
/**
* Transition to the started state, synchronously.
*/
virtual void
start() = 0;
/**
* Transition to the stopped state, synchronously.
*/
virtual void
stop() = 0;
/**
* Returns the configuration for the manager.
*/
virtual Config
config() = 0;
/**
* Add a peer that should always be connected.
* This is useful for maintaining a private cluster of peers.
* The string is the name as specified in the configuration
* file, along with the set of corresponding IP addresses.
*/
virtual void
addFixedPeer(std::string_view name, std::vector<beast::IP::Endpoint> const& addresses) = 0;
/**
* Add a set of strings as fallback IP::Endpoint sources.
* @param name A label used for diagnostics.
*/
virtual void
addFallbackStrings(std::string const& name, std::vector<std::string> const& strings) = 0;
/**
* Add a URL as a fallback location to obtain IP::Endpoint sources.
* @param name A label used for diagnostics.
*/
/* VFALCO NOTE Unimplemented
virtual void addFallbackURL (std::string const& name,
std::string const& url) = 0;
*/
//--------------------------------------------------------------------------
/**
* Create a new inbound slot with the specified remote endpoint.
* If nullptr is returned, then the slot could not be assigned.
* Usually this is because of a detected self-connection.
*/
virtual std::pair<std::shared_ptr<Slot>, Result>
newInboundSlot(
beast::IP::Endpoint const& localEndpoint,
beast::IP::Endpoint const& remoteEndpoint) = 0;
/**
* Create a new outbound slot with the specified remote endpoint.
* If nullptr is returned, then the slot could not be assigned.
* Usually this is because of a duplicate connection.
*/
virtual std::pair<std::shared_ptr<Slot>, Result>
newOutboundSlot(beast::IP::Endpoint const& remoteEndpoint) = 0;
/**
* Called when mtENDPOINTS is received.
*/
virtual void
onEndpoints(std::shared_ptr<Slot> const& slot, Endpoints const& endpoints) = 0;
/**
* Called when the slot is closed.
* This always happens when the socket is closed, unless the socket
* was canceled.
*/
virtual void
onClosed(std::shared_ptr<Slot> const& slot) = 0;
/**
* Called when an outbound connection is deemed to have failed
*/
virtual void
onFailure(std::shared_ptr<Slot> const& slot) = 0;
/**
* Called when we received redirect IPs from a busy peer.
*/
virtual void
onRedirects(
boost::asio::ip::tcp::endpoint const& remoteAddress,
std::vector<boost::asio::ip::tcp::endpoint> const& eps) = 0;
//--------------------------------------------------------------------------
/**
* Called when an outbound connection attempt succeeds.
* The local endpoint must be valid. If the caller receives an error
* when retrieving the local endpoint from the socket, it should
* proceed as if the connection attempt failed by calling on_closed
* instead of on_connected.
* @return `true` if the connection should be kept
*/
virtual bool
onConnected(std::shared_ptr<Slot> const& slot, beast::IP::Endpoint const& localEndpoint) = 0;
/**
* Request an active slot type.
*/
virtual Result
activate(std::shared_ptr<Slot> const& slot, PublicKey const& key, bool reserved) = 0;
/**
* Returns a set of endpoints suitable for redirection.
*/
virtual std::vector<Endpoint>
redirect(std::shared_ptr<Slot> const& slot) = 0;
/**
* Return a set of addresses we should connect to.
*/
virtual std::vector<beast::IP::Endpoint>
autoconnect() = 0;
virtual std::vector<std::pair<std::shared_ptr<Slot>, std::vector<Endpoint>>>
buildEndpointsForPeers() = 0;
/**
* Perform periodic activity.
* This should be called once per second.
*/
virtual void
oncePerSecond() = 0;
};
} // namespace xrpl::PeerFinder

View File

@@ -0,0 +1,46 @@
#pragma once
#include <xrpl/beast/clock/abstract_clock.h>
#include <xrpl/beast/net/IPEndpoint.h>
#include <xrpl/peerfinder/detail/Tuning.h>
#include <chrono>
#include <cstdint>
#include <vector>
namespace xrpl::PeerFinder {
using clock_type = beast::AbstractClock<std::chrono::steady_clock>;
/**
* Represents a set of addresses.
*/
using IPAddresses = std::vector<beast::IP::Endpoint>;
//------------------------------------------------------------------------------
/**
* Describes a connectable peer address along with some metadata.
*/
struct Endpoint
{
Endpoint() = default;
Endpoint(beast::IP::Endpoint ep, std::uint32_t hops);
std::uint32_t hops = 0;
beast::IP::Endpoint address;
};
inline bool
operator<(Endpoint const& lhs, Endpoint const& rhs)
{
return lhs.address < rhs.address;
}
/**
* A set of Endpoint used for connecting.
*/
using Endpoints = std::vector<Endpoint>;
} // namespace xrpl::PeerFinder

View File

@@ -1,11 +1,11 @@
#pragma once
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/detail/Store.h>
#include <xrpl/beast/net/IPEndpoint.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/PropertyStream.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/peerfinder/detail/Store.h>
#include <xrpl/peerfinder/detail/Tuning.h>
#include <boost/bimap.hpp>
#include <boost/bimap/multiset_of.hpp>

View File

@@ -1,11 +1,10 @@
#pragma once
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/Slot.h>
#include <xrpld/peerfinder/detail/Tuning.h>
#include <xrpl/beast/utility/PropertyStream.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/peerfinder/Config.h>
#include <xrpl/peerfinder/Slot.h>
#include <xrpl/peerfinder/detail/Tuning.h>
#include <cstddef>
#include <sstream>

View File

@@ -1,7 +1,7 @@
#pragma once
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/detail/Tuning.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/peerfinder/detail/Tuning.h>
#include <algorithm>
#include <chrono>

View File

@@ -1,12 +1,11 @@
#pragma once
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/detail/SlotImp.h>
#include <xrpld/peerfinder/detail/Tuning.h>
#include <xrpl/beast/container/aged_set.h>
#include <xrpl/beast/net/IPAddress.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/peerfinder/detail/SlotImp.h>
#include <xrpl/peerfinder/detail/Tuning.h>
#include <algorithm>
#include <cstddef>

View File

@@ -1,9 +1,5 @@
#pragma once
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/detail/Tuning.h>
#include <xrpld/peerfinder/detail/iosformat.h>
#include <xrpl/basics/Log.h>
#include <xrpl/basics/random.h>
#include <xrpl/beast/container/aged_map.h>
@@ -12,6 +8,8 @@
#include <xrpl/beast/utility/PropertyStream.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/beast/utility/maybe_const.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/peerfinder/detail/Tuning.h>
#include <boost/intrusive/list.hpp>
#include <boost/iterator/transform_iterator.hpp>
@@ -22,6 +20,8 @@
#include <cstddef>
#include <cstdint>
#include <functional>
#include <iomanip>
#include <ios>
#include <iterator>
#include <memory>
#include <sstream>
@@ -411,7 +411,7 @@ Livecache<Allocator>::expire()
}
if (n > 0)
{
JLOG(journal_.debug()) << beast::Leftw(18) << "Livecache expired " << n
JLOG(journal_.debug()) << std::left << std::setw(18) << "Livecache expired " << n
<< ((n > 1) ? " entries" : " entry");
}
}
@@ -434,7 +434,7 @@ Livecache<Allocator>::insert(Endpoint const& ep)
if (result.second)
{
hops.insert(e);
JLOG(journal_.debug()) << beast::Leftw(18) << "Livecache insert " << ep.address
JLOG(journal_.debug()) << std::left << std::setw(18) << "Livecache insert " << ep.address
<< " at hops " << ep.hops;
return;
}
@@ -442,7 +442,7 @@ Livecache<Allocator>::insert(Endpoint const& ep)
{
// Drop duplicates at higher hops
std::size_t const excess(ep.hops - e.endpoint.hops);
JLOG(journal_.trace()) << beast::Leftw(18) << "Livecache drop " << ep.address
JLOG(journal_.trace()) << std::left << std::setw(18) << "Livecache drop " << ep.address
<< " at hops +" << excess;
return;
}
@@ -453,12 +453,12 @@ Livecache<Allocator>::insert(Endpoint const& ep)
if (ep.hops < e.endpoint.hops)
{
hops.reinsert(e, ep.hops);
JLOG(journal_.debug()) << beast::Leftw(18) << "Livecache update " << ep.address
JLOG(journal_.debug()) << std::left << std::setw(18) << "Livecache update " << ep.address
<< " at hops " << ep.hops;
}
else
{
JLOG(journal_.trace()) << beast::Leftw(18) << "Livecache refresh " << ep.address
JLOG(journal_.trace()) << std::left << std::setw(18) << "Livecache refresh " << ep.address
<< " at hops " << ep.hops;
}
}

View File

@@ -1,17 +1,5 @@
#pragma once
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/Slot.h>
#include <xrpld/peerfinder/detail/Bootcache.h>
#include <xrpld/peerfinder/detail/Counts.h>
#include <xrpld/peerfinder/detail/Fixed.h>
#include <xrpld/peerfinder/detail/Handouts.h>
#include <xrpld/peerfinder/detail/Livecache.h>
#include <xrpld/peerfinder/detail/SlotImp.h>
#include <xrpld/peerfinder/detail/Source.h>
#include <xrpld/peerfinder/detail/Store.h>
#include <xrpld/peerfinder/detail/iosformat.h>
#include <xrpl/basics/Log.h>
#include <xrpl/basics/contract.h>
#include <xrpl/basics/random.h>
@@ -22,18 +10,34 @@
#include <xrpl/beast/utility/PropertyStream.h>
#include <xrpl/beast/utility/WrappedSink.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/peerfinder/Config.h>
#include <xrpl/peerfinder/Slot.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/peerfinder/detail/Bootcache.h>
#include <xrpl/peerfinder/detail/Counts.h>
#include <xrpl/peerfinder/detail/Fixed.h>
#include <xrpl/peerfinder/detail/Handouts.h>
#include <xrpl/peerfinder/detail/Livecache.h>
#include <xrpl/peerfinder/detail/SlotImp.h>
#include <xrpl/peerfinder/detail/Source.h>
#include <xrpl/peerfinder/detail/Store.h>
#include <xrpl/protocol/PublicKey.h>
#include <boost/asio/error.hpp>
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <iomanip>
#include <ios>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <set>
#include <stdexcept>
#include <string>
#include <string_view>
#include <tuple>
#include <utility>
@@ -197,8 +201,8 @@ public:
if (result.second)
{
JLOG(journal.debug())
<< beast::Leftw(18) << "Logic add fixed '" << name << "' at " << remoteAddress;
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic add fixed '" << name
<< "' at " << remoteAddress;
return;
}
}
@@ -221,7 +225,7 @@ public:
if (iter == slots.end())
{
// The slot disconnected before we finished the check
JLOG(journal.debug()) << beast::Leftw(18) << "Logic tested " << checkedAddress
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic tested " << checkedAddress
<< " but the connection was closed";
return;
}
@@ -255,7 +259,7 @@ public:
beast::IP::Endpoint const& localEndpoint,
beast::IP::Endpoint const& remoteEndpoint)
{
JLOG(journal.debug()) << beast::Leftw(18) << "Logic accept" << remoteEndpoint
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic accept" << remoteEndpoint
<< " on local " << localEndpoint;
std::scoped_lock const _(lock);
@@ -266,7 +270,7 @@ public:
auto const count = connectedAddresses.count(remoteEndpoint.address());
if (count + 1 > config_.ipLimit)
{
JLOG(journal.debug()) << beast::Leftw(18) << "Logic dropping inbound "
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic dropping inbound "
<< remoteEndpoint << " because of ip limits.";
return {SlotImp::ptr(), Result::IpLimitExceeded};
}
@@ -275,8 +279,8 @@ public:
// Check for duplicate connection
if (slots.contains(remoteEndpoint))
{
JLOG(journal.debug()) << beast::Leftw(18) << "Logic dropping " << remoteEndpoint
<< " as duplicate incoming";
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic dropping "
<< remoteEndpoint << " as duplicate incoming";
return {SlotImp::ptr(), Result::DuplicatePeer};
}
@@ -304,15 +308,15 @@ public:
std::pair<SlotImp::ptr, Result>
newOutboundSlot(beast::IP::Endpoint const& remoteEndpoint)
{
JLOG(journal.debug()) << beast::Leftw(18) << "Logic connect " << remoteEndpoint;
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic connect " << remoteEndpoint;
std::scoped_lock const _(lock);
// Check for duplicate connection
if (slots.contains(remoteEndpoint))
{
JLOG(journal.debug()) << beast::Leftw(18) << "Logic dropping " << remoteEndpoint
<< " as duplicate connect";
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic dropping "
<< remoteEndpoint << " as duplicate connect";
return {SlotImp::ptr(), Result::DuplicatePeer};
}
@@ -506,15 +510,15 @@ public:
if (!h.list().empty())
{
JLOG(journal.debug())
<< beast::Leftw(18) << "Logic connect " << h.list().size() << " fixed";
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic connect "
<< h.list().size() << " fixed";
return h.list();
}
if (counts_.attempts() > 0)
{
JLOG(journal.debug())
<< beast::Leftw(18) << "Logic waiting on " << counts_.attempts() << " attempts";
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic waiting on "
<< counts_.attempts() << " attempts";
return none;
}
}
@@ -535,14 +539,14 @@ public:
if (!h.list().empty())
{
JLOG(journal.debug())
<< beast::Leftw(18) << "Logic connect " << h.list().size() << " live "
<< std::left << std::setw(18) << "Logic connect " << h.list().size() << " live "
<< ((h.list().size() > 1) ? "endpoints" : "endpoint");
return h.list();
}
if (counts_.attempts() > 0)
{
JLOG(journal.debug())
<< beast::Leftw(18) << "Logic waiting on " << counts_.attempts() << " attempts";
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic waiting on "
<< counts_.attempts() << " attempts";
return none;
}
}
@@ -568,8 +572,9 @@ public:
if (!h.list().empty())
{
JLOG(journal.debug()) << beast::Leftw(18) << "Logic connect " << h.list().size()
<< " boot " << ((h.list().size() > 1) ? "addresses" : "address");
JLOG(journal.debug()) << std::left << std::setw(18) << "Logic connect "
<< h.list().size() << " boot "
<< ((h.list().size() > 1) ? "addresses" : "address");
return h.list();
}
@@ -689,8 +694,8 @@ public:
// Enforce hop limit
if (ep.hops > Tuning::kMaxHops)
{
JLOG(journal.debug()) << beast::Leftw(18) << "Endpoints drop " << ep.address
<< " for excess hops " << ep.hops;
JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop "
<< ep.address << " for excess hops " << ep.hops;
iter = list.erase(iter);
continue;
}
@@ -706,18 +711,18 @@ public:
}
else
{
JLOG(journal.debug())
<< beast::Leftw(18) << "Endpoints drop " << ep.address << " for extra self";
JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop "
<< ep.address << " for extra self";
iter = list.erase(iter);
continue;
}
}
// Discard invalid addresses
if (config_.verifyEndpoints && !isValidAddress(ep.address))
if (!isValidAddress(ep.address))
{
JLOG(journal.debug())
<< beast::Leftw(18) << "Endpoints drop " << ep.address << " as invalid";
JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop "
<< ep.address << " as invalid";
iter = list.erase(iter);
continue;
}
@@ -727,8 +732,8 @@ public:
return ep.address == other.address;
}))
{
JLOG(journal.debug())
<< beast::Leftw(18) << "Endpoints drop " << ep.address << " as duplicate";
JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop "
<< ep.address << " as duplicate";
iter = list.erase(iter);
continue;
}
@@ -1074,13 +1079,13 @@ public:
if (!results.error)
{
int const count(addBootcacheAddresses(results.addresses));
JLOG(journal.info()) << beast::Leftw(18) << "Logic added " << count << " new "
JLOG(journal.info()) << std::left << std::setw(18) << "Logic added " << count << " new "
<< ((count == 1) ? "address" : "addresses") << " from "
<< source->name();
}
else
{
JLOG(journal.error()) << beast::Leftw(18) << "Logic failed "
JLOG(journal.error()) << std::left << std::setw(18) << "Logic failed "
<< "'" << source->name() << "' fetch, "
<< results.error.message();
}
@@ -1098,8 +1103,6 @@ public:
{
if (isUnspecified(address))
return false;
if (isLoopback(address))
return false;
if (!isPublic(address))
return false;
if (address.port() == 0)
@@ -1221,8 +1224,8 @@ Logic<Checker>::onRedirects(
bootcache.insert(beast::IPAddressConversion::fromAsio(*first));
if (n > 0)
{
JLOG(journal.trace()) << beast::Leftw(18) << "Logic add " << n << " redirect IPs from "
<< remoteAddress;
JLOG(journal.trace()) << std::left << std::setw(18) << "Logic add " << n
<< " redirect IPs from " << remoteAddress;
}
}

View File

@@ -1,10 +1,9 @@
#pragma once
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/Slot.h>
#include <xrpl/beast/container/aged_unordered_map.h>
#include <xrpl/beast/net/IPEndpoint.h>
#include <xrpl/peerfinder/Slot.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/protocol/PublicKey.h>
#include <atomic>
@@ -172,7 +171,7 @@ private:
std::optional<beast::IP::Endpoint> localEndpoint_;
std::optional<PublicKey> publicKey_;
static constexpr std::int32_t kUnknownPort = -1;
static std::int32_t constexpr kUnknownPort = -1;
std::atomic<std::int32_t> listeningPort_;
public:

View File

@@ -1,8 +1,7 @@
#pragma once
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/peerfinder/Types.h>
#include <boost/system/error_code.hpp>

View File

@@ -1,6 +1,6 @@
#pragma once
#include <xrpld/peerfinder/detail/Source.h>
#include <xrpl/peerfinder/detail/Source.h>
#include <memory>
#include <string>

View File

@@ -0,0 +1,36 @@
#pragma once
#include <xrpl/beast/insight/Collector.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/peerfinder/PeerfinderManager.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/peerfinder/detail/Store.h>
#include <boost/asio/io_context.hpp>
#include <memory>
namespace xrpl::PeerFinder {
/**
* @brief Create a new Manager.
*
* @param ioContext The io_context used to schedule asynchronous work.
* @param clock The clock used for timekeeping.
* @param journal The journal used for logging.
* @param store The persistence backend for the bootstrap cache. The caller
* retains ownership and must keep it alive (and opened) for the lifetime of
* the returned Manager. This lets consumers supply their own Store
* implementation (e.g. the SQLite-backed StoreSqdb in xrpld).
* @param collector The collector used to report metrics.
* @return The newly created Manager.
*/
std::unique_ptr<Manager>
makeManager(
boost::asio::io_context& ioContext,
clock_type& clock,
beast::Journal journal,
Store& store,
beast::insight::Collector::ptr const& collector);
} // namespace xrpl::PeerFinder

View File

@@ -229,13 +229,6 @@ public:
[[nodiscard]] AccountID
getAccountID(SField const& field) const;
/**
* The account responsible for the authorization: the delegate when
* sfDelegate is present, otherwise the account.
*/
[[nodiscard]] AccountID
getInitiator() const;
[[nodiscard]] Blob
getFieldVL(SField const& field) const;
[[nodiscard]] STAmount const&

View File

@@ -142,9 +142,26 @@ public:
TxnSql status,
std::string const& escapedMetaData) const;
[[nodiscard]] std::vector<uint256> const&
/**
* The IDs of the inner transactions of a Batch.
*/
[[nodiscard]] std::vector<uint256>
getBatchTransactionIDs() const;
/**
* The inner transactions of a Batch, built and validated at construction.
* Always seated for Batch STTx instances (construction throws if oversized).
*/
[[nodiscard]] std::vector<std::shared_ptr<STTx const>> const&
getBatchTransactions() const;
/**
* The account responsible for the authorization: the delegate when
* sfDelegate is present, otherwise the account.
*/
[[nodiscard]] AccountID
getInitiator() const;
[[nodiscard]] AccountID
getFeePayerID() const;
@@ -166,13 +183,16 @@ private:
checkMultiSign(Rules const& rules, STObject const& sigObject) const;
[[nodiscard]] std::expected<void, std::string>
checkBatchSingleSign(STObject const& batchSigner) const;
checkBatchSingleSign(STObject const& batchSigner, std::vector<uint256> const& txIds) const;
[[nodiscard]] std::expected<void, std::string>
checkBatchMultiSign(STObject const& batchSigner, Rules const& rules) const;
checkBatchMultiSign(
STObject const& batchSigner,
Rules const& rules,
std::vector<uint256> const& txIds) const;
void
buildBatchTxnIds();
buildBatchTxns();
STBase*
copy(std::size_t n, void* buf) const override;
@@ -180,11 +200,11 @@ private:
move(std::size_t n, void* buf) override;
friend class detail::STVar;
std::optional<std::vector<uint256>> batchTxnIds_;
std::optional<std::vector<std::shared_ptr<STTx const>>> batchTxns_;
};
bool
passesLocalChecks(STObject const& st, std::string&);
passesLocalChecks(STTx const& tx, std::string&);
/**
* Sterilize a transaction.

View File

@@ -15,6 +15,8 @@
// Add new amendments to the top of this list.
// Keep it sorted in reverse chronological order.
XRPL_FEATURE(DefragDirectories, Supported::No, VoteBehavior::DefaultNo)
XRPL_FIX (Cleanup3_4_0, Supported::Yes, VoteBehavior::DefaultNo)
XRPL_FEATURE(Sponsor, Supported::Yes, VoteBehavior::DefaultNo)
XRPL_FEATURE(BatchV1_1, Supported::Yes, VoteBehavior::DefaultNo)
XRPL_FEATURE(LendingProtocolV1_1, Supported::No, VoteBehavior::DefaultNo)

View File

@@ -17,7 +17,8 @@ class ValidPermissionedDEX
bool regularOffers_ = false; // post-fixCleanup3_2_0: excludes deleted offers
bool badHybridsOld_ = false; // pre-fixCleanup3_1_3: missing field/domain or size > 1
bool badHybrids_ = false; // post-fixCleanup3_1_3: also catches size == 0 (size != 1)
hash_set<uint256> domains_;
hash_set<uint256> domainsOld_; // pre-fixCleanup3_4_0: also flags deleted domains
hash_set<uint256> domains_; // post-fixCleanup3_4_0: excludes deleted domains
public:
void

View File

@@ -12,6 +12,7 @@
#include <array>
#include <cstdint>
#include <optional>
namespace xrpl {
@@ -39,6 +40,9 @@ public:
static NotTEC
checkSign(PreclaimContext const& ctx);
static TER
preclaim(PreclaimContext const& ctx);
TER
doApply() override;
@@ -76,6 +80,10 @@ private:
// only be reached through Batch::checkSign.
static NotTEC
checkBatchSign(PreclaimContext const& ctx);
// nullopt on overflow or oversized signer arrays.
static std::optional<XRPAmount>
calculateBaseFeeImpl(ReadView const& view, STTx const& tx);
};
} // namespace xrpl

View File

@@ -6,106 +6,18 @@
let
inherit (import ./packages.nix { inherit pkgs; })
commonPackages
gccPackage
llvmPackages
gccVersion
llvmVersion
mkVersionedToolLinks
;
# Underlying compiler toolchains to wrap (versions pinned in packages.nix).
customGccPackage = gccPackage;
customLlvmPackages = llvmPackages;
# binutils wrapped to emit binaries that reference the custom glibc
# (dynamic linker path, library search path, RPATH).
customBinutils = pkgs.wrapBintoolsWith {
bintools = pkgs.binutils-unwrapped;
libc = customGlibc;
};
# Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom
# glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses
# the existing gcc binary but links against the custom glibc, so the
# resulting compiler ships runtime libraries that only reference symbols
# available in that glibc.
customGccCc = customGccPackage.cc.override {
stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv (
pkgs.wrapCCWith {
cc = customGccPackage.cc;
libc = customGlibc;
bintools = customBinutils;
}
);
};
# cc-wrapper around the rebuilt compiler, pointing at the custom glibc
# headers and libraries. This is what we actually expose to users.
customGcc = pkgs.wrapCCWith {
cc = customGccCc;
libc = customGlibc;
bintools = customBinutils;
};
# gcov ships in gcc's `cc` output, but the cc-wrapper doesn't expose it.
# Surface the gcov from our rebuilt gcc (linked against the custom glibc, so
# it runs under the loader installed in the image) and matching the exact
# compiler version, so gcovr can produce coverage reports in the CI env.
customGcov = pkgs.runCommand "gcov-custom-for-ci-env" { } ''
mkdir -p "$out/bin"
ln -s "${customGccCc}/bin/gcov" "$out/bin/gcov"
'';
# stdenv built around the rebuilt gcc / custom glibc. Used to rebuild
# compiler-rt below so its sanitizer runtimes see the custom glibc
# headers.
customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc;
# Rebuild compiler-rt against the custom glibc so the sanitizer runtimes
# don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that
# only exist in newer glibc versions. scudo is dropped because its CMake
# includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which
# makes std::atomic unfindable in our stdenv; we don't use scudo (only
# asan/ubsan/tsan etc.).
customCompilerRt =
(customLlvmPackages.compiler-rt.override {
stdenv = customStdenv;
}).overrideAttrs
(old: {
postPatch = (old.postPatch or "") + ''
substituteInPlace lib/CMakeLists.txt \
--replace-quiet 'add_subdirectory(scudo/standalone)' \
'# scudo/standalone disabled in xrpld ci-env'
'';
});
# cc-wrapper around clang, pointing at the custom glibc headers and
# libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that
# C++ binaries produced by clang also only reference symbols available
# in the custom glibc. compiler-rt is wired into a resource-root so
# sanitizer runtimes (libclang_rt.*.a) are found at link time; this
# mirrors what nixpkgs does internally when building llvmPackages.clang.
customClang = pkgs.wrapCCWith {
cc = customLlvmPackages.clang-unwrapped;
libc = customGlibc;
bintools = customBinutils;
gccForLibs = customGccCc;
extraPackages = [ customCompilerRt ];
extraBuildCommands = ''
rsrc="$out/resource-root"
mkdir "$rsrc"
ln -s "${customLlvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include"
ln -s "${customCompilerRt.out}/lib" "$rsrc/lib"
ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true
echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags
# compiler-rt ships the sanitizer/profile/xray interface headers (e.g.
# <sanitizer/lsan_interface.h>) in its `dev` output. In a normal Nix
# build these reach the include path because compiler-rt is propagated
# via depsTargetTargetPropagated and stdenv's setup hooks add its
# dev/include. The CI image runs clang outside a Nix stdenv (binaries
# on PATH, no setup hooks), so that never happens; add the headers
# explicitly. gcc ships its own copy, which is why this is clang-only.
echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags
'';
};
# Custom-glibc toolchain, shared with the Linux dev shell (see compilers.nix).
inherit (import ./compilers.nix { inherit pkgs customGlibc; })
customGcc
customClang
customBinutils
customGcov
;
# Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can
# coexist with the gcc wrapper in buildEnv. gcc remains the default
@@ -127,6 +39,25 @@ in
customGcov
customClangForCiEnv
customBinutils
(mkVersionedToolLinks {
name = "gcc";
package = customGcc;
version = gccVersion;
tools = [
"gcc"
"g++"
"cpp"
];
})
(mkVersionedToolLinks {
name = "clang";
package = customClang;
version = llvmVersion;
tools = [
"clang"
"clang++"
];
})
# CA certificate bundle so HTTPS clients (git, curl, conan) can verify
# TLS connections without ca-certificates being installed in the system.
pkgs.cacert

117
nix/compilers.nix Normal file
View File

@@ -0,0 +1,117 @@
# Custom-glibc compiler toolchain shared by the CI environment (ci-env.nix) and
# the Linux dev shell (devshell.nix): gcc / clang / binutils rebuilt to target
# the pinned custom glibc. Linux only — the pinned glibc snapshot does not build
# on darwin, so callers must not evaluate this on macOS.
{
pkgs,
customGlibc,
}:
let
inherit (import ./packages.nix { inherit pkgs; })
gccPackage
llvmPackages
llvmVersion
mkGcov
;
# binutils wrapped to emit binaries that reference the custom glibc
# (dynamic linker path, library search path, RPATH).
customBinutils = pkgs.wrapBintoolsWith {
bintools = pkgs.binutils-unwrapped;
libc = customGlibc;
};
# Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom
# glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses
# the existing gcc binary but links against the custom glibc, so the
# resulting compiler ships runtime libraries that only reference symbols
# available in that glibc.
customGccCc = gccPackage.cc.override {
stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv (
pkgs.wrapCCWith {
cc = gccPackage.cc;
libc = customGlibc;
bintools = customBinutils;
}
);
};
# cc-wrapper around the rebuilt compiler, pointing at the custom glibc
# headers and libraries. This is what we actually expose to users.
customGcc = pkgs.wrapCCWith {
cc = customGccCc;
libc = customGlibc;
bintools = customBinutils;
};
# gcov matching the rebuilt gcc (linked against the custom glibc), so gcovr
# can produce coverage reports both in CI and in the dev shell.
customGcov = mkGcov {
name = "custom";
cc = customGccCc;
};
# stdenv built around the rebuilt gcc / custom glibc. Exported as the dev
# shell's gcc stdenv, and used below to rebuild compiler-rt so its sanitizer
# runtimes see the custom glibc headers.
customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc;
# Rebuild compiler-rt against the custom glibc so the sanitizer runtimes
# don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that
# only exist in newer glibc versions. scudo is dropped because its CMake
# includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which
# makes std::atomic unfindable in our stdenv; we don't use scudo (only
# asan/ubsan/tsan etc.).
customCompilerRt =
(llvmPackages.compiler-rt.override {
stdenv = customStdenv;
}).overrideAttrs
(old: {
postPatch = (old.postPatch or "") + ''
substituteInPlace lib/CMakeLists.txt \
--replace-quiet 'add_subdirectory(scudo/standalone)' \
'# scudo/standalone disabled in xrpld ci-env'
'';
});
# cc-wrapper around clang, pointing at the custom glibc headers and
# libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that
# C++ binaries produced by clang also only reference symbols available
# in the custom glibc. compiler-rt is wired into a resource-root so
# sanitizer runtimes (libclang_rt.*.a) are found at link time; this
# mirrors what nixpkgs does internally when building llvmPackages.clang.
customClang = pkgs.wrapCCWith {
cc = llvmPackages.clang-unwrapped;
libc = customGlibc;
bintools = customBinutils;
gccForLibs = customGccCc;
extraPackages = [ customCompilerRt ];
extraBuildCommands = ''
rsrc="$out/resource-root"
mkdir "$rsrc"
ln -s "${llvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include"
ln -s "${customCompilerRt.out}/lib" "$rsrc/lib"
ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true
echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags
# compiler-rt ships the sanitizer/profile/xray interface headers (e.g.
# <sanitizer/lsan_interface.h>) in its `dev` output. In a normal Nix
# build these reach the include path because compiler-rt is propagated
# via depsTargetTargetPropagated and stdenv's setup hooks add its
# dev/include. The CI image runs clang outside a Nix stdenv (binaries
# on PATH, no setup hooks), so that never happens; add the headers
# explicitly. gcc ships its own copy, which is why this is clang-only.
echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags
'';
};
in
{
inherit
customGcc
customClang
customBinutils
customStdenv
customGcov
;
customClangStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customClang;
}

View File

@@ -1,23 +1,64 @@
{ pkgs, ... }:
{ pkgs, customGlibc, ... }:
let
inherit (import ./packages.nix { inherit pkgs; })
commonPackages
gccPackage
gccVersion
llvmVersion
llvmPackages
mkVersionedToolLinks
mkGcov
;
# Plain nixpkgs stdenvs — no custom glibc, unlike ci-env.nix.
gccStdenv = pkgs."gcc${toString gccVersion}Stdenv";
clangStdenv = llvmPackages.stdenv;
# Plain nixpkgs stdenvs — no custom glibc.
plainGccStdenv = pkgs."gcc${toString gccVersion}Stdenv";
plainClangStdenv = llvmPackages.stdenv;
# Custom-glibc stdenvs, matching the CI environment (see compilers.nix). The
# pinned glibc snapshot only builds on Linux, so on darwin these fall back to
# the plain stdenvs; the `if isLinux` guard keeps `customGlibc` from being
# forced (and erroring) on macOS.
customCompilers = import ./compilers.nix { inherit pkgs customGlibc; };
customGccStdenv = if pkgs.stdenv.isLinux then customCompilers.customStdenv else plainGccStdenv;
customClangStdenv =
if pkgs.stdenv.isLinux then customCompilers.customClangStdenv else plainClangStdenv;
# gcov matching each gcc shell, so `-Dcoverage=ON` builds work in the shell.
plainGcov = mkGcov {
name = "plain";
cc = gccPackage.cc;
};
customGccGcov = if pkgs.stdenv.isLinux then customCompilers.customGcov else plainGcov;
# Shown when entering a *-plain shell. These exist only on Linux (see below),
# where the stock toolchain diverges from CI.
plainWarningHook = ''
echo "⚠️ WARNING: this is the stock nixpkgs toolchain and does not match CI's glibc. Prefer 'nix develop .#gcc' / '.#clang' unless you need to skip the custom-glibc build."
'';
# Tools to expose under version-suffixed names (see mkVersionedToolLinks).
gccVersionedTools = [
"gcc"
"g++"
"cpp"
];
clangVersionedTools = [
"clang"
"clang++"
];
# compilerName is the command used to print the version, or null for none.
makeShell =
{
stdenv,
compilerName,
version ? null,
versionedTools ? [ ],
extraPackages ? [ ],
warningHook ? "",
}:
let
compilerVersion =
compilerVersionHook =
if compilerName == null then
''echo "No compiler specified - using system compiler"''
else
@@ -25,12 +66,19 @@ let
echo "Compiler: "
${compilerName} --version
'';
versionedLinks = pkgs.lib.optional (version != null) (mkVersionedToolLinks {
name = compilerName;
package = stdenv.cc;
inherit version;
tools = versionedTools;
});
in
(pkgs.mkShell.override { inherit stdenv; }) {
packages = commonPackages;
packages = commonPackages ++ versionedLinks ++ extraPackages;
shellHook = ''
echo "Welcome to xrpld development shell";
${compilerVersion}
${compilerVersionHook}
${warningHook}
'';
};
in
@@ -38,14 +86,21 @@ rec {
# macOS: Nix Clang. Linux: Nix GCC.
default = if pkgs.stdenv.isDarwin then clang else gcc;
# gcc/clang use the custom-glibc toolchain, matching CI. On darwin there is no
# custom glibc, so they fall back to the plain nixpkgs toolchain.
gcc = makeShell {
stdenv = gccStdenv;
stdenv = customGccStdenv;
compilerName = "gcc";
version = gccVersion;
versionedTools = gccVersionedTools;
extraPackages = [ customGccGcov ];
};
clang = makeShell {
stdenv = clangStdenv;
stdenv = customClangStdenv;
compilerName = "clang";
version = llvmVersion;
versionedTools = clangVersionedTools;
};
# Nix provides no compiler; use the one from your system (e.g. Apple Clang).
@@ -55,3 +110,24 @@ rec {
};
apple-clang = no-compiler;
}
# The *-plain shells (stock nixpkgs toolchain) exist only on Linux: on darwin
# gcc/clang are already plain, so these would be redundant and are omitted, which
# makes `nix develop .#gcc-plain` fail there rather than silently aliasing gcc.
// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux {
gcc-plain = makeShell {
stdenv = plainGccStdenv;
compilerName = "gcc";
version = gccVersion;
versionedTools = gccVersionedTools;
extraPackages = [ plainGcov ];
warningHook = plainWarningHook;
};
clang-plain = makeShell {
stdenv = plainClangStdenv;
compilerName = "clang";
version = llvmVersion;
versionedTools = clangVersionedTools;
warningHook = plainWarningHook;
};
}

View File

@@ -8,10 +8,12 @@ RUN mkdir -p ~/.config/nix && \
# Copy our source and setup our working dir.
COPY nix/ci-env.nix /tmp/build/nix/ci-env.nix
COPY nix/compilers.nix /tmp/build/nix/compilers.nix
COPY nix/packages.nix /tmp/build/nix/packages.nix
COPY nix/utils.nix /tmp/build/nix/utils.nix
COPY flake.nix /tmp/build/
COPY flake.lock /tmp/build/
COPY rust-toolchain.toml /tmp/build/
WORKDIR /tmp/build
FROM builder-source AS builder

View File

@@ -47,7 +47,9 @@ work without `ca-certificates` being installed in the base image.
[`test_files/cpp/sources/`](./test_files/cpp/sources) with both `g++` and
`clang++`, and sanitizers, and
- compiles the Rust test programs in
[`test_files/rust/sources/`](./test_files/rust/sources) with `rustc`.
[`test_files/rust/sources/`](./test_files/rust/sources) with `rustc`, and
builds the [`test_files/rust/proc_macro/`](./test_files/rust/proc_macro)
workspace with `cargo` to exercise proc-macro dylib loading.
3. **`tester`** — Start again from a clean `BASE_IMAGE` (no Nix toolchain),
install only the sanitizer runtime libraries
([`install-sanitizer-libs.sh`](./install-sanitizer-libs.sh)), and run the
@@ -76,20 +78,23 @@ toolchain being present at runtime. Two pieces make that work:
[`loader-path.sh`](./loader-path.sh) reports the expected loader path for the
current architecture, so we can patch the binaries to use the correct loader.
The build then verifies all of this end to end: the C++ test programs in
`test_files/cpp/sources/` (a regular binary plus ASan/TSan/UBSan variants) and
the Rust test programs in `test_files/rust/sources/` (a hello binary plus panic
and overflow-check variants) are compiled in `final`, their `PT_INTERP` is
patched to the target loader, and they are run in the clean `tester` stage to
confirm each emits the expected diagnostic on a stock base image.
The build then verifies all of this end to end, and the C++ and Rust programs
go through the same pipeline: each is compiled in `final`, has its `PT_INTERP`
patched to the target loader, and is then run in the clean `tester` stage to
confirm it emits the expected diagnostic on a stock base image. The C++ programs
are in `test_files/cpp/sources/` (a regular binary plus ASan/TSan/UBSan
variants); the Rust programs are in `test_files/rust/sources/` (a hello binary
plus panic and overflow-check variants), plus the `test_files/rust/proc_macro/`
workspace — a crate whose compilation additionally loads a proc-macro dylib, and
whose resulting binary is patched and run like the others.
## Files
| File | Purpose |
| ----------------------------------------------------------------------- | ----------------------------------------------------------------------------- |
| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. |
| [`./loader-path.sh`](./loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. |
| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. |
| [`./test_files/rust/`](./test_files/rust) | Rust rustc smoke test: sources + compile/run scripts. |
| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. |
| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. |
| File | Purpose |
| ----------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
| [`./Dockerfile`](./Dockerfile) | Multi-stage build described above. |
| [`./loader-path.sh`](./loader-path.sh) | Print the dynamic-linker (`PT_INTERP`) path for the current architecture. |
| [`./test_files/cpp/`](./test_files/cpp) | C++ sanitizer smoke test: sources + compile/run scripts. |
| [`./test_files/rust/`](./test_files/rust) | Rust smoke test: rustc sources + a cargo proc-macro workspace + compile/run scripts. |
| [`/bin/check-tools.sh`](../../bin/check-tools.sh) | Verify every expected tools are present and runnable. |
| [`/bin/install-sanitizer-libs.sh`](../../bin/install-sanitizer-libs.sh) | Install `libasan`/`libtsan`/`libubsan` runtimes on the supported base images. |

View File

@@ -40,6 +40,29 @@ compile hello
compile panic
compile overflow "-C overflow-checks=on"
function compile_proc_macro() {
local proj="${src_dir}/../proc_macro"
echo "=== Building proc-macro workspace (cargo) ==="
cargo build --manifest-path "${proj}/Cargo.toml" --offline
local built="${proj}/target/debug/test_macro"
if [ ! -f "${built}" ]; then
echo "ERROR: built test_macro binary not found at ${built}" >&2
exit 1
fi
local binary="${dst_dir}/proc_macro"
cp "${built}" "${binary}"
echo "=== Patching ${binary} to use ${loader} as PT_INTERP ==="
patchelf --set-interpreter "${loader}" --remove-rpath "${binary}"
rm -rf "${proj}/target"
}
compile_proc_macro
echo "=== All binaries compiled ==="
ls -la "${dst_dir}"

View File

@@ -0,0 +1,14 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "echo_macro"
version = "0.0.0"
[[package]]
name = "test_macro"
version = "0.0.0"
dependencies = [
"echo_macro",
]

View File

@@ -0,0 +1,3 @@
[workspace]
resolver = "2"
members = ["echo_macro", "test_macro"]

View File

@@ -0,0 +1,8 @@
[package]
name = "echo_macro"
version = "0.0.0"
edition = "2024"
publish = false
[lib]
proc-macro = true

View File

@@ -0,0 +1,6 @@
use proc_macro::TokenStream;
#[proc_macro]
pub fn define_echo(item: TokenStream) -> TokenStream {
format!("fn echo() -> u32 {{ {item} }}").parse().unwrap()
}

View File

@@ -0,0 +1,8 @@
[package]
name = "test_macro"
version = "0.0.0"
edition = "2024"
publish = false
[dependencies]
echo_macro = { path = "../echo_macro" }

View File

@@ -0,0 +1,9 @@
use echo_macro::define_echo;
define_echo!(42);
fn main() {
let a = echo();
println!("proc-macro answer = {a}");
assert_eq!(a, 42, "proc-macro expansion produced the wrong value");
}

View File

@@ -1,7 +1,7 @@
#!/bin/bash
# Run pre-compiled Rust binaries and confirm each emits its expected diagnostic.
# Binaries must already exist in <bins_dir> as <name> for name in
# {hello,panic,overflow}.
# {hello,panic,overflow,proc_macro}.
set -eo pipefail
@@ -54,12 +54,13 @@ declare -A expect=(
[hello]="Hello from main thread"
[panic]="explicit panic from test"
[overflow]="attempt to add with overflow"
[proc_macro]="proc-macro answer = 42"
)
for name in hello panic overflow; do
for name in hello panic overflow proc_macro; do
binary="${bins_dir}/${name}"
if [ "${name}" = "hello" ]; then
if [ "${name}" = "hello" ] || [ "${name}" = "proc_macro" ]; then
expected_rc=0
else
expected_rc=nonzero

View File

@@ -15,6 +15,66 @@ let
runClangTidy = pkgs.writeShellScriptBin "run-clang-tidy" ''
exec ${pkgs.python3}/bin/python3 ${llvmPackages.clang-unwrapped}/bin/run-clang-tidy "$@"
'';
rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml;
# Nix wraps its toolchain so that binaries are exposed only under unsuffixed
# names (gcc, g++, clang-tidy, ...). Several tools probe for a
# version-suffixed name first and fall back to a system binary on the PATH
# when Nix doesn't provide it:
# - Conan's Boost recipe looks up `g++-<major>` before plain `g++`.
# - bin/pre-commit/clang_tidy_check.py looks up `run-clang-tidy-<v>` and
# `clang-apply-replacements-<v>` before the unsuffixed names.
# On a host that also has the matching system binary (e.g. Ubuntu's
# `/usr/bin/g++-15` or `clang-tidy-22`) the probe escapes Nix and mixes a
# system tool into the Nix environment. Generate version-suffixed symlinks
# next to a package's tools so those probes resolve to the Nix ones.
#
# Compiler links must point at whichever compiler is active in a given
# environment (the plain stdenv compiler in the dev shell, the custom-glibc
# wrappers in ci-env.nix), so those callers pass their own `package`; the
# clang tooling is environment-independent and is linked in commonPackages.
mkVersionedToolLinks =
{
name,
package,
version,
tools,
}:
pkgs.linkFarm "${name}-${toString version}-versioned-links" (
map (tool: {
name = "bin/${tool}-${toString version}";
path = "${package}/bin/${tool}";
}) tools
);
# The cc-wrapper doesn't re-export gcov, but coverage tooling (gcovr) needs a
# gcov that exactly matches the compiler. Surface it from a gcc `cc` output.
mkGcov =
{ name, cc }:
pkgs.linkFarm "gcov-${name}" [
{
name = "bin/gcov";
path = "${cc}/bin/gcov";
}
];
clangToolLinks = mkVersionedToolLinks {
name = "clang-tools";
package = clangTools;
version = llvmVersion;
tools = [
"clang-tidy"
"clang-apply-replacements"
"clang-format"
];
};
runClangTidyLink = mkVersionedToolLinks {
name = "run-clang-tidy";
package = runClangTidy;
version = llvmVersion;
tools = [ "run-clang-tidy" ];
};
in
{
inherit
@@ -22,9 +82,13 @@ in
llvmVersion
gccPackage
llvmPackages
mkVersionedToolLinks
mkGcov
;
commonPackages = with pkgs; [
clangToolLinks
runClangTidyLink
ccache
clangbuildanalyzer
clangTools
@@ -63,14 +127,10 @@ in
vim
zip
# Rust packages
cargo
cargo-audit
cargo-llvm-cov
cargo-nextest
clippy
corrosion
rust-analyzer
rustc
rustfmt
rustToolchain
];
}

View File

@@ -1,4 +1,8 @@
{ nixpkgs, nixpkgs-custom-glibc }:
{
nixpkgs,
nixpkgs-custom-glibc,
rust-overlay,
}:
function:
nixpkgs.lib.genAttrs
[
@@ -10,7 +14,12 @@ nixpkgs.lib.genAttrs
(
system:
function {
pkgs = import nixpkgs { inherit system; };
# rust-overlay adds `pkgs.rust-bin`, from which we build the pinned Rust
# toolchain (see packages.nix). Consumed by both the CI image and dev shell.
pkgs = import nixpkgs {
inherit system;
overlays = [ (import rust-overlay) ];
};
# glibc 2.31 — matches the system libc on Ubuntu 20.04 LTS. Sourced
# from the nixpkgs snapshot pinned via the `nixpkgs-custom-glibc`
# flake input, so the build uses the compiler from that snapshot

4
rust-toolchain.toml Normal file
View File

@@ -0,0 +1,4 @@
[toolchain]
channel = "1.95"
components = ["rustfmt", "clippy", "rust-analyzer", "llvm-tools-preview"]
profile = "minimal"

View File

@@ -0,0 +1,22 @@
include(XrplAddBenchmark)
# Benchmark requirements.
find_package(benchmark REQUIRED)
# Custom target for all benchmarks defined in this file.
add_custom_target(xrpl.benchmarks)
# Common library dependencies for every benchmark module. `benchmark_main`
# supplies a `main()` that parses the standard Google Benchmark CLI flags
# (`--benchmark_filter`, `--benchmark_format`, ...), so no per-module main.cpp
# is needed.
add_library(xrpl.imports.bench INTERFACE)
target_link_libraries(
xrpl.imports.bench
INTERFACE benchmark::benchmark_main xrpl.libxrpl
)
# One benchmark executable for each module.
xrpl_add_benchmark(nodestore)
target_link_libraries(xrpl.bench.nodestore PRIVATE xrpl.imports.bench)
add_dependencies(xrpl.benchmarks xrpl.bench.nodestore)

View File

@@ -0,0 +1,329 @@
#include <xrpl/nodestore/Backend.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Types.h>
#include <benchmark/benchmark.h>
#include <benchmarks/libxrpl/nodestore/NodeStoreBench.h>
#include <array>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace xrpl::NodeStore {
namespace {
constexpr std::size_t kPoolSizes[] = {1000, 10000, 100000};
constexpr int kThreadCounts[] = {1, 4, 8};
constexpr std::size_t kBatchSize = 256;
constexpr std::string_view kNamePrefix = "BM_Backend_";
constexpr std::string_view kNameSeparator = "/";
struct RunState
{
std::unique_ptr<BackendHarness> harness;
Batch present; // prefix-1 objects, eligible to be stored
Batch recent; // prefix-1 objects in the "future" key space
std::vector<uint256> missing; // prefix-2 keys that are never stored
std::vector<std::size_t> shuffle; // [0, poolSize) permutation for random-like access
std::size_t avgPayload = 0; // mean getData().size() over `present`
void
release()
{
harness.reset();
Batch{}.swap(present);
Batch{}.swap(recent);
std::vector<uint256>{}.swap(missing);
std::vector<std::size_t>{}.swap(shuffle);
}
};
struct SetupContext
{
RunState& rs;
Backend& backend;
std::size_t poolSize;
};
struct IterateContext
{
RunState& rs;
Backend& backend;
std::size_t index;
std::size_t poolSize;
};
struct Workload
{
std::string_view name;
std::function<void(SetupContext const&)> setup;
std::function<void(IterateContext const&)> iterate;
bool reportBytes = false; // SetBytesProcessed from rs.avgPayload
bool clobber = true; // ClobberMemory after the loop (false for pure stores)
bool pinToPool = false; // pin iterations to one pool sweep instead of autotuning
};
// One store() per iteration. Iterations are pinned to one pool sweep (per
// thread) so the index never wraps past the pool - otherwise NuDB::doInsert
// swallows key_exists and the workload degenerates into duplicate-detection
// no-ops.
Workload const kInsert{
.name = "Insert",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.avgPayload = averagePayload(ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
backend.store(rs.present[index % poolSize]);
},
.reportBytes = true,
.clobber = false,
.pinToPool = true,
};
// One fetch() of a present key (a hit) per iteration.
Workload const kFetch{
.name = "Fetch",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.avgPayload = averagePayload(ctx.rs.present);
prepopulate(ctx.backend, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
std::shared_ptr<NodeObject> result;
backend.fetch(rs.present[index % poolSize]->getHash(), &result);
benchmark::DoNotOptimize(result);
},
.reportBytes = true,
};
// One fetch() of a never-stored key (a miss); the backend is left empty.
Workload const kMissing{
.name = "Missing",
.setup = [](SetupContext const& ctx) { ctx.rs.missing = makeMissingKeys(ctx.poolSize); },
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
std::shared_ptr<NodeObject> result;
backend.fetch(rs.missing[index % poolSize], &result);
benchmark::DoNotOptimize(result);
},
};
// 80% hits / 20% misses. The fetch index comes from a shuffle table so access
// is random-like without per-iteration RNG cost; sequential `index % poolSize`
// would be artificially cache-friendly to RocksDB's block cache.
Workload const kMixed{
.name = "Mixed",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.missing = makeMissingKeys(ctx.poolSize);
ctx.rs.shuffle = makeShuffle(ctx.poolSize, /*seed=*/1);
prepopulate(ctx.backend, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
std::shared_ptr<NodeObject> result;
auto const pick = rs.shuffle[index % poolSize];
if (index % 5 == 0)
{
backend.fetch(rs.missing[pick], &result);
}
else
{
backend.fetch(rs.present[pick]->getHash(), &result);
}
benchmark::DoNotOptimize(result);
},
};
// An xrpld-like cycle: a hit, a maybe-miss recent fetch, and a store. The
// recent fetch uses the shuffle table (not `slot`) so it doesn't fetch the item
// it's about to store this iteration - which would give an all-miss-then-hit
// step instead of a smooth ramp. The store walks sequentially so each recent
// object is stored once.
Workload const kWork{
.name = "Work",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.recent = makePool(1, ctx.poolSize, ctx.poolSize);
ctx.rs.shuffle = makeShuffle(ctx.poolSize, /*seed=*/2);
prepopulate(ctx.backend, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, backend, index, poolSize] = ctx;
auto const slot = index % poolSize;
auto const pick = rs.shuffle[slot];
std::shared_ptr<NodeObject> historical;
backend.fetch(rs.present[pick]->getHash(), &historical);
benchmark::DoNotOptimize(historical);
std::shared_ptr<NodeObject> recent;
backend.fetch(rs.recent[pick]->getHash(), &recent);
benchmark::DoNotOptimize(recent);
backend.store(rs.recent[slot]);
},
.clobber = true,
.pinToPool = true,
};
auto
makeRunner(Workload w, std::string cfg, std::shared_ptr<RunState> rs)
{
return [w = std::move(w), cfg = std::move(cfg), rs = std::move(rs)](benchmark::State& state) {
auto const poolSize = static_cast<std::size_t>(state.range(0));
if (state.thread_index() == 0)
{
rs->harness = std::make_unique<BackendHarness>(cfg);
w.setup(
SetupContext{.rs = *rs, .backend = *rs->harness->backend, .poolSize = poolSize});
}
std::size_t index = state.thread_index();
for (auto _ : state)
{
w.iterate(
IterateContext{
.rs = *rs,
.backend = *rs->harness->backend,
.index = index,
.poolSize = poolSize});
index += state.threads();
}
if (w.clobber)
benchmark::ClobberMemory();
state.SetItemsProcessed(state.iterations());
if (w.reportBytes)
state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations() * rs->avgPayload));
if (state.thread_index() == 0)
rs->release();
};
}
// Register workload `w` against backend `bc`, choosing the registration shape
// from `w.pinToPool`.
void
registerWorkload(BackendConfig const& bc, Workload const& w)
{
std::string const cfg = bc.config;
std::string name{kNamePrefix};
name += w.name;
name += kNameSeparator;
name += bc.name;
if (!w.pinToPool)
{
auto rs = std::make_shared<RunState>();
auto* b = benchmark::RegisterBenchmark(name, makeRunner(w, cfg, rs));
b->RangeMultiplier(10)->Range(kPoolSizes[0], kPoolSizes[std::size(kPoolSizes) - 1]);
b->Threads(1)->Threads(4)->Threads(8)->UseRealTime();
return;
}
for (auto const poolSize : kPoolSizes)
{
for (auto const threads : kThreadCounts)
{
if (poolSize % static_cast<std::size_t>(threads) != 0)
continue;
auto rs = std::make_shared<RunState>();
benchmark::RegisterBenchmark(name, makeRunner(w, cfg, rs))
->Arg(poolSize)
->Iterations(poolSize / static_cast<std::size_t>(threads))
->Threads(threads)
->UseRealTime();
}
}
}
// One storeBatch() of kBatchSize objects per iteration. Single-threaded:
// Backend::storeBatch must not run concurrently with itself or store().
// Iterations are pinned to the batch count so the index never wraps into
// key_exists no-ops. Kept separate from Workload: batch slicing and the
// per-batch item/byte accounting don't fit the thread-axis mold.
void
registerStoreBatch(BackendConfig const& bc)
{
std::string const cfg = bc.config;
std::string name{kNamePrefix};
name += "StoreBatch";
name += kNameSeparator;
name += bc.name;
for (auto const poolSize : kPoolSizes)
{
auto const numBatches = poolSize / kBatchSize;
if (numBatches == 0)
continue;
auto rs = std::make_shared<RunState>();
benchmark::RegisterBenchmark(
name,
[rs, cfg](benchmark::State& state) {
auto const poolSize = static_cast<std::size_t>(state.range(0));
rs->harness = std::make_unique<BackendHarness>(cfg);
rs->present = makePool(1, poolSize);
rs->avgPayload = averagePayload(rs->present);
std::vector<Batch> const batches = sliceBatches(rs->present, kBatchSize);
if (batches.empty())
{
state.SkipWithError("pool smaller than one batch");
return;
}
std::size_t index = 0;
for (auto _ : state)
{
rs->harness->backend->storeBatch(batches[index % batches.size()]);
++index;
}
state.SetItemsProcessed(static_cast<std::int64_t>(state.iterations() * kBatchSize));
state.SetBytesProcessed(
static_cast<std::int64_t>(state.iterations() * kBatchSize * rs->avgPayload));
rs->release();
})
->Arg(poolSize)
->Iterations(numBatches);
}
}
[[maybe_unused]] bool const kRegistered = [] {
auto const workloads = std::to_array({&kInsert, &kFetch, &kMissing, &kMixed, &kWork});
for (auto const& bc : backendConfigs())
{
for (auto const* w : workloads)
registerWorkload(bc, *w);
registerStoreBatch(bc);
}
return true;
}();
} // namespace
} // namespace xrpl::NodeStore

View File

@@ -0,0 +1,243 @@
#include <xrpl/nodestore/Database.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Types.h>
#include <benchmark/benchmark.h>
#include <benchmarks/libxrpl/nodestore/NodeStoreBench.h>
#include <array>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace xrpl::NodeStore {
namespace {
// Number of distinct objects pre-generated per run.
constexpr std::size_t kDefaultPoolSize = 100000;
// Async read threads the Database spawns. Unused by the synchronous fetch path
// these benchmarks take; kept fixed so runs are comparable.
constexpr int kReadThreads = 4;
constexpr std::string_view kNamePrefix = "BM_Database_";
constexpr std::string_view kNameSeparator = "/";
struct RunState
{
std::unique_ptr<DatabaseHarness> harness;
Batch present; // prefix-1 objects, eligible to be stored
Batch recent; // prefix-1 objects in the "future" key space
std::vector<uint256> missing; // prefix-2 keys that are never stored
std::vector<std::size_t> shuffle; // [0, poolSize) permutation for random-like access
std::size_t avgPayload = 0; // mean getData().size() over `present`
};
struct SetupContext
{
RunState& rs;
Database& db;
std::size_t poolSize;
};
struct IterateContext
{
RunState& rs;
Database& db;
std::uint32_t seq;
std::size_t index;
std::size_t poolSize;
};
struct Workload
{
std::string_view name;
std::function<void(SetupContext const&)> setup;
std::function<void(IterateContext const&)> iterate;
bool reportBytes = false;
bool pinIterations = false;
};
void
prepopulate(Database& db, Batch const& objects)
{
auto const seq = db.earliestLedgerSeq();
for (auto const& obj : objects)
{
Blob data(obj->getData());
db.store(obj->getType(), std::move(data), obj->getHash(), seq);
}
db.sync();
}
// One store() per iteration; a fresh Blob copy is handed over each time.
Workload const kStore{
.name = "Store",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.avgPayload = averagePayload(ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto const& obj = rs.present[index % poolSize];
Blob data(obj->getData());
db.store(obj->getType(), std::move(data), obj->getHash(), seq);
},
.reportBytes = true,
.pinIterations = true,
};
// One fetchNodeObject() of a stored key (a hit) per iteration.
Workload const kFetch{
.name = "Fetch",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.avgPayload = averagePayload(ctx.rs.present);
prepopulate(ctx.db, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto obj = db.fetchNodeObject(rs.present[index % poolSize]->getHash(), seq);
benchmark::DoNotOptimize(obj);
},
.reportBytes = true,
};
// One fetchNodeObject() of a never-stored key (a miss) per iteration.
Workload const kMissing{
.name = "Missing",
.setup = [](SetupContext const& ctx) { ctx.rs.missing = makeMissingKeys(ctx.poolSize); },
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto obj = db.fetchNodeObject(rs.missing[index % poolSize], seq);
benchmark::DoNotOptimize(obj);
},
};
// 80% hits / 20% misses. The fetch index comes from a shuffle table so access
// is random-like without per-iteration RNG cost; sequential `index % poolSize`
// would be artificially cache-friendly.
Workload const kMixed{
.name = "Mixed",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.missing = makeMissingKeys(ctx.poolSize);
ctx.rs.shuffle = makeShuffle(ctx.poolSize, /*seed=*/1);
prepopulate(ctx.db, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto const pick = rs.shuffle[index % poolSize];
std::shared_ptr<NodeObject> obj;
if (index % 5 == 0)
{
obj = db.fetchNodeObject(rs.missing[pick], seq);
}
else
{
obj = db.fetchNodeObject(rs.present[pick]->getHash(), seq);
}
benchmark::DoNotOptimize(obj);
},
};
// An xrpld-like cycle: a hit, a maybe-miss recent fetch, and a store. The
// recent fetch uses the shuffle table (not `slot`) so it doesn't fetch the item
// it's about to store this iteration - which would give an all-miss-then-hit
// step instead of a smooth ramp. The store walks sequentially so each recent
// object is stored once.
Workload const kWork{
.name = "Work",
.setup =
[](SetupContext const& ctx) {
ctx.rs.present = makePool(1, ctx.poolSize);
ctx.rs.recent = makePool(1, ctx.poolSize, ctx.poolSize);
ctx.rs.shuffle = makeShuffle(ctx.poolSize, /*seed=*/2);
prepopulate(ctx.db, ctx.rs.present);
},
.iterate =
[](IterateContext const& ctx) {
auto& [rs, db, seq, index, poolSize] = ctx;
auto const slot = index % poolSize;
auto const pick = rs.shuffle[slot];
auto historical = db.fetchNodeObject(rs.present[pick]->getHash(), seq);
benchmark::DoNotOptimize(historical);
auto recent = db.fetchNodeObject(rs.recent[pick]->getHash(), seq);
benchmark::DoNotOptimize(recent);
auto const& obj = rs.recent[slot];
Blob data(obj->getData());
db.store(obj->getType(), std::move(data), obj->getHash(), seq);
},
.pinIterations = true,
};
void
registerWorkload(BackendConfig const& bc, Workload const& w)
{
auto rs = std::make_shared<RunState>();
std::string const cfg = bc.config;
std::string name{kNamePrefix};
name += w.name;
name += kNameSeparator;
name += bc.name;
auto* b = benchmark::RegisterBenchmark(name, [rs, cfg, w](benchmark::State& state) {
auto const poolSize = static_cast<std::size_t>(state.range(0));
rs->harness = std::make_unique<DatabaseHarness>(cfg, kReadThreads);
auto& db = *rs->harness->db;
w.setup(SetupContext{.rs = *rs, .db = db, .poolSize = poolSize});
auto const seq = db.earliestLedgerSeq();
std::size_t index = 0;
for (auto _ : state)
{
w.iterate(
IterateContext{
.rs = *rs, .db = db, .seq = seq, .index = index, .poolSize = poolSize});
++index;
}
benchmark::ClobberMemory();
state.SetItemsProcessed(state.iterations());
if (w.reportBytes)
{
state.SetBytesProcessed(static_cast<std::int64_t>(state.iterations() * rs->avgPayload));
}
rs->harness.reset();
});
b->Arg(kDefaultPoolSize);
if (w.pinIterations)
b->Iterations(kDefaultPoolSize);
}
[[maybe_unused]] bool const kRegistered = [] {
auto const workloads = std::to_array({&kStore, &kFetch, &kMissing, &kMixed, &kWork});
for (auto const& bc : backendConfigs())
{
for (auto const* w : workloads)
registerWorkload(bc, *w);
}
return true;
}();
} // namespace
} // namespace xrpl::NodeStore

View File

@@ -0,0 +1,318 @@
#pragma once
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/ByteUtilities.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/safe_cast.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/temp_dir.h>
#include <xrpl/beast/xor_shift_engine.h>
#include <xrpl/config/BasicConfig.h>
#include <xrpl/nodestore/Backend.h>
#include <xrpl/nodestore/Database.h>
#include <xrpl/nodestore/DummyScheduler.h>
#include <xrpl/nodestore/Manager.h>
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Scheduler.h>
#include <xrpl/nodestore/Types.h>
#include <boost/algorithm/string/classification.hpp>
#include <boost/algorithm/string/split.hpp>
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <memory>
#include <numeric>
#include <random>
#include <string>
#include <utility>
#include <vector>
// Shared helpers for the NodeStore benchmarks.
//
namespace xrpl::NodeStore {
// Fill `bytes` of memory at `buffer` with random bits drawn from `g`.
template <class Generator>
inline void
rngcpy(void* buffer, std::size_t bytes, Generator& g)
{
using result_type = typename Generator::result_type;
while (bytes >= sizeof(result_type))
{
auto const v = g();
std::memcpy(buffer, &v, sizeof(v));
buffer = reinterpret_cast<std::uint8_t*>(buffer) + sizeof(v);
bytes -= sizeof(v);
}
if (bytes > 0)
{
auto const v = g();
std::memcpy(buffer, &v, bytes);
}
}
/**
* @brief Deterministic generator of a reproducible sequence of random NodeObjects.
*
* Indexing is stable: `obj(n)` and `key(n)` always return the same value for a
* given `n`, regardless of call order, because the engine is reseeded from `n`
* on every call.
*
* Using different prefixes guarantees the two key spaces are disjoint for the fetch-miss
* workloads.
*/
class Sequence
{
private:
static constexpr auto kMinSize = 250;
static constexpr auto kMaxSize = 1250;
beast::xor_shift_engine gen_;
std::uint8_t prefix_;
std::discrete_distribution<std::uint32_t> dType_;
std::uniform_int_distribution<std::uint32_t> dSize_;
public:
explicit Sequence(std::uint8_t prefix)
: prefix_(prefix)
// uniform distribution over hotLEDGER - hotTRANSACTION_NODE
// but exclude hotTRANSACTION = 2 (removed)
, dType_({1, 1, 0, 1, 1})
, dSize_(kMinSize, kMaxSize)
{
}
// Returns the n-th key. Used to generate keys that are never stored.
// The layout mirrors obj()'s: prefix at byte 0, RNG over the rest, so the
// two key spaces stay disjoint by construction (not by coincidence).
uint256
key(std::size_t n)
{
gen_.seed(n + 1);
uint256 result;
auto const data = static_cast<std::uint8_t*>(&*result.begin());
*data = prefix_;
rngcpy(data + 1, result.size() - 1, gen_);
return result;
}
// Returns the n-th complete NodeObject.
std::shared_ptr<NodeObject>
obj(std::size_t n)
{
gen_.seed(n + 1);
uint256 key;
auto const data = static_cast<std::uint8_t*>(&*key.begin());
*data = prefix_;
rngcpy(data + 1, key.size() - 1, gen_);
Blob value(dSize_(gen_));
rngcpy(&value[0], value.size(), gen_);
return NodeObject::createObject(
safeCast<NodeObjectType>(dType_(gen_)), std::move(value), key);
}
// Fills `b` with `size` consecutive NodeObjects starting at index `n`.
void
batch(std::size_t n, Batch& b, std::size_t size)
{
b.clear();
b.reserve(size);
while ((size--) != 0u)
b.push_back(obj(n++));
}
};
// Parse a comma-separated "key=value,key=value" string into a config Section.
inline Section
parseConfig(std::string const& s)
{
Section section;
std::vector<std::string> values;
boost::split(values, s, boost::algorithm::is_any_of(","));
section.append(values);
return section;
}
// Pre-generate `count` distinct objects from key space `prefix`, starting at
// sequence index `start`.
inline Batch
makePool(std::uint8_t prefix, std::size_t count, std::size_t start = 0)
{
Sequence seq(prefix);
Batch pool;
pool.reserve(count);
for (std::size_t i = 0; i < count; ++i)
pool.push_back(seq.obj(start + i));
return pool;
}
// Pre-generate `count` keys disjoint from every `makePool(...)` object, for
// measuring fetches that miss.
inline std::vector<uint256>
makeMissingKeys(std::size_t count)
{
Sequence seq(2);
std::vector<uint256> keys;
keys.reserve(count);
for (std::size_t i = 0; i < count; ++i)
keys.push_back(seq.key(i));
return keys;
}
// Mean payload size across a pool, used for SetBytesProcessed throughput.
inline std::size_t
averagePayload(Batch const& pool)
{
if (pool.empty())
return 0;
std::size_t total = 0;
for (auto const& obj : pool)
total += obj->getData().size();
return total / pool.size();
}
// Store every object and flush, so a following fetch exercises the real read
// path rather than an in-memory write buffer.
//
// We chunk the write at kBatchWriteLimitSize because Types.h documents that as
// the maximum allowed batch size. NuDB happens to tolerate larger batches
// today, but the benchmark should not rely on that.
//
// sync() is a no-op for both NuDB and RocksDB at the moment (NuDB has a small
// internal burst buffer that the timed loop will warm up). That is a contract
// hint, not a guarantee; if either backend ever grows a real flush we get it
// here for free.
inline void
prepopulate(Backend& backend, Batch const& objects)
{
for (std::size_t i = 0; i < objects.size(); i += kBatchWriteLimitSize)
{
auto const end = std::min(i + kBatchWriteLimitSize, objects.size());
backend.storeBatch(Batch(objects.begin() + i, objects.begin() + end));
}
backend.sync();
}
// A deterministic permutation of [0, size). Lets the timed loop visit the
// pre-generated pool in a random-like order with zero RNG cost per iteration -
// the Timing_test workloads it replaces used uniform_int_distribution per
// fetch, and a shuffle table reproduces that access pattern without paying for
// the distribution inside the timed region.
inline std::vector<std::size_t>
makeShuffle(std::size_t size, std::uint64_t seed)
{
std::vector<std::size_t> v(size);
std::iota(v.begin(), v.end(), std::size_t{0});
beast::xor_shift_engine gen(seed);
std::shuffle(v.begin(), v.end(), gen);
return v;
}
// Partition a pool into fixed-size batches. Any trailing remainder shorter than
// `batchSize` is dropped, so every returned batch has exactly `batchSize`.
inline std::vector<Batch>
sliceBatches(Batch const& pool, std::size_t batchSize)
{
std::vector<Batch> batches;
if (batchSize == 0)
return batches;
batches.reserve(pool.size() / batchSize);
for (std::size_t i = 0; i + batchSize <= pool.size(); i += batchSize)
batches.emplace_back(pool.begin() + i, pool.begin() + i + batchSize);
return batches;
}
/**
* @brief RAII owner of a NodeStore Backend opened on a private temporary directory.
*
* Member declaration order matters: `tempDir` is declared first so it is
* destroyed last, after the backend has closed and released its files.
*/
struct BackendHarness
{
beast::TempDir tempDir;
DummyScheduler scheduler;
beast::Journal journal{beast::Journal::getNullSink()};
std::unique_ptr<Backend> backend;
explicit BackendHarness(std::string const& configString)
{
Section config = parseConfig(configString);
// A private, unique path per harness, so concurrent or repeated runs
// never share on-disk state.
config.set("path", tempDir.path());
backend =
Manager::instance().makeBackend(config, megabytes(std::size_t{4}), scheduler, journal);
backend->setDeletePath();
backend->open();
}
~BackendHarness()
{
if (backend)
backend->close();
}
};
/**
* RAII owner of a NodeStore Database - the application-facing wrapper around a
* Backend, which adds fetch/store accounting and the async read-thread pool.
*/
struct DatabaseHarness
{
beast::TempDir tempDir;
DummyScheduler scheduler;
beast::Journal journal{beast::Journal::getNullSink()};
std::unique_ptr<Database> db;
DatabaseHarness(std::string const& configString, int readThreads)
{
Section config = parseConfig(configString);
config.set("path", tempDir.path());
db = Manager::instance().makeDatabase(
megabytes(std::size_t{4}), scheduler, readThreads, config, journal);
}
~DatabaseHarness()
{
if (db)
db->stop();
}
};
// A NodeStore backend to benchmark, named for the --benchmark_filter CLI flag.
struct BackendConfig
{
char const* name; // short label, e.g. "nudb"
char const* config; // parseConfig() string, e.g. "type=nudb"
};
// The backends every workload is registered against.
//
// The in-memory backend is intentionally excluded. It keeps its table in a
// process-global map keyed by path, with no removal API, so building a fresh
// backend per run - as a microbenchmark must - would leak the whole dataset on
// every run. Timing_test, the suite this benchmark replaces, excluded it for
// the same reason. NuDB and RocksDB are the production backends worth timing.
//
// RocksDB is included only when it was compiled in (xrpl.libxrpl carries
// XRPL_ROCKSDB_AVAILABLE transitively).
inline std::vector<BackendConfig> const&
backendConfigs()
{
static std::vector<BackendConfig> const kConfigs = {
{.name = "nudb", .config = "type=nudb"},
#if XRPL_ROCKSDB_AVAILABLE
{.name = "rocksdb",
.config = "type=rocksdb,open_files=2000,filter_bits=12,cache_mb=256,"
"file_size_mb=8,file_size_mult=2"},
#endif
};
return kConfigs;
}
} // namespace xrpl::NodeStore

View File

@@ -1,10 +1,9 @@
#include <xrpld/consensus/Consensus.h>
#include <xrpld/consensus/ConsensusParms.h>
#include <xrpld/consensus/ConsensusTypes.h>
#include <xrpl/consensus/Consensus.h>
#include <xrpl/basics/Log.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/consensus/ConsensusParms.h>
#include <xrpl/consensus/ConsensusTypes.h>
#include <algorithm>
#include <chrono>

View File

@@ -26,6 +26,14 @@ namespace xrpl {
namespace directory {
struct Gap
{
uint64_t const page;
SLE::pointer node;
uint64_t const nextPage;
SLE::pointer next;
};
std::uint64_t
createRoot(
ApplyView& view,
@@ -126,7 +134,9 @@ insertPage(
if (page == 0)
return std::nullopt;
if (!view.rules().enabled(fixDirectoryLimit) && page >= kDirNodeMaxPages) // Old pages limit
{
return std::nullopt;
}
// We are about to create a new node; we'll link it to
// the chain first:
@@ -147,12 +157,8 @@ insertPage(
// Save some space by not specifying the value 0 since it's the default.
if (page != 1)
node->setFieldU64(sfIndexPrevious, page - 1);
XRPL_ASSERT_PARTS(!nextPage, "xrpl::directory::insertPage", "nextPage has default value");
/* Reserved for future use when directory pages may be inserted in
* between two other pages instead of only at the end of the chain.
if (nextPage)
node->setFieldU64(sfIndexNext, nextPage);
*/
describe(node);
view.insert(node);
@@ -168,7 +174,7 @@ ApplyView::dirAdd(
uint256 const& key,
std::function<void(SLE::ref)> const& describe)
{
auto root = peek(directory);
auto const root = peek(directory);
if (!root)
{
@@ -178,6 +184,43 @@ ApplyView::dirAdd(
auto [page, node, indexes] = directory::findPreviousPage(*this, directory, root);
if (rules().enabled(featureDefragDirectories))
{
// If there are more nodes than just the root, and there's no space in
// the last one, walk backwards to find one with space, or to find one
// missing.
std::optional<directory::Gap> gapPages;
while (page && indexes.size() >= kDIR_NODE_MAX_PAGES)
{
// Find a page with space, or a gap in pages.
auto [prevPage, prevNode, prevIndexes] =
directory::findPreviousPage(*this, directory, node);
if (!gapPages && prevPage != page - 1)
gapPages.emplace(prevPage, prevNode, page, node);
page = prevPage;
node = prevNode;
indexes = prevIndexes;
}
// We looped through all the pages back to the root.
if (!page)
{
// If we found a gap, use it.
if (gapPages)
{
return directory::insertPage(
*this,
gapPages->page,
gapPages->node,
gapPages->nextPage,
gapPages->next,
key,
directory,
describe);
}
std::tie(page, node, indexes) = directory::findPreviousPage(*this, directory, root);
}
}
// If there's space, we use it:
if (indexes.size() < kDirNodeMaxEntries)
{

View File

@@ -312,6 +312,25 @@ computeInterestAndFeeParts(
return std::make_pair(interest - fee, fee);
}
/* Rounds a raw (unrounded) interest amount to the loan's scale, then splits
* the rounded amount into net interest (to the vault) and management fee (to
* the broker).
*
* This is the common "round then split" step shared by late payment, full
* payment, and overpayment interest calculations.
*/
std::pair<Number, Number>
roundAndSplitInterest(
Asset const& asset,
Number const& rawInterest,
TenthBips16 managementFeeRate,
std::int32_t loanScale,
Number::RoundingMode mode = Number::getround())
{
auto const interest = roundToAsset(asset, rawInterest, loanScale, mode);
return computeInterestAndFeeParts(asset, interest, managementFeeRate, loanScale);
}
/* Calculates penalty interest accrued on overdue payments.
* Returns 0 if payment is not late.
*
@@ -387,22 +406,18 @@ loanAccruedInterest(
*
* This is the core function that updates the Loan ledger object fields based on
* a computed payment.
* The function is templated to work with both direct Number/uint32_t values
* (for testing/simulation) and ValueProxy types (for actual ledger updates).
*/
template <class NumberProxy, class UInt32Proxy, class UInt32OptionalProxy>
LoanPaymentParts
doPayment(
ExtendedPaymentComponents const& payment,
NumberProxy& totalValueOutstandingProxy,
NumberProxy& principalOutstandingProxy,
NumberProxy& managementFeeOutstandingProxy,
UInt32Proxy& paymentRemainingProxy,
UInt32Proxy& prevPaymentDateProxy,
UInt32OptionalProxy& nextDueDateProxy,
std::uint32_t paymentInterval)
doPayment(ExtendedPaymentComponents const& payment, SLE::ref loan)
{
auto totalValueOutstandingProxy = loan->at(sfTotalValueOutstanding);
auto principalOutstandingProxy = loan->at(sfPrincipalOutstanding);
auto managementFeeOutstandingProxy = loan->at(sfManagementFeeOutstanding);
auto paymentRemainingProxy = loan->at(sfPaymentRemaining);
auto prevPaymentDateProxy = loan->at(sfPreviousPaymentDueDate);
auto nextDueDateProxy = loan->at(sfNextPaymentDueDate);
std::uint32_t const paymentInterval = loan->at(sfPaymentInterval);
XRPL_ASSERT_PARTS(nextDueDateProxy, "xrpl::detail::doPayment", "Next due date proxy set");
if (payment.specialCase == PaymentSpecialCase::Final)
@@ -470,16 +485,12 @@ doPayment(
// Principal can never exceed total value (principal is part of total value)
XRPL_ASSERT_PARTS(
// Use an explicit cast because the template parameter can be
// ValueProxy<Number> or Number
static_cast<Number>(principalOutstandingProxy) <=
static_cast<Number>(totalValueOutstandingProxy),
"xrpl::detail::doPayment",
"principal does not exceed total");
XRPL_ASSERT_PARTS(
// Use an explicit cast because the template parameter can be
// ValueProxy<Number> or Number
static_cast<Number>(managementFeeOutstandingProxy) >= beast::kZero,
"xrpl::detail::doPayment",
"fee outstanding stays valid");
@@ -717,22 +728,23 @@ tryOverpayment(
* overpayment would leave the loan in an invalid state, we can reject it
* gracefully without corrupting the ledger data.
*/
template <class NumberProxy>
std::expected<LoanPaymentParts, TER>
doOverpayment(
Rules const& rules,
Asset const& asset,
std::int32_t loanScale,
ExtendedPaymentComponents const& overpaymentComponents,
NumberProxy& totalValueOutstandingProxy,
NumberProxy& principalOutstandingProxy,
NumberProxy& managementFeeOutstandingProxy,
NumberProxy& periodicPaymentProxy,
SLE::ref loan,
Number const& periodicRate,
std::uint32_t const paymentRemaining,
TenthBips16 const managementFeeRate,
beast::Journal j)
{
auto totalValueOutstandingProxy = loan->at(sfTotalValueOutstanding);
auto principalOutstandingProxy = loan->at(sfPrincipalOutstanding);
auto managementFeeOutstandingProxy = loan->at(sfManagementFeeOutstanding);
auto periodicPaymentProxy = loan->at(sfPeriodicPayment);
auto const paymentsRemaining = loan->at(sfPaymentRemaining);
auto const loanState = constructLoanState(
totalValueOutstandingProxy, principalOutstandingProxy, managementFeeOutstandingProxy);
auto const periodicPayment = periodicPaymentProxy;
@@ -744,7 +756,7 @@ doOverpayment(
<< ", interestPart: " << overpaymentComponents.trackedInterestPart()
<< ", untrackedInterest: " << overpaymentComponents.untrackedInterest
<< ", totalDue: " << overpaymentComponents.totalDue
<< ", payments remaining :" << paymentRemaining;
<< ", payments remaining :" << paymentsRemaining;
// Attempt to re-amortize the loan with the overpayment applied.
// This modifies the temporary copies, leaving the proxies unchanged.
@@ -756,7 +768,7 @@ doOverpayment(
loanState,
periodicPayment,
periodicRate,
paymentRemaining,
paymentsRemaining,
managementFeeRate,
j);
if (!ret)
@@ -864,16 +876,15 @@ std::expected<ExtendedPaymentComponents, TER>
computeLatePayment(
Asset const& asset,
ApplyView const& view,
Number const& principalOutstanding,
std::int32_t nextDueDate,
SLE::const_ref loan,
ExtendedPaymentComponents const& periodic,
TenthBips32 lateInterestRate,
std::int32_t loanScale,
Number const& latePaymentFee,
STAmount const& amount,
TenthBips16 managementFeeRate,
beast::Journal j)
{
std::int32_t const nextDueDate = loan->at(sfNextPaymentDueDate);
std::int32_t const loanScale = loan->at(sfLoanScale);
// Check if the due date has passed. If not, reject the payment as
// being too soon
if (!hasExpired(view, nextDueDate))
@@ -881,15 +892,15 @@ computeLatePayment(
// Calculate the penalty interest based on how long the payment is overdue.
auto const latePaymentInterest = loanLatePaymentInterest(
principalOutstanding, lateInterestRate, view.parentCloseTime(), nextDueDate);
loan->at(sfPrincipalOutstanding),
TenthBips32{loan->at(sfLateInterestRate)},
view.parentCloseTime(),
nextDueDate);
// Round the late interest and split it between the vault (net interest)
// and the broker (management fee portion). This lambda ensures we
// round before splitting to maintain precision.
auto const [roundedLateInterest, roundedLateManagementFee] = [&]() {
auto const interest = roundToAsset(asset, latePaymentInterest, loanScale);
return computeInterestAndFeeParts(asset, interest, managementFeeRate, loanScale);
}();
// and the broker (management fee portion).
auto const [roundedLateInterest, roundedLateManagementFee] =
roundAndSplitInterest(asset, latePaymentInterest, managementFeeRate, loanScale);
XRPL_ASSERT(roundedLateInterest >= 0, "xrpl::detail::computeLatePayment : valid late interest");
XRPL_ASSERT_PARTS(
@@ -908,7 +919,7 @@ computeLatePayment(
// 1. Regular service fee (from periodic.untrackedManagementFee)
// 2. Late payment fee (fixed penalty)
// 3. Management fee portion of late interest
periodic.untrackedManagementFee + latePaymentFee + roundedLateManagementFee,
periodic.untrackedManagementFee + loan->at(sfLatePaymentFee) + roundedLateManagementFee,
// Untracked interest includes:
// 1. Any untracked interest from the regular payment (usually 0)
@@ -958,22 +969,15 @@ std::expected<ExtendedPaymentComponents, TER>
computeFullPayment(
Asset const& asset,
ApplyView& view,
Number const& principalOutstanding,
Number const& managementFeeOutstanding,
Number const& periodicPayment,
std::uint32_t paymentRemaining,
std::uint32_t prevPaymentDate,
std::uint32_t const startDate,
std::uint32_t const paymentInterval,
TenthBips32 const closeInterestRate,
std::int32_t loanScale,
Number const& totalInterestOutstanding,
SLE::const_ref loan,
Number const& periodicRate,
Number const& closePaymentFee,
STAmount const& amount,
TenthBips16 managementFeeRate,
beast::Journal j)
{
std::uint32_t const paymentRemaining = loan->at(sfPaymentRemaining);
std::int32_t const loanScale = loan->at(sfLoanScale);
// Full payment must be made before the final scheduled payment.
if (paymentRemaining <= 1)
{
@@ -986,7 +990,7 @@ computeFullPayment(
// This theoretical (unrounded) value is used to compute interest and
// penalties accurately.
Number const theoreticalPrincipalOutstanding = loanPrincipalFromPeriodicPayment(
view.rules(), periodicPayment, periodicRate, paymentRemaining);
view.rules(), loan->at(sfPeriodicPayment), periodicRate, paymentRemaining);
// Full payment interest includes both accrued interest (time since last
// payment) and prepayment penalty (for closing early).
@@ -994,18 +998,21 @@ computeFullPayment(
theoreticalPrincipalOutstanding,
periodicRate,
view.parentCloseTime(),
paymentInterval,
prevPaymentDate,
startDate,
closeInterestRate);
loan->at(sfPaymentInterval),
loan->at(sfPreviousPaymentDueDate),
loan->at(sfStartDate),
TenthBips32{loan->at(sfCloseInterestRate)});
// Split the full payment interest into net interest (to vault) and
// management fee (to broker), applying proper rounding.
auto const [roundedFullInterest, roundedFullManagementFee] = [&]() {
auto const interest =
roundToAsset(asset, fullPaymentInterest, loanScale, Number::RoundingMode::Downward);
return computeInterestAndFeeParts(asset, interest, managementFeeRate, loanScale);
}();
// Split the full payment interest into net interest (to vault) and management fee (to broker),
// applying proper rounding.
auto const [roundedFullInterest, roundedFullManagementFee] = roundAndSplitInterest(
asset, fullPaymentInterest, managementFeeRate, loanScale, Number::RoundingMode::Downward);
LoanState const loanState = constructLoanState(loan);
Number const principalOutstanding = loanState.principalOutstanding;
Number const managementFeeOutstanding = loanState.managementFeeDue;
Number const totalInterestOutstanding = loanState.interestDue;
Number const closePaymentFee = roundToAsset(asset, loan->at(sfClosePaymentFee), loanScale);
ExtendedPaymentComponents const full{
PaymentComponents{
@@ -1046,8 +1053,7 @@ computeFullPayment(
"xrpl::detail::computeFullPayment",
"total due is rounded");
JLOG(j.trace()) << "computeFullPayment result: periodicPayment: " << periodicPayment
<< ", periodicRate: " << periodicRate
JLOG(j.trace()) << "computeFullPayment result: periodicRate: " << periodicRate
<< ", paymentRemaining: " << paymentRemaining
<< ", theoreticalPrincipalOutstanding: " << theoreticalPrincipalOutstanding
<< ", fullPaymentInterest: " << fullPaymentInterest
@@ -1298,6 +1304,34 @@ computePaymentComponents(
};
}
/* Thin overload of computePaymentComponents() that unwraps the tracked
* fields directly from the Loan ledger object. `periodicRate` is derived
* rather than stored, and `managementFeeRate` comes from the LoanBroker, not
* the Loan, so both remain explicit parameters. Kept separate from the
* value-based overload above, which is exercised directly by unit tests
* against simulated (non-ledger) loan states.
*/
PaymentComponents
computePaymentComponents(
Rules const& rules,
Asset const& asset,
SLE::ref loan,
Number const& periodicRate,
TenthBips16 managementFeeRate)
{
return computePaymentComponents(
rules,
asset,
loan->at(sfLoanScale),
loan->at(sfTotalValueOutstanding),
loan->at(sfPrincipalOutstanding),
loan->at(sfManagementFeeOutstanding),
loan->at(sfPeriodicPayment),
periodicRate,
loan->at(sfPaymentRemaining),
managementFeeRate);
}
/* Computes payment components for an overpayment scenario.
*
* An overpayment occurs when a borrower pays more than the scheduled periodic
@@ -1342,11 +1376,12 @@ computeOverpaymentComponents(
// This interest doesn't follow the normal amortization schedule - it's
// a one-time charge for paying early.
// Equation (20) and (21) from XLS-66 spec, Section A-2 Equation Glossary
auto const [roundedOverpaymentInterest, roundedOverpaymentManagementFee] = [&]() {
auto const interest =
roundToAsset(asset, tenthBipsOfValue(overpayment, overpaymentInterestRate), loanScale);
return detail::computeInterestAndFeeParts(asset, interest, managementFeeRate, loanScale);
}();
auto const [roundedOverpaymentInterest, roundedOverpaymentManagementFee] =
roundAndSplitInterest(
asset,
tenthBipsOfValue(overpayment, overpaymentInterestRate),
managementFeeRate,
loanScale);
auto const result = detail::ExtendedPaymentComponents{
// Build the payment components, after fees and penalty
@@ -1373,6 +1408,265 @@ computeOverpaymentComponents(
return result;
}
/* Derives the two rate values every make*Payment() helper needs: the
* broker's management fee rate, and the loan's periodic (per-payment-period)
* interest rate.
*/
std::pair<TenthBips16, Number>
loanRatesFor(SLE::const_ref loan, SLE::const_ref brokerSle)
{
TenthBips16 const managementFeeRate{brokerSle->at(sfManagementFeeRate)};
TenthBips32 const interestRate{loan->at(sfInterestRate)};
Number const periodicRate = loanPeriodicRate(interestRate, loan->at(sfPaymentInterval));
XRPL_ASSERT(interestRate == 0 || periodicRate > 0, "xrpl::detail::loanRatesFor : valid rate");
return {managementFeeRate, periodicRate};
}
/* Handles a full (early payoff) payment. Implements the "full payment"
* branch of the make_payment function from the XLS-66 spec, Section
* 3.2.4.4.
*/
std::expected<LoanPaymentParts, TER>
makeFullPayment(
Asset const& asset,
ApplyView& view,
SLE::ref loan,
SLE::const_ref brokerSle,
STAmount const& amount,
beast::Journal j)
{
auto const [managementFeeRate, periodicRate] = loanRatesFor(loan, brokerSle);
auto const fullPaymentComponents =
computeFullPayment(asset, view, loan, periodicRate, amount, managementFeeRate, j);
// computeFullPayment only ever fails with a genuine error TER (never
// tesSUCCESS), so there is no separate "no-op" outcome to handle here.
if (fullPaymentComponents.has_value())
return doPayment(*fullPaymentComponents, loan);
return std::unexpected(fullPaymentComponents.error());
}
/* Handles a late payment (past due date, with the late-payment flag set).
* Implements the "late payment" branch of the make_payment function from
* the XLS-66 spec, Section 3.2.4.4.
*/
std::expected<LoanPaymentParts, TER>
makeLatePayment(
Asset const& asset,
ApplyView const& view,
SLE::ref loan,
SLE::const_ref brokerSle,
STAmount const& amount,
beast::Journal j)
{
auto const [managementFeeRate, periodicRate] = loanRatesFor(loan, brokerSle);
Number const serviceFee = loan->at(sfLoanServiceFee);
ExtendedPaymentComponents const periodic{
computePaymentComponents(view.rules(), asset, loan, periodicRate, managementFeeRate),
serviceFee};
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::detail::makeLatePayment",
"regular payment valid principal");
auto const latePaymentComponents =
computeLatePayment(asset, view, loan, periodic, amount, managementFeeRate, j);
// computeLatePayment only ever fails with a genuine error TER (never
// tesSUCCESS), so there is no separate "no-op" outcome to handle here.
if (latePaymentComponents.has_value())
return doPayment(*latePaymentComponents, loan);
return std::unexpected(latePaymentComponents.error());
}
/* Handles regular scheduled payments, including an optional overpayment tail.
* Implements the "regular" and "overpayment" branches of the make_payment
* function from the XLS-66 spec, Section 3.2.4.4.
*/
std::expected<LoanPaymentParts, TER>
makeRegularPayment(
Asset const& asset,
ApplyView const& view,
SLE::ref loan,
SLE::const_ref brokerSle,
STAmount const& amount,
LoanPaymentType const paymentType,
beast::Journal j)
{
using namespace Lending;
XRPL_ASSERT_PARTS(
paymentType == LoanPaymentType::Regular || paymentType == LoanPaymentType::Overpayment,
"xrpl::detail::makeRegularPayment",
"regular payment type");
auto const [managementFeeRate, periodicRate] = loanRatesFor(loan, brokerSle);
std::int32_t const loanScale = loan->at(sfLoanScale);
Number const serviceFee = loan->at(sfLoanServiceFee);
ExtendedPaymentComponents periodic{
computePaymentComponents(view.rules(), asset, loan, periodicRate, managementFeeRate),
serviceFee};
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::detail::makeRegularPayment",
"regular payment valid principal");
// Keep a running total of the actual parts paid
LoanPaymentParts totalParts;
Number totalPaid = kNumZero;
std::size_t numPayments = 0;
// Cached here (rather than re-looking up loan->at(sfPaymentRemaining) at each use) since it's
// read multiple times below. It's a write-through proxy, so it still reflects doPayment's
// mutations each iteration.
auto paymentRemainingProxy = loan->at(sfPaymentRemaining);
while ((amount >= (totalPaid + periodic.totalDue)) && paymentRemainingProxy > 0 &&
numPayments < kLoanMaximumPaymentsPerTransaction)
{
// Try to make more payments
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::detail::makeRegularPayment",
"payment pays non-negative principal");
totalPaid += periodic.totalDue;
totalParts += doPayment(periodic, loan);
++numPayments;
XRPL_ASSERT_PARTS(
(periodic.specialCase == PaymentSpecialCase::Final) == (paymentRemainingProxy == 0),
"xrpl::detail::makeRegularPayment",
"final payment is the final payment");
// Don't compute the next payment if this was the last payment
if (periodic.specialCase == PaymentSpecialCase::Final)
break;
periodic = ExtendedPaymentComponents{
computePaymentComponents(view.rules(), asset, loan, periodicRate, managementFeeRate),
serviceFee};
}
if (numPayments == 0)
{
JLOG(j.warn()) << "Regular loan payment amount is insufficient. Due: " << periodic.totalDue
<< ", paid: " << amount;
return std::unexpected(tecINSUFFICIENT_PAYMENT);
}
XRPL_ASSERT_PARTS(
totalParts.principalPaid + totalParts.interestPaid + totalParts.feePaid == totalPaid,
"xrpl::detail::makeRegularPayment",
"payment parts add up");
XRPL_ASSERT_PARTS(
totalParts.valueChange == 0, "xrpl::detail::makeRegularPayment", "no value change");
// -------------------------------------------------------------
// overpayment handling
//
// If the "fixCleanup3_1_3" amendment is enabled, truncate "amount",
// at the loan scale. If the raw value is used, the overpayment
// amount could be meaningless dust. Trying to process such a small
// amount will, at best, waste time when all the result values round
// to zero. At worst, it can cause logical errors with tiny amounts
// of interest that don't add up correctly.
auto const roundedAmount = view.rules().enabled(fixCleanup3_1_3)
? roundToAsset(asset, amount, loanScale, Number::RoundingMode::TowardsZero)
: amount;
bool const overpaymentSupported =
paymentType == LoanPaymentType::Overpayment && loan->isFlag(lsfLoanOverpayment);
bool const overpaymentAllowed = //
paymentRemainingProxy > 0 && //
totalPaid < roundedAmount && //
numPayments < kLoanMaximumPaymentsPerTransaction;
if (overpaymentSupported && overpaymentAllowed)
{
TenthBips32 const overpaymentInterestRate{loan->at(sfOverpaymentInterestRate)};
TenthBips32 const overpaymentFeeRate{loan->at(sfOverpaymentFee)};
// It shouldn't be possible for the overpayment to be greater than
// totalValueOutstanding, because that would have been processed as
// another normal payment. But cap it just in case.
Number const overpaymentRaw =
std::min(roundedAmount - totalPaid, *loan->at(sfTotalValueOutstanding));
bool const fixEnabled = view.rules().enabled(fixCleanup3_2_0);
Number const overpayment = fixEnabled
? roundToAsset(asset, overpaymentRaw, loanScale, Number::RoundingMode::Downward)
: overpaymentRaw;
// Post-amendment, the rounded overpayment can be zero; pre-amendment
// it's always positive given the surrounding guards.
if (!fixEnabled || overpayment > 0)
{
ExtendedPaymentComponents const overpaymentComponents = computeOverpaymentComponents(
view.rules(),
asset,
loanScale,
overpayment,
overpaymentInterestRate,
overpaymentFeeRate,
managementFeeRate);
// Don't process an overpayment if the whole amount (or more!)
// gets eaten by fees and interest.
if (overpaymentComponents.trackedPrincipalDelta > 0)
{
XRPL_ASSERT_PARTS(
overpaymentComponents.untrackedInterest >= beast::kZero,
"xrpl::detail::makeRegularPayment",
"overpayment penalty did not reduce value of loan");
if (auto const overResult = doOverpayment(
view.rules(),
asset,
loanScale,
overpaymentComponents,
loan,
periodicRate,
managementFeeRate,
j))
{
totalParts += *overResult;
}
else if (overResult.error())
{
// error() will be the TER returned if a payment is not
// made. It will only evaluate to true if it's unsuccessful.
// Otherwise, tesSUCCESS means nothing was done, so
// continue.
return std::unexpected(overResult.error());
}
}
}
}
// Check the final results are rounded, to double-check that the
// intermediate steps were rounded.
XRPL_ASSERT(
isRounded(asset, totalParts.principalPaid, loanScale) &&
totalParts.principalPaid >= beast::kZero,
"xrpl::detail::makeRegularPayment : total principal paid is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.interestPaid, loanScale) &&
totalParts.interestPaid >= beast::kZero,
"xrpl::detail::makeRegularPayment : total interest paid is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.valueChange, loanScale),
"xrpl::detail::makeRegularPayment : loan value change is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.feePaid, loanScale) && totalParts.feePaid >= beast::kZero,
"xrpl::detail::makeRegularPayment : fee paid is valid");
return totalParts;
}
} // namespace detail
detail::LoanStateDeltas
@@ -1632,8 +1926,10 @@ constructLoanState(
}
LoanState
constructRoundedLoanState(SLE::const_ref loan)
constructLoanState(SLE::const_ref loan)
{
XRPL_ASSERT(loan && loan->getType() == ltLOAN, "xrpl::constructLoanState : valid loan SLE");
return constructLoanState(
loan->at(sfTotalValueOutstanding),
loan->at(sfPrincipalOutstanding),
@@ -1790,12 +2086,7 @@ loanMakePayment(
LoanPaymentType const paymentType,
beast::Journal j)
{
using namespace Lending;
auto principalOutstandingProxy = loan->at(sfPrincipalOutstanding);
auto paymentRemainingProxy = loan->at(sfPaymentRemaining);
if (paymentRemainingProxy == 0 || principalOutstandingProxy == 0)
if (loan->at(sfPaymentRemaining) == 0 || loan->at(sfPrincipalOutstanding) == 0)
{
// Loan complete this is already checked in LoanPay::preclaim()
// LCOV_EXCL_START
@@ -1804,9 +2095,6 @@ loanMakePayment(
// LCOV_EXCL_STOP
}
auto totalValueOutstandingProxy = loan->at(sfTotalValueOutstanding);
auto managementFeeOutstandingProxy = loan->at(sfManagementFeeOutstanding);
// Next payment due date must be set unless the loan is complete
auto nextDueDateProxy = loan->at(sfNextPaymentDueDate);
if (*nextDueDateProxy == 0)
@@ -1815,26 +2103,8 @@ loanMakePayment(
return std::unexpected(tecINTERNAL);
}
std::int32_t const loanScale = loan->at(sfLoanScale);
TenthBips32 const interestRate{loan->at(sfInterestRate)};
Number const serviceFee = loan->at(sfLoanServiceFee);
TenthBips16 const managementFeeRate{brokerSle->at(sfManagementFeeRate)};
Number const periodicPayment = loan->at(sfPeriodicPayment);
auto prevPaymentDateProxy = loan->at(sfPreviousPaymentDueDate);
std::uint32_t const startDate = loan->at(sfStartDate);
std::uint32_t const paymentInterval = loan->at(sfPaymentInterval);
// Compute the periodic rate that will be used for calculations
// throughout
Number const periodicRate = loanPeriodicRate(interestRate, paymentInterval);
XRPL_ASSERT(interestRate == 0 || periodicRate > 0, "xrpl::loanMakePayment : valid rate");
XRPL_ASSERT(*totalValueOutstandingProxy > 0, "xrpl::loanMakePayment : valid total value");
XRPL_ASSERT(
*loan->at(sfTotalValueOutstanding) > 0, "xrpl::loanMakePayment : valid total value");
view.update(loan);
@@ -1844,311 +2114,29 @@ loanMakePayment(
{
// If the payment is late, and the late flag was not set, it's not
// valid
JLOG(j.warn()) << "Loan payment is overdue. Use the tfLoanLatePayment "
"transaction "
"flag to make a late payment. Loan was created on "
<< startDate << ", prev payment due date is " << prevPaymentDateProxy
<< ", next payment due date is " << nextDueDateProxy << ", ledger time is "
JLOG(j.warn()) << "Loan payment is overdue. Use the tfLoanLatePayment transaction flag to "
"make a late payment. Loan was created on "
<< loan->at(sfStartDate) << ", prev payment due date is "
<< loan->at(sfPreviousPaymentDueDate) << ", next payment due date is "
<< nextDueDateProxy << ", ledger time is "
<< view.parentCloseTime().time_since_epoch().count();
return std::unexpected(tecEXPIRED);
}
// -------------------------------------------------------------
// full payment handling
if (paymentType == LoanPaymentType::Full)
switch (paymentType)
{
TenthBips32 const closeInterestRate{loan->at(sfCloseInterestRate)};
Number const closePaymentFee = roundToAsset(asset, loan->at(sfClosePaymentFee), loanScale);
LoanState const roundedLoanState = constructLoanState(
totalValueOutstandingProxy, principalOutstandingProxy, managementFeeOutstandingProxy);
auto const fullPaymentComponents = detail::computeFullPayment(
asset,
view,
principalOutstandingProxy,
managementFeeOutstandingProxy,
periodicPayment,
paymentRemainingProxy,
prevPaymentDateProxy,
startDate,
paymentInterval,
closeInterestRate,
loanScale,
roundedLoanState.interestDue,
periodicRate,
closePaymentFee,
amount,
managementFeeRate,
j);
if (fullPaymentComponents.has_value())
{
return doPayment(
*fullPaymentComponents,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
paymentRemainingProxy,
prevPaymentDateProxy,
nextDueDateProxy,
paymentInterval);
}
if (fullPaymentComponents.error())
{
// error() will be the TER returned if a payment is not made. It
// will only evaluate to true if it's unsuccessful. Otherwise,
// tesSUCCESS means nothing was done, so continue.
return std::unexpected(fullPaymentComponents.error());
}
// LCOV_EXCL_START
UNREACHABLE("xrpl::loanMakePayment : invalid full payment result");
JLOG(j.error()) << "Full payment computation failed unexpectedly.";
return std::unexpected(tecINTERNAL);
// LCOV_EXCL_STOP
case LoanPaymentType::Full:
return detail::makeFullPayment(asset, view, loan, brokerSle, amount, j);
case LoanPaymentType::Late:
return detail::makeLatePayment(asset, view, loan, brokerSle, amount, j);
case LoanPaymentType::Regular:
case LoanPaymentType::Overpayment:
return detail::makeRegularPayment(asset, view, loan, brokerSle, amount, paymentType, j);
}
// -------------------------------------------------------------
// compute the periodic payment info that will be needed whether the
// payment is late or regular
detail::ExtendedPaymentComponents periodic{
detail::computePaymentComponents(
view.rules(),
asset,
loanScale,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
periodicPayment,
periodicRate,
paymentRemainingProxy,
managementFeeRate),
serviceFee};
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::loanMakePayment",
"regular payment valid principal");
// -------------------------------------------------------------
// late payment handling
if (paymentType == LoanPaymentType::Late)
{
TenthBips32 const lateInterestRate{loan->at(sfLateInterestRate)};
Number const latePaymentFee = loan->at(sfLatePaymentFee);
auto const latePaymentComponents = detail::computeLatePayment(
asset,
view,
principalOutstandingProxy,
nextDueDateProxy,
periodic,
lateInterestRate,
loanScale,
latePaymentFee,
amount,
managementFeeRate,
j);
if (latePaymentComponents.has_value())
{
return doPayment(
*latePaymentComponents,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
paymentRemainingProxy,
prevPaymentDateProxy,
nextDueDateProxy,
paymentInterval);
}
if (latePaymentComponents.error())
{
// error() will be the TER returned if a payment is not made. It
// will only evaluate to true if it's unsuccessful.
return std::unexpected(latePaymentComponents.error());
}
// LCOV_EXCL_START
UNREACHABLE("xrpl::loanMakePayment : invalid late payment result");
JLOG(j.error()) << "Late payment computation failed unexpectedly.";
return std::unexpected(tecINTERNAL);
// LCOV_EXCL_STOP
}
// -------------------------------------------------------------
// regular periodic payment handling
XRPL_ASSERT_PARTS(
paymentType == LoanPaymentType::Regular || paymentType == LoanPaymentType::Overpayment,
"xrpl::loanMakePayment",
"regular payment type");
// Keep a running total of the actual parts paid
LoanPaymentParts totalParts;
Number totalPaid;
std::size_t numPayments = 0;
while ((amount >= (totalPaid + periodic.totalDue)) && paymentRemainingProxy > 0 &&
numPayments < kLoanMaximumPaymentsPerTransaction)
{
// Try to make more payments
XRPL_ASSERT_PARTS(
periodic.trackedPrincipalDelta >= 0,
"xrpl::loanMakePayment",
"payment pays non-negative principal");
totalPaid += periodic.totalDue;
totalParts += detail::doPayment(
periodic,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
paymentRemainingProxy,
prevPaymentDateProxy,
nextDueDateProxy,
paymentInterval);
++numPayments;
XRPL_ASSERT_PARTS(
(periodic.specialCase == detail::PaymentSpecialCase::Final) ==
(paymentRemainingProxy == 0),
"xrpl::loanMakePayment",
"final payment is the final payment");
// Don't compute the next payment if this was the last payment
if (periodic.specialCase == detail::PaymentSpecialCase::Final)
break;
periodic = detail::ExtendedPaymentComponents{
detail::computePaymentComponents(
view.rules(),
asset,
loanScale,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
periodicPayment,
periodicRate,
paymentRemainingProxy,
managementFeeRate),
serviceFee};
}
if (numPayments == 0)
{
JLOG(j.warn()) << "Regular loan payment amount is insufficient. Due: " << periodic.totalDue
<< ", paid: " << amount;
return std::unexpected(tecINSUFFICIENT_PAYMENT);
}
XRPL_ASSERT_PARTS(
totalParts.principalPaid + totalParts.interestPaid + totalParts.feePaid == totalPaid,
"xrpl::loanMakePayment",
"payment parts add up");
XRPL_ASSERT_PARTS(totalParts.valueChange == 0, "xrpl::loanMakePayment", "no value change");
// -------------------------------------------------------------
// overpayment handling
//
// If the "fixCleanup3_1_3" amendment is enabled, truncate "amount",
// at the loan scale. If the raw value is used, the overpayment
// amount could be meaningless dust. Trying to process such a small
// amount will, at best, waste time when all the result values round
// to zero. At worst, it can cause logical errors with tiny amounts
// of interest that don't add up correctly.
auto const roundedAmount = view.rules().enabled(fixCleanup3_1_3)
? roundToAsset(asset, amount, loanScale, Number::RoundingMode::TowardsZero)
: amount;
if (paymentType == LoanPaymentType::Overpayment && loan->isFlag(lsfLoanOverpayment) &&
paymentRemainingProxy > 0 && totalPaid < roundedAmount &&
numPayments < kLoanMaximumPaymentsPerTransaction)
{
TenthBips32 const overpaymentInterestRate{loan->at(sfOverpaymentInterestRate)};
TenthBips32 const overpaymentFeeRate{loan->at(sfOverpaymentFee)};
// It shouldn't be possible for the overpayment to be greater than
// totalValueOutstanding, because that would have been processed as
// another normal payment. But cap it just in case.
Number const overpaymentRaw =
std::min(roundedAmount - totalPaid, *totalValueOutstandingProxy);
bool const fixEnabled = view.rules().enabled(fixCleanup3_2_0);
Number const overpayment = fixEnabled
? roundToAsset(asset, overpaymentRaw, loanScale, Number::RoundingMode::Downward)
: overpaymentRaw;
// Post-amendment, the rounded overpayment can be zero; pre-amendment
// it's always positive given the surrounding guards.
if (!fixEnabled || overpayment > 0)
{
detail::ExtendedPaymentComponents const overpaymentComponents =
detail::computeOverpaymentComponents(
view.rules(),
asset,
loanScale,
overpayment,
overpaymentInterestRate,
overpaymentFeeRate,
managementFeeRate);
// Don't process an overpayment if the whole amount (or more!)
// gets eaten by fees and interest.
if (overpaymentComponents.trackedPrincipalDelta > 0)
{
XRPL_ASSERT_PARTS(
overpaymentComponents.untrackedInterest >= beast::kZero,
"xrpl::loanMakePayment",
"overpayment penalty did not reduce value of loan");
// Can't just use `periodicPayment` here, because it might
// change
auto periodicPaymentProxy = loan->at(sfPeriodicPayment);
if (auto const overResult = detail::doOverpayment(
view.rules(),
asset,
loanScale,
overpaymentComponents,
totalValueOutstandingProxy,
principalOutstandingProxy,
managementFeeOutstandingProxy,
periodicPaymentProxy,
periodicRate,
paymentRemainingProxy,
managementFeeRate,
j))
{
totalParts += *overResult;
}
else if (overResult.error())
{
// error() will be the TER returned if a payment is not
// made. It will only evaluate to true if it's unsuccessful.
// Otherwise, tesSUCCESS means nothing was done, so
// continue.
return std::unexpected(overResult.error());
}
}
}
}
// Check the final results are rounded, to double-check that the
// intermediate steps were rounded.
XRPL_ASSERT(
isRounded(asset, totalParts.principalPaid, loanScale) &&
totalParts.principalPaid >= beast::kZero,
"xrpl::loanMakePayment : total principal paid is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.interestPaid, loanScale) &&
totalParts.interestPaid >= beast::kZero,
"xrpl::loanMakePayment : total interest paid is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.valueChange, loanScale),
"xrpl::loanMakePayment : loan value change is valid");
XRPL_ASSERT(
isRounded(asset, totalParts.feePaid, loanScale) && totalParts.feePaid >= beast::kZero,
"xrpl::loanMakePayment : fee paid is valid");
return totalParts;
// LCOV_EXCL_START
UNREACHABLE("xrpl::loanMakePayment : invalid payment type");
return std::unexpected(tecINTERNAL);
// LCOV_EXCL_STOP
}
} // namespace xrpl

View File

@@ -13,6 +13,7 @@
#include <xrpl/nodestore/Scheduler.h>
#include <xrpl/nodestore/Types.h>
#include <atomic>
#include <cstdint>
#include <exception>
#include <functional>
@@ -52,6 +53,7 @@ DatabaseRotatingImp::rotate(
// callback finishes. Only then will the archive directory be
// deleted.
std::shared_ptr<NodeStore::Backend> oldArchiveBackend;
std::uint64_t copyForwards = 0;
{
std::scoped_lock const lock(mutex_);
@@ -62,11 +64,28 @@ DatabaseRotatingImp::rotate(
newArchiveBackendName = archiveBackend_->getName();
writableBackend_ = std::move(newBackend);
copyForwards = copyForwardCount_.exchange(0, std::memory_order_relaxed);
}
if (copyForwards > 0)
{
JLOG(j_.warn()) << "Rotating: copied forward " << copyForwards
<< " archive-served reads into the writable backend "
"during the rotation window";
}
f(newWritableBackendName, newArchiveBackendName);
}
void
DatabaseRotatingImp::setRotationInFlight(bool inFlight)
{
rotationInFlight_.store(inFlight, std::memory_order_release);
JLOG(j_.debug()) << "Rotating: copy-forward on archive reads "
<< (inFlight ? "enabled" : "disabled");
}
std::string
DatabaseRotatingImp::getName() const
{
@@ -177,9 +196,18 @@ DatabaseRotatingImp::fetchNodeObject(
writable = writableBackend_;
}
// Update writable backend with data from the archive backend
if (duplicate)
// Update writable backend with data from the archive backend.
// While a rotation is in flight, ordinary (duplicate == false)
// reads served by the archive are copied forward too: the
// archive is about to be deleted, and a body canonicalized
// into the cache after the freshen getKeys() snapshot would
// otherwise survive only in RAM once the archive is dropped.
if (duplicate || rotationInFlight_.load(std::memory_order_acquire))
{
if (!duplicate)
copyForwardCount_.fetch_add(1, std::memory_order_relaxed);
writable->store(nodeObject);
}
}
}

View File

@@ -1,19 +1,19 @@
#include <xrpld/peerfinder/detail/Bootcache.h>
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/detail/Store.h>
#include <xrpld/peerfinder/detail/Tuning.h>
#include <xrpld/peerfinder/detail/iosformat.h>
#include <xrpl/peerfinder/detail/Bootcache.h>
#include <xrpl/basics/Log.h>
#include <xrpl/beast/net/IPEndpoint.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/PropertyStream.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/peerfinder/detail/Store.h>
#include <xrpl/peerfinder/detail/Tuning.h>
#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <iomanip>
#include <ios>
#include <vector>
namespace xrpl::PeerFinder {
@@ -82,13 +82,14 @@ Bootcache::load()
auto const result(this->map_.insert(value_type(endpoint, valence)));
if (!result.second)
{
JLOG(this->journal_.error()) << beast::Leftw(18) << "Bootcache discard " << endpoint;
JLOG(this->journal_.error())
<< std::left << std::setw(18) << "Bootcache discard " << endpoint;
}
}));
if (n > 0)
{
JLOG(journal_.info()) << beast::Leftw(18) << "Bootcache loaded " << n
JLOG(journal_.info()) << std::left << std::setw(18) << "Bootcache loaded " << n
<< ((n > 1) ? " addresses" : " address");
prune();
}
@@ -100,7 +101,7 @@ Bootcache::insert(beast::IP::Endpoint const& endpoint)
auto const result(map_.insert(value_type(endpoint, 0)));
if (result.second)
{
JLOG(journal_.trace()) << beast::Leftw(18) << "Bootcache insert " << endpoint;
JLOG(journal_.trace()) << std::left << std::setw(18) << "Bootcache insert " << endpoint;
prune();
flagForUpdate();
}
@@ -121,7 +122,7 @@ Bootcache::insertStatic(beast::IP::Endpoint const& endpoint)
if (result.second)
{
JLOG(journal_.trace()) << beast::Leftw(18) << "Bootcache insert " << endpoint;
JLOG(journal_.trace()) << std::left << std::setw(18) << "Bootcache insert " << endpoint;
prune();
flagForUpdate();
}
@@ -146,8 +147,9 @@ Bootcache::onSuccess(beast::IP::Endpoint const& endpoint)
XRPL_ASSERT(result.second, "xrpl::PeerFinder::Bootcache::onSuccess : endpoint inserted");
}
Entry const& entry(result.first->right);
JLOG(journal_.info()) << beast::Leftw(18) << "Bootcache connect " << endpoint << " with "
<< entry.valence() << ((entry.valence() > 1) ? " successes" : " success");
JLOG(journal_.info()) << std::left << std::setw(18) << "Bootcache connect " << endpoint
<< " with " << entry.valence()
<< ((entry.valence() > 1) ? " successes" : " success");
flagForUpdate();
}
@@ -170,8 +172,8 @@ Bootcache::onFailure(beast::IP::Endpoint const& endpoint)
}
Entry const& entry(result.first->right);
auto const n(std::abs(entry.valence()));
JLOG(journal_.debug()) << beast::Leftw(18) << "Bootcache failed " << endpoint << " with " << n
<< ((n > 1) ? " attempts" : " attempt");
JLOG(journal_.debug()) << std::left << std::setw(18) << "Bootcache failed " << endpoint
<< " with " << n << ((n > 1) ? " attempts" : " attempt");
flagForUpdate();
}
@@ -209,17 +211,19 @@ Bootcache::prune()
// Work backwards because bimap doesn't handle
// erasing using a reverse iterator very well.
//
for (auto iter(map_.right.end()); count-- > 0 && iter != map_.right.begin(); ++pruned)
for (auto iter(map_.right.end()); count > 0 && iter != map_.right.begin(); ++pruned)
{
--count;
--iter;
beast::IP::Endpoint const& endpoint(iter->get_left());
Entry const& entry(iter->get_right());
JLOG(journal_.trace()) << beast::Leftw(18) << "Bootcache pruned" << endpoint
JLOG(journal_.trace()) << std::left << std::setw(18) << "Bootcache pruned" << endpoint
<< " at valence " << entry.valence();
iter = map_.right.erase(iter);
}
JLOG(journal_.debug()) << beast::Leftw(18) << "Bootcache pruned " << pruned << " entries total";
JLOG(journal_.debug()) << std::left << std::setw(18) << "Bootcache pruned " << pruned
<< " entries total";
}
// Updates the Store with the current set of entries if needed.

View File

@@ -0,0 +1,135 @@
#include <xrpl/peerfinder/Config.h>
#include <xrpl/beast/utility/PropertyStream.h>
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <stdexcept>
namespace xrpl::PeerFinder {
std::size_t
Config::calcOutPeers() const
{
return std::max(
((maxPeers * Tuning::kOutPercent) + 50) / 100, std::size_t(Tuning::kMinOutCount));
}
void
Config::applyTuning()
{
if (ipLimit == 0)
{
// Unless a limit is explicitly set, we allow between
// 2 and 5 connections from non RFC-1918 "private"
// IP addresses.
ipLimit = 2;
if (inPeers > Tuning::kDefaultMaxPeers)
ipLimit += std::min(5, static_cast<int>(inPeers / Tuning::kDefaultMaxPeers));
}
// We don't allow a single IP to consume all incoming slots,
// unless we only have one incoming slot available.
ipLimit = std::max(1, std::min(ipLimit, static_cast<int>(inPeers / 2)));
}
void
Config::onWrite(beast::PropertyStream::Map& map) const
{
map["max_peers"] = maxPeers;
map["out_peers"] = outPeers;
map["want_incoming"] = wantIncoming;
map["auto_connect"] = autoConnect;
map["port"] = listeningPort;
map["features"] = features;
map["ip_limit"] = ipLimit;
map["verify_endpoints"] = verifyEndpoints;
}
Config
Config::makeConfig(
bool peerPrivate,
bool standalone,
PeerLimitConfig const& limits,
std::uint16_t port,
bool validationPublicKey,
int ipLimit,
bool verifyEndpoints)
{
PeerFinder::Config config;
if (!limits.maxPeers)
{
if (limits.inPeers && !limits.outPeers)
throw std::runtime_error("Both inbound and outbound peer limits must be configured");
if (limits.outPeers && !limits.inPeers)
throw std::runtime_error("Both inbound and outbound peer limits must be configured");
if (limits.inPeers && *limits.inPeers > 1000)
throw std::runtime_error("Inbound peer limit must be less than or equal to 1000");
if (limits.outPeers && (*limits.outPeers < 10 || *limits.outPeers > 1000))
throw std::runtime_error("Outbound peer limit must be in the range 10-1000");
}
config.peerPrivate = peerPrivate;
// Servers with peer privacy don't want to allow incoming connections
config.wantIncoming = (!config.peerPrivate) && (port != 0);
if (limits.maxPeers || (!limits.inPeers && !limits.outPeers))
{
if (limits.maxPeers && *limits.maxPeers != 0)
config.maxPeers = *limits.maxPeers;
config.maxPeers = std::max<std::size_t>(config.maxPeers, Tuning::kMinOutCount);
config.outPeers = config.calcOutPeers();
// Calculate the number of outbound peers we want. If we dont want
// or can't accept incoming, this will simply be equal to maxPeers.
if (!config.wantIncoming)
config.outPeers = config.maxPeers;
// Calculate the largest number of inbound connections we could
// take.
if (config.maxPeers >= config.outPeers)
{
config.inPeers = config.maxPeers - config.outPeers;
}
else
{
config.inPeers = 0;
}
}
else
{
config.outPeers = *limits.outPeers;
config.inPeers = *limits.inPeers;
config.maxPeers = 0;
}
// This will cause servers configured as validators to request that
// peers they connect to never report their IP address. We set this
// after we set the 'wantIncoming' because we want a "soft" version
// of peer privacy unless the operator explicitly asks for it.
if (validationPublicKey)
config.peerPrivate = true;
// if it's a private peer or we are running as standalone
// automatic connections would defeat the purpose.
config.autoConnect = !standalone && !peerPrivate;
config.listeningPort = port;
config.features = "";
config.ipLimit = ipLimit;
config.verifyEndpoints = verifyEndpoints;
// Enforce business rules
config.applyTuning();
return config;
}
} // namespace xrpl::PeerFinder

View File

@@ -1,7 +1,5 @@
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/detail/Tuning.h>
#include <xrpl/beast/net/IPEndpoint.h>
#include <xrpl/peerfinder/Types.h>
#include <algorithm>
#include <cstdint>

View File

@@ -1,11 +1,4 @@
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/Slot.h>
#include <xrpld/peerfinder/detail/Checker.h>
#include <xrpld/peerfinder/detail/Logic.h>
#include <xrpld/peerfinder/detail/SlotImp.h>
#include <xrpld/peerfinder/detail/SourceStrings.h>
#include <xrpld/peerfinder/detail/StoreSqdb.h>
#include <xrpl/peerfinder/PeerfinderManager.h>
#include <xrpl/beast/insight/Collector.h>
#include <xrpl/beast/insight/Gauge.h>
@@ -13,7 +6,15 @@
#include <xrpl/beast/net/IPEndpoint.h>
#include <xrpl/beast/utility/Journal.h>
#include <xrpl/beast/utility/PropertyStream.h>
#include <xrpl/config/BasicConfig.h>
#include <xrpl/peerfinder/Config.h>
#include <xrpl/peerfinder/Slot.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/peerfinder/detail/Checker.h>
#include <xrpl/peerfinder/detail/Logic.h>
#include <xrpl/peerfinder/detail/SlotImp.h>
#include <xrpl/peerfinder/detail/SourceStrings.h>
#include <xrpl/peerfinder/detail/Store.h>
#include <xrpl/peerfinder/make_Manager.h>
#include <xrpl/protocol/PublicKey.h>
#include <boost/asio/executor_work_guard.hpp>
@@ -38,10 +39,9 @@ public:
std::optional<boost::asio::executor_work_guard<boost::asio::io_context::executor_type>> work_;
clock_type& clock_;
beast::Journal journal_;
StoreSqdb store_;
Store& store_;
Checker<boost::asio::ip::tcp> checker_;
Logic<decltype(checker_)> logic_;
BasicConfig const& config_;
// NOLINTEND(readability-identifier-naming)
//--------------------------------------------------------------------------
@@ -50,16 +50,15 @@ public:
boost::asio::io_context& ioContext,
clock_type& clock,
beast::Journal journal,
BasicConfig const& config,
Store& store,
beast::insight::Collector::ptr const& collector)
: io_context_(ioContext)
, work_(std::in_place, boost::asio::make_work_guard(io_context_))
, clock_(clock)
, journal_(journal)
, store_(journal)
, store_(store)
, checker_(io_context_)
, logic_(clock, store_, checker_, journal)
, config_(config)
, stats_([this] { collectMetrics(); }, collector)
{
}
@@ -206,7 +205,6 @@ public:
void
start() override
{
store_.open(config_);
logic_.load();
}
@@ -261,10 +259,10 @@ makeManager(
boost::asio::io_context& ioContext,
clock_type& clock,
beast::Journal journal,
BasicConfig const& config,
Store& store,
beast::insight::Collector::ptr const& collector)
{
return std::make_unique<ManagerImp>(ioContext, clock, journal, config, collector);
return std::make_unique<ManagerImp>(ioContext, clock, journal, store, collector);
}
} // namespace xrpl::PeerFinder

View File

@@ -1,12 +1,10 @@
#include <xrpld/peerfinder/detail/SlotImp.h>
#include <xrpl/peerfinder/detail/SlotImp.h>
#include <xrpld/peerfinder/PeerfinderManager.h>
#include <xrpld/peerfinder/Slot.h>
#include <xrpld/peerfinder/detail/Tuning.h>
#include <xrpl/beast/container/detail/aged_unordered_container.h>
#include <xrpl/beast/net/IPEndpoint.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/peerfinder/Slot.h>
#include <xrpl/peerfinder/Types.h>
#include <xrpl/peerfinder/detail/Tuning.h>
#include <cstdint>
#include <utility>

Some files were not shown because too many files have changed in this diff Show More