Compare commits

..

165 Commits

Author SHA1 Message Date
Ed Hennis
a34d1e5537 Merge branch 'develop' into ximinez/online-delete-gaps 2026-04-01 17:26:14 -04:00
Ed Hennis
78a122943d Merge branch 'develop' into ximinez/online-delete-gaps 2026-04-01 13:39:05 -04:00
Ed Hennis
06135e7203 Merge branch 'develop' into ximinez/online-delete-gaps 2026-04-01 11:45:32 -04:00
Ed Hennis
f20425fa4f Merge branch 'develop' into ximinez/online-delete-gaps 2026-03-30 21:29:37 -04:00
Ed Hennis
3e94546acc Merge branch 'develop' into ximinez/online-delete-gaps 2026-03-12 15:05:04 -04:00
Ed Hennis
3b088ed0dc Merge branch 'develop' into ximinez/online-delete-gaps 2026-03-10 12:46:18 -04:00
Ed Hennis
26182ed52e Merge branch 'develop' into ximinez/online-delete-gaps 2026-03-06 13:04:00 -04:00
Ed Hennis
de2a3e10f5 Merge branch 'develop' into ximinez/online-delete-gaps 2026-03-05 21:34:29 -04:00
Ed Hennis
e17f8554fc Merge branch 'develop' into ximinez/online-delete-gaps 2026-03-04 17:11:27 -04:00
Ed Hennis
386a7192ba Merge branch 'develop' into ximinez/online-delete-gaps 2026-03-03 20:46:41 -04:00
Ed Hennis
12cc6e424d Merge branch 'develop' into ximinez/online-delete-gaps 2026-03-03 15:54:32 -04:00
Ed Hennis
c9deecf1b7 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-24 17:34:32 -04:00
Ed Hennis
ddd1b49f38 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-24 16:45:55 -04:00
Ed Hennis
c1b2a24005 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-20 18:49:39 -04:00
Ed Hennis
86d88eca31 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-20 18:25:58 -04:00
Ed Hennis
ef09eaea00 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-20 17:31:41 -04:00
Ed Hennis
c504cfb291 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-20 17:21:03 -04:00
Ed Hennis
0c217dfa2b Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-20 15:14:23 -04:00
Ed Hennis
b0198d2566 Merge remote-tracking branch 'upstream/develop' into ximinez/online-delete-gaps
* upstream/develop:
  ci: Add dependabot config (6379)
  Fix tautological assertion (6393)
  chore: Apply clang-format width 100 (6387)
2026-02-20 12:20:07 -05:00
Ed Hennis
7eee8ca802 Update formatting 2026-02-20 12:15:30 -05:00
Ed Hennis
2a079a0154 Merge commit '25cca465538a56cce501477f9e5e2c1c7ea2d84c' into ximinez/online-delete-gaps
* commit '25cca465538a56cce501477f9e5e2c1c7ea2d84c':
  chore: Set clang-format width to 100 in config file (6387)
2026-02-20 12:12:56 -05:00
Ed Hennis
40989c1178 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-19 16:21:11 -05:00
Ed Hennis
addc831eb3 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-18 18:06:20 -04:00
Ed Hennis
b4efc6d116 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-04 16:29:49 -04:00
Ed Hennis
125d075d6e Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-04 14:16:24 -04:00
Ed Hennis
370a775479 Merge branch 'develop' into ximinez/online-delete-gaps 2026-02-03 16:07:47 -04:00
Ed Hennis
1a2ee706eb Fix formatting 2026-01-28 19:43:23 -05:00
Ed Hennis
2a981357ba Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-28 18:40:24 -04:00
Ed Hennis
1ae475e724 Merge commit '5f638f55536def0d88b970d1018a465a238e55f4' into ximinez/online-delete-gaps
* commit '5f638f55536def0d88b970d1018a465a238e55f4':
  chore: Set ColumnLimit to 120 in clang-format (6288)
2026-01-28 17:38:29 -05:00
Ed Hennis
a3e9401fbc Merge commit '92046785d1fea5f9efe5a770d636792ea6cab78b' into ximinez/online-delete-gaps
* commit '92046785d1fea5f9efe5a770d636792ea6cab78b':
  test: Fix the `xrpl.net` unit test using async read (6241)
  ci: Upload Conan recipes for develop, release candidates, and releases (6286)
  fix: Stop embedded tests from hanging on ARM by using `atomic_flag` (6248)
  fix:  Remove DEFAULT fields that change to the default in associateAsset (6259) (6273)
  refactor: Update Boost to 1.90 (6280)
  refactor: clean up uses of `std::source_location` (6272)
  ci: Pass missing sanitizers input to actions (6266)
  ci: Properly propagate Conan credentials (6265)
  ci: Explicitly set version when exporting the Conan recipe (6264)
  ci: Use plus instead of hyphen for Conan recipe version suffix (6261)
  chore: Detect uninitialized variables in CMake files (6247)
  ci: Run on-trigger and on-pr when generate-version is modified (6257)
  refactor: Enforce 15-char limit and simplify labels for thread naming (6212)
  docs: Update Ripple Bug Bounty public key (6258)
  ci: Add missing commit hash to Conan recipe version (6256)
  fix: Include `<functional>` header in `Number.h` (6254)
  ci: Upload Conan recipe for merges into develop and commits to release (6235)
  Limit reply size on `TMGetObjectByHash` queries (6110)
  ci: remove 'master' branch as a trigger (6234)
  Improve ledger_entry lookups for fee, amendments, NUNL, and hashes (5644)
2026-01-28 17:38:13 -05:00
Ed Hennis
9091469f9e Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-15 13:03:03 -04:00
Ed Hennis
17fa54f1f9 Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-15 12:05:41 -04:00
Ed Hennis
8fb5347c2d Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-14 19:20:11 -04:00
Ed Hennis
6739bf998f Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-13 18:04:18 -04:00
Ed Hennis
6eea38ba67 Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-13 16:15:44 -04:00
Ed Hennis
e9cf88b359 Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-13 15:01:56 -04:00
Ed Hennis
645b203476 Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-12 21:07:23 -04:00
Ed Hennis
be2aff1f4c Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-12 14:51:34 -04:00
Ed Hennis
56ed237e82 Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-11 00:48:27 -04:00
Ed Hennis
fd7b0fd135 Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-08 17:05:28 -04:00
Ed Hennis
e700994891 Merge branch 'develop' into ximinez/online-delete-gaps 2026-01-08 13:03:41 -04:00
Ed Hennis
c76f7029ac Merge remote-tracking branch 'XRPLF/develop' into ximinez/online-delete-gaps
* XRPLF/develop:
  test: add more tests for `ledger_entry` RPC (5858)
  refactor: Rename `rippled.cfg` to `xrpld.cfg` (6098)
  Revert "chore: Pin ruamel.yaml<0.19 in pre-commit-hooks (6166)" (6167)
  chore: Pin ruamel.yaml<0.19 in pre-commit-hooks (6166)
  fix: Remove cryptographic libs from libxrpl Conan package (6163)
2026-01-06 11:27:23 -05:00
Ed Hennis
d535c5fb2a Merge branch 'develop' into ximinez/online-delete-gaps 2025-12-22 17:39:25 -05:00
Ed Hennis
54f860463e Merge branch 'develop' into ximinez/online-delete-gaps 2025-12-18 18:15:15 -05:00
Ed Hennis
950434b8ff Merge branch 'develop' into ximinez/online-delete-gaps 2025-12-17 12:12:39 -05:00
Ed Hennis
ee365e876d Merge branch 'develop' into ximinez/online-delete-gaps 2025-12-12 20:34:29 -05:00
Ed Hennis
c6c59834b9 Update View info() to header() 2025-12-12 15:35:07 -05:00
Ed Hennis
63b47914b8 Merge remote-tracking branch 'XRPLF/develop' into ximinez/online-delete-gaps
* XRPLF/develop:
  refactor: Rename `ripple` namespace to `xrpl` (5982)
  refactor: Move JobQueue and related classes into xrpl.core module (6121)
  refactor: Rename `rippled` binary to `xrpld` (5983)
2025-12-11 15:25:43 -05:00
Ed Hennis
9e02e5be2e Merge branch 'develop' into ximinez/online-delete-gaps 2025-12-10 18:55:14 -05:00
Ed Hennis
093cd70fa1 Merge branch 'develop' into ximinez/online-delete-gaps 2025-12-05 21:12:43 -05:00
Ed Hennis
376d65a483 Merge branch 'develop' into ximinez/online-delete-gaps 2025-12-02 17:21:39 -05:00
Ed Hennis
a0d9a2458e Merge branch 'develop' into ximinez/online-delete-gaps 2025-12-01 14:40:15 -05:00
Ed Hennis
456f639cf7 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-28 15:46:19 -05:00
Ed Hennis
2c559ec2f3 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-27 01:48:33 -05:00
Ed Hennis
619c81f463 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-26 00:24:50 -05:00
Ed Hennis
f1490df960 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-25 14:54:38 -05:00
Ed Hennis
7bdf74de98 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-24 21:48:42 -05:00
Ed Hennis
1743d6fb98 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-24 21:29:52 -05:00
Ed Hennis
ca7a5bb926 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-21 12:47:28 -05:00
Ed Hennis
ce8b1a3f1e Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-18 22:39:06 -05:00
Ed Hennis
486fa75a10 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-15 03:08:18 -05:00
Ed Hennis
f8d68cd3d3 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-13 12:18:08 -05:00
Ed Hennis
ef7a3f5606 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-12 13:59:45 -05:00
Ed Hennis
4f84ed7490 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-10 19:52:42 -05:00
Ed Hennis
d534103131 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-10 15:34:55 -05:00
Ed Hennis
82dff3c2ce Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-08 22:49:26 -05:00
Ed Hennis
30d73eb5ba Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-06 23:50:23 -05:00
Ed Hennis
1b2754bac2 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-05 22:23:07 -05:00
Ed Hennis
cf80cafc75 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-04 18:02:30 -05:00
Ed Hennis
b8897d51de Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-03 18:03:56 -05:00
Ed Hennis
3ff25eeb65 Merge branch 'develop' into ximinez/online-delete-gaps 2025-11-03 12:41:48 -05:00
Ed Hennis
2bbfc4e786 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-31 13:51:15 -04:00
Ed Hennis
2b1eb052e6 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-31 12:51:24 -04:00
Ed Hennis
360e214e54 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-29 14:23:10 -04:00
Ed Hennis
2618afed94 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-29 13:42:21 -04:00
Ed Hennis
698ba2c788 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-29 12:54:16 -04:00
Ed Hennis
b614e99588 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-28 17:38:21 -04:00
Ed Hennis
fe8e4af2fa Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-26 19:12:33 -04:00
Ed Hennis
0a897f1528 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-23 13:24:25 -04:00
Ed Hennis
cf8a3f5779 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-22 11:38:49 -04:00
Ed Hennis
db39a39868 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-21 22:20:04 -04:00
Ed Hennis
37a03d28c2 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-17 18:21:36 -04:00
Ed Hennis
19d275425a Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-16 13:12:08 -04:00
Ed Hennis
88e9045602 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-16 10:48:43 -04:00
Ed Hennis
5adbc536b6 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-10 13:01:32 -04:00
Ed Hennis
e27af94ba9 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-09 15:14:47 -04:00
Ed Hennis
43fe1e7e9c Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-08 14:21:34 -04:00
Ed Hennis
f456a858c8 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-02 11:03:07 -04:00
Ed Hennis
084c3aa88e Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-01 18:10:57 -04:00
Ed Hennis
34f9b63921 Merge branch 'develop' into ximinez/online-delete-gaps 2025-10-01 13:14:22 -04:00
Ed Hennis
bd3de79817 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-30 22:28:53 -04:00
Ed Hennis
304eee2259 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-29 18:34:43 -04:00
Ed Hennis
9e729b7f59 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-29 17:37:16 -04:00
Ed Hennis
dd141468c4 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-29 13:32:03 -04:00
Ed Hennis
933147c21f Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-26 19:25:58 -04:00
Ed Hennis
9201a4f591 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-26 13:41:27 -04:00
Ed Hennis
5adb1e9b8b Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-26 12:09:09 -04:00
Ed Hennis
4df84d7988 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-25 13:27:09 -04:00
Bart
cd87c0968b Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-24 09:35:19 +02:00
Ed Hennis
8a8e7c90bf Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-20 15:44:32 -04:00
Ed Hennis
e806069065 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-18 14:08:23 -04:00
Ed Hennis
ce948cbec0 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-18 12:26:34 -04:00
Ed Hennis
6ed34b3294 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-18 11:54:30 -04:00
Ed Hennis
7161a235ca Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-17 10:49:03 -04:00
Ed Hennis
71463810de Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-16 10:46:37 -04:00
Ed Hennis
e997219a85 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-15 11:13:28 -04:00
Ed Hennis
895cc13fa6 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-11 10:33:14 -04:00
Ed Hennis
8d3c3ca29a Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-10 18:53:24 -04:00
Ed Hennis
9829553807 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-09 17:14:20 -04:00
Ed Hennis
e551f9731a Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-08 11:41:46 -04:00
Ed Hennis
fd827bf58b Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-05 17:44:06 -04:00
Ed Hennis
5a3baba34d Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-04 20:24:55 -04:00
Ed Hennis
c78f5b160f Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-04 16:43:48 -04:00
Ed Hennis
485f78761a Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-04 12:26:55 -04:00
Ed Hennis
23cd2f7b21 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-04 10:14:10 -04:00
Ed Hennis
5753266c43 Merge branch 'develop' into ximinez/online-delete-gaps 2025-09-03 14:04:01 -04:00
Ed Hennis
4722d2607d Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-29 15:52:53 -04:00
Ed Hennis
85b5b4f855 Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-29 10:42:50 -04:00
Ed Hennis
a16f492f0f Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-28 18:17:26 -04:00
Ed Hennis
3633dc632c Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-27 11:14:59 -04:00
Ed Hennis
b3b30c3a86 Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-25 14:10:52 -04:00
Ed Hennis
c78a7684f4 Remove trailing space 2025-08-25 14:09:57 -04:00
Ed Hennis
cf83d92630 Merge remote-tracking branch 'upstream/develop' into ximinez/online-delete-gaps
* upstream/develop:
  chore: Remove codecov token check to support tokenless uploads on forks (5722)
  Set version to 2.6.0-rc3
  Revert "perf: Move mutex to the partition level (5486)"
  chore: Update clang-format and prettier with pre-commit (5709)
  fix(test): handle null metadata for unvalidated tx in Env::meta (5715)
  chore: Workaround for CI build errors on arm64 (5717)
  chore: Fix file formatting (5718)
  fix: Skip notify-clio when running in a fork, reorder config fields (5712)
  chore: Reverts formatting changes to external files, adds formatting changes to proto files (5711)
2025-08-25 14:05:49 -04:00
Ed Hennis
a56b1274d8 Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-21 11:38:54 -04:00
Ed Hennis
ae4bdd0492 Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-19 16:05:17 -04:00
Ed Hennis
e90102dd3b Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-18 12:15:38 -04:00
Ed Hennis
71f0e8db3d Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-08 18:23:24 -04:00
Ed Hennis
638929373a Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-08 11:10:20 -04:00
Ed Hennis
8440654377 Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-06 21:03:00 -04:00
Ed Hennis
9fa66c4741 Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-05 21:16:44 -04:00
Ed Hennis
38a9235145 Merge branch 'develop' into ximinez/online-delete-gaps 2025-08-04 13:04:52 -04:00
Ed Hennis
c7a3cc9108 Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-29 20:33:29 -04:00
Ed Hennis
248337908d Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-29 11:54:16 -04:00
Ed Hennis
3d003619fd Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-28 20:57:13 -04:00
Ed Hennis
f163dca12c Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-24 15:50:09 -04:00
Ed Hennis
6e0ce458e5 Revert "TEMP: Change some logging to fatal to diagnose CI failures"
This reverts commit 69cf18158b.
2025-07-22 19:41:02 -04:00
Ed Hennis
5fae8480f1 Revert "TEMP: Add logging to SHAMapStore test"
This reverts commit fe7d0798a7.
2025-07-22 19:40:58 -04:00
Ed Hennis
e6587d374a fixup! Tweak SHAMapStore test timing more 2025-07-22 19:39:58 -04:00
Ed Hennis
376cc404e0 Tweak SHAMapStore test timing more 2025-07-22 18:44:07 -04:00
Ed Hennis
9898ca638f Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-22 14:09:33 -04:00
Ed Hennis
34b46d8f7c Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-21 18:20:41 -04:00
Ed Hennis
fe7d0798a7 TEMP: Add logging to SHAMapStore test 2025-07-21 18:19:53 -04:00
Ed Hennis
0cecc09d71 Tweak timing of SHAMapStore test 2025-07-21 18:19:28 -04:00
Ed Hennis
e091d55561 Try to fix timing of LedgerMaster test 2025-07-21 15:14:42 -04:00
Ed Hennis
69cf18158b TEMP: Change some logging to fatal to diagnose CI failures 2025-07-21 14:32:08 -04:00
Ed Hennis
6513c53817 Improve logging
- There's still a race condition in there
2025-07-21 14:30:49 -04:00
Ed Hennis
e13baa58a5 Fix build errors 2025-07-21 13:31:09 -04:00
Ed Hennis
951056fe9b Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-18 18:33:10 -04:00
Ed Hennis
67700ea6bd Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-16 12:54:23 -04:00
Ed Hennis
e5442cf3f1 Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-15 19:36:25 -04:00
Ed Hennis
da68076f04 Change default recovery wait time to 1s
See https://github.com/XRPLF/rippled/pull/5531#issuecomment-3058218837
2025-07-14 14:13:32 -04:00
Ed Hennis
b24116a118 Improve locking, logging, and test output
- Add more info to the error message on some failed tests.
- Add logging details to SHAMapStoreImp
2025-07-14 14:13:27 -04:00
Ed Hennis
f67398c6bf Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-11 19:27:31 -04:00
Ed Hennis
43d3eb1a24 Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-10 21:25:22 -04:00
Ed Hennis
0993315ed5 Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-10 12:30:18 -04:00
Ed Hennis
0bc383ada9 Fix build errors 2025-07-08 20:15:05 -04:00
Ed Hennis
1841ceca43 Add more logging to SHAMapStore rotation 2025-07-08 16:05:03 -04:00
Ed Hennis
2714cebabd Revert "TEMP: Change logging to show progress during unit test"
This reverts commit e184db4ce2.
2025-07-08 16:02:45 -04:00
Ed Hennis
e184db4ce2 TEMP: Change logging to show progress during unit test 2025-07-08 16:02:19 -04:00
Ed Hennis
ac6dc6943c Tweak when the starting range of ledger gap detection is set
- Add a test to exercise online delete ledger gap detection
2025-07-08 16:01:23 -04:00
Ed Hennis
ddd53806df Add a test to exercise LedgerMaster::missingFromCompleteLedgerRange 2025-07-08 13:22:47 -04:00
Ed Hennis
e629a1f70e Merge branch 'develop' into ximinez/online-delete-gaps 2025-07-03 15:51:23 -04:00
Ed Hennis
68076d969c fixup! fixup! Pause online delete if there any any gaps in recent ledger history 2025-07-02 19:05:06 -04:00
Ed Hennis
d3009d3e1c fixup! Pause online delete if there any any gaps in recent ledger history 2025-07-02 18:51:42 -04:00
Ed Hennis
54f7f3c894 Pause online delete if there any any gaps in recent ledger history 2025-07-02 18:45:48 -04:00
30 changed files with 440 additions and 793 deletions

View File

@@ -44,7 +44,6 @@ libxrpl.tx > xrpl.server
libxrpl.tx > xrpl.tx
test.app > test.jtx
test.app > test.rpc
test.app > test.shamap
test.app > test.toplevel
test.app > test.unit_test
test.app > xrpl.basics
@@ -60,7 +59,6 @@ test.app > xrpl.protocol
test.app > xrpl.rdb
test.app > xrpl.resource
test.app > xrpl.server
test.app > xrpl.shamap
test.app > xrpl.tx
test.basics > test.jtx
test.basics > test.unit_test

View File

@@ -1033,10 +1033,11 @@
# The online delete process checks periodically
# that xrpld is still in sync with the network,
# and that the validated ledger is less than
# 'age_threshold_seconds' old. If not, then continue
# 'age_threshold_seconds' old, and that all
# recent ledgers are available. If not, then continue
# sleeping for this number of seconds and
# checking until healthy.
# Default is 5.
# Default is 1.
#
# Notes:
# The 'node_db' entry configures the primary, persistent storage.

View File

@@ -11,7 +11,6 @@
#include <limits>
#include <stdexcept>
#include <string>
#include <string_view>
#include <type_traits>
#include <vector>
@@ -232,11 +231,4 @@ makeSlice(std::basic_string<char, Traits, Alloc> const& s)
return Slice(s.data(), s.size());
}
template <class Traits>
Slice
makeSlice(std::basic_string_view<char, Traits> const& s)
{
return Slice(s.data(), s.size());
}
} // namespace xrpl

View File

@@ -244,15 +244,7 @@ message TMGetObjectByHash {
message TMLedgerNode {
required bytes nodedata = 1;
// Used when protocol version <2.3. Not set for ledger base data.
optional bytes nodeid = 2;
// Used when protocol version >=2.3. Neither value is set for ledger base data.
oneof reference {
bytes id = 3; // Set for inner nodes.
uint32 depth = 4; // Set for leaf nodes.
}
optional bytes nodeid = 2; // missing for ledger base data
}
enum TMLedgerInfoType {

View File

@@ -16,7 +16,6 @@
#include <set>
#include <stack>
#include <tuple>
#include <vector>
namespace xrpl {
@@ -254,7 +253,7 @@ public:
bool
getNodeFat(
SHAMapNodeID const& wanted,
std::vector<std::tuple<SHAMapNodeID, Blob, bool>>& data,
std::vector<std::pair<SHAMapNodeID, Blob>>& data,
bool fatLeaves,
std::uint32_t depth) const;
@@ -281,45 +280,10 @@ public:
void
serializeRoot(Serializer& s) const;
/** Add a root node to the SHAMap during synchronization.
*
* This function is used when receiving the root node of a SHAMap from a peer during ledger
* synchronization. The node must already have been deserialized.
*
* @param hash The expected hash of the root node.
* @param rootNode A deserialized root node to add.
* @param filter Optional sync filter to track received nodes.
* @return Status indicating whether the node was useful, duplicate, or invalid.
*
* @note This function expects the rootNode to be a valid, deserialized SHAMapTreeNode. The
* caller is responsible for deserialization and basic validation before calling this
* function.
*/
SHAMapAddNode
addRootNode(
SHAMapHash const& hash,
intr_ptr::SharedPtr<SHAMapTreeNode> rootNode,
SHAMapSyncFilter const* filter);
/** Add a known node at a specific position in the SHAMap during synchronization.
*
* This function is used when receiving nodes from peers during ledger synchronization. The node
* is inserted at the position specified by nodeID. The node must already have been
* deserialized.
*
* @param nodeID The position in the tree where this node belongs.
* @param treeNode A deserialized tree node to add.
* @param filter Optional sync filter to track received nodes.
* @return Status indicating whether the node was useful, duplicate, or invalid.
*
* @note This function expects that the caller has already validated that the nodeID is
* consistent with the node's content.
*/
addRootNode(SHAMapHash const& hash, Slice const& rootNode, SHAMapSyncFilter* filter);
SHAMapAddNode
addKnownNode(
SHAMapNodeID const& nodeID,
intr_ptr::SharedPtr<SHAMapTreeNode> treeNode,
SHAMapSyncFilter const* filter);
addKnownNode(SHAMapNodeID const& nodeID, Slice const& rawNode, SHAMapSyncFilter* filter);
// status functions
void
@@ -380,11 +344,11 @@ private:
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNodeNT(SHAMapHash const& hash) const;
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const;
fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
intr_ptr::SharedPtr<SHAMapTreeNode>
fetchNode(SHAMapHash const& hash) const;
intr_ptr::SharedPtr<SHAMapTreeNode>
checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const;
checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const;
/** Update hashes up to the root */
void
@@ -456,7 +420,7 @@ private:
descendAsync(
SHAMapInnerNode* parent,
int branch,
SHAMapSyncFilter const* filter,
SHAMapSyncFilter* filter,
bool& pending,
descendCallback&&) const;
@@ -465,7 +429,7 @@ private:
SHAMapInnerNode* parent,
SHAMapNodeID const& parentID,
int branch,
SHAMapSyncFilter const* filter) const;
SHAMapSyncFilter* filter) const;
// Non-storing
// Does not hook the returned node to its parent

View File

@@ -179,7 +179,7 @@ SHAMap::finishFetch(SHAMapHash const& hash, std::shared_ptr<NodeObject> const& o
// See if a sync filter has a node
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const
SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
{
if (auto nodeData = filter->getNode(hash))
{
@@ -205,7 +205,7 @@ SHAMap::checkFilter(SHAMapHash const& hash, SHAMapSyncFilter const* filter) cons
// Get a node without throwing
// Used on maps where missing nodes are expected
intr_ptr::SharedPtr<SHAMapTreeNode>
SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter const* filter) const
SHAMap::fetchNodeNT(SHAMapHash const& hash, SHAMapSyncFilter* filter) const
{
auto node = cacheLookup(hash);
if (node)
@@ -318,7 +318,7 @@ SHAMap::descend(
SHAMapInnerNode* parent,
SHAMapNodeID const& parentID,
int branch,
SHAMapSyncFilter const* filter) const
SHAMapSyncFilter* filter) const
{
XRPL_ASSERT(parent->isInner(), "xrpl::SHAMap::descend : valid parent input");
XRPL_ASSERT(
@@ -347,7 +347,7 @@ SHAMapTreeNode*
SHAMap::descendAsync(
SHAMapInnerNode* parent,
int branch,
SHAMapSyncFilter const* filter,
SHAMapSyncFilter* filter,
bool& pending,
descendCallback&& callback) const
{

View File

@@ -122,9 +122,7 @@ selectBranch(SHAMapNodeID const& id, uint256 const& hash)
SHAMapNodeID
SHAMapNodeID::createID(int depth, uint256 const& key)
{
XRPL_ASSERT(
depth >= 0 && depth <= SHAMap::leafDepth,
"xrpl::SHAMapNodeID::createID : valid branch input");
XRPL_ASSERT((depth >= 0) && (depth < 65), "xrpl::SHAMapNodeID::createID : valid branch input");
return SHAMapNodeID(depth, key & depthMask(depth));
}

View File

@@ -392,7 +392,7 @@ SHAMap::getMissingNodes(int max, SHAMapSyncFilter* filter)
bool
SHAMap::getNodeFat(
SHAMapNodeID const& wanted,
std::vector<std::tuple<SHAMapNodeID, Blob, bool>>& data,
std::vector<std::pair<SHAMapNodeID, Blob>>& data,
bool fatLeaves,
std::uint32_t depth) const
{
@@ -438,7 +438,7 @@ SHAMap::getNodeFat(
// Add this node to the reply
s.erase();
node->serializeForWire(s);
data.emplace_back(std::make_tuple(nodeID, s.getData(), node->isLeaf()));
data.emplace_back(std::make_pair(nodeID, s.getData()));
if (node->isInner())
{
@@ -468,8 +468,7 @@ SHAMap::getNodeFat(
// Just include this node
s.erase();
childNode->serializeForWire(s);
data.emplace_back(
std::make_tuple(childID, s.getData(), childNode->isLeaf()));
data.emplace_back(std::make_pair(childID, s.getData()));
}
}
}
@@ -487,18 +486,8 @@ SHAMap::serializeRoot(Serializer& s) const
}
SHAMapAddNode
SHAMap::addRootNode(
SHAMapHash const& hash,
intr_ptr::SharedPtr<SHAMapTreeNode> rootNode,
SHAMapSyncFilter const* filter)
SHAMap::addRootNode(SHAMapHash const& hash, Slice const& rootNode, SHAMapSyncFilter* filter)
{
XRPL_ASSERT(rootNode, "xrpl::SHAMap::addRootNode : non-null root node");
if (!rootNode)
{
JLOG(journal_.error()) << "Null node received";
return SHAMapAddNode::invalid();
}
// we already have a root_ node
if (root_->getHash().isNonZero())
{
@@ -508,16 +497,14 @@ SHAMap::addRootNode(
}
XRPL_ASSERT(cowid_ >= 1, "xrpl::SHAMap::addRootNode : valid cowid");
if (rootNode->getHash() != hash)
{
JLOG(journal_.warn()) << "Corrupt node received";
auto node = SHAMapTreeNode::makeFromWire(rootNode);
if (!node || node->getHash() != hash)
return SHAMapAddNode::invalid();
}
if (backed_)
canonicalize(hash, rootNode);
canonicalize(hash, node);
root_ = std::move(rootNode);
root_ = node;
if (root_->isLeaf())
clearSynching();
@@ -534,23 +521,9 @@ SHAMap::addRootNode(
}
SHAMapAddNode
SHAMap::addKnownNode(
SHAMapNodeID const& nodeID,
intr_ptr::SharedPtr<SHAMapTreeNode> treeNode,
SHAMapSyncFilter const* filter)
SHAMap::addKnownNode(SHAMapNodeID const& node, Slice const& rawNode, SHAMapSyncFilter* filter)
{
XRPL_ASSERT(!nodeID.isRoot(), "xrpl::SHAMap::addKnownNode : valid node input");
if (nodeID.isRoot())
{
JLOG(journal_.error()) << "Root node received";
return SHAMapAddNode::invalid();
}
XRPL_ASSERT(treeNode, "xrpl::SHAMap::addKnownNode : non-null tree node");
if (!treeNode)
{
JLOG(journal_.error()) << "Null node received";
return SHAMapAddNode::invalid();
}
XRPL_ASSERT(!node.isRoot(), "xrpl::SHAMap::addKnownNode : valid node input");
if (!isSynching())
{
@@ -564,14 +537,14 @@ SHAMap::addKnownNode(
while (currNode->isInner() &&
!safe_downcast<SHAMapInnerNode*>(currNode)->isFullBelow(generation) &&
(currNodeID.getDepth() < nodeID.getDepth()))
(currNodeID.getDepth() < node.getDepth()))
{
int const branch = selectBranch(currNodeID, nodeID.getNodeID());
int const branch = selectBranch(currNodeID, node.getNodeID());
XRPL_ASSERT(branch >= 0, "xrpl::SHAMap::addKnownNode : valid branch");
auto inner = safe_downcast<SHAMapInnerNode*>(currNode);
if (inner->isEmptyBranch(branch))
{
JLOG(journal_.warn()) << "Add known node for empty branch" << nodeID;
JLOG(journal_.warn()) << "Add known node for empty branch" << node;
return SHAMapAddNode::invalid();
}
@@ -587,44 +560,67 @@ SHAMap::addKnownNode(
if (currNode != nullptr)
continue;
if (childHash != treeNode->getHash())
auto newNode = SHAMapTreeNode::makeFromWire(rawNode);
if (!newNode || childHash != newNode->getHash())
{
JLOG(journal_.warn()) << "Corrupt node received";
return SHAMapAddNode::invalid();
}
// In rare cases, a node can still be corrupt even after hash
// validation. For leaf nodes, we perform an additional check to
// ensure the node's position in the tree is consistent with its
// content to prevent inconsistencies that could
// propagate further down the line.
if (newNode->isLeaf())
{
auto const& actualKey =
safe_downcast<SHAMapLeafNode const*>(newNode.get())->peekItem()->key();
// Validate that this leaf belongs at the target position
auto const expectedNodeID = SHAMapNodeID::createID(node.getDepth(), actualKey);
if (expectedNodeID.getNodeID() != node.getNodeID())
{
JLOG(journal_.debug())
<< "Leaf node position mismatch: "
<< "expected=" << expectedNodeID.getNodeID() << ", actual=" << node.getNodeID();
return SHAMapAddNode::invalid();
}
}
// Inner nodes must be at a level strictly less than 64
// but leaf nodes (while notionally at level 64) can be
// at any depth up to and including 64:
if ((currNodeID.getDepth() > leafDepth) ||
(treeNode->isInner() && currNodeID.getDepth() == leafDepth))
(newNode->isInner() && currNodeID.getDepth() == leafDepth))
{
// Map is provably invalid
state_ = SHAMapState::Invalid;
return SHAMapAddNode::useful();
}
if (currNodeID != nodeID)
if (currNodeID != node)
{
// Either this node is broken or we didn't request it (yet)
JLOG(journal_.warn()) << "unable to hook node " << nodeID;
JLOG(journal_.warn()) << "unable to hook node " << node;
JLOG(journal_.info()) << " stuck at " << currNodeID;
JLOG(journal_.info()) << "got depth=" << nodeID.getDepth()
JLOG(journal_.info()) << "got depth=" << node.getDepth()
<< ", walked to= " << currNodeID.getDepth();
return SHAMapAddNode::useful();
}
if (backed_)
canonicalize(childHash, treeNode);
canonicalize(childHash, newNode);
treeNode = prevNode->canonicalizeChild(branch, std::move(treeNode));
newNode = prevNode->canonicalizeChild(branch, std::move(newNode));
if (filter != nullptr)
{
Serializer s;
treeNode->serializeWithPrefix(s);
newNode->serializeWithPrefix(s);
filter->gotNode(
false, childHash, ledgerSeq_, std::move(s.modData()), treeNode->getType());
false, childHash, ledgerSeq_, std::move(s.modData()), newNode->getType());
}
return SHAMapAddNode::useful();

View File

@@ -2,6 +2,7 @@
#include <test/jtx/Env.h>
#include <xrpld/app/ledger/LedgerMaster.h>
#include <xrpld/app/misc/SHAMapStore.h>
namespace xrpl {
namespace test {
@@ -97,6 +98,73 @@ class LedgerMaster_test : public beast::unit_test::suite
}
}
void
testCompleteLedgerRange(FeatureBitset features)
{
// Note that this test is intentionally very similar to
// SHAMapStore_test::testLedgerGaps, but has a different
// focus.
testcase("Complete Ledger operations");
using namespace test::jtx;
auto const deleteInterval = 8;
Env env{*this, envconfig([](auto cfg) {
return online_delete(std::move(cfg), deleteInterval);
})};
auto const alice = Account("alice");
env.fund(XRP(1000), alice);
env.close();
auto& lm = env.app().getLedgerMaster();
LedgerIndex minSeq = 2;
LedgerIndex maxSeq = env.closed()->header().seq;
auto& store = env.app().getSHAMapStore();
store.rendezvous();
LedgerIndex lastRotated = store.getLastRotated();
BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
// Close enough ledgers to rotate a few times
for (int i = 0; i < 24; ++i)
{
for (int t = 0; t < 3; ++t)
{
env(noop(alice));
}
env.close();
store.rendezvous();
++maxSeq;
if (maxSeq == lastRotated + deleteInterval)
{
minSeq = lastRotated;
lastRotated = maxSeq;
}
BEAST_EXPECTS(
env.closed()->header().seq == maxSeq, to_string(env.closed()->header().seq));
BEAST_EXPECTS(store.getLastRotated() == lastRotated, to_string(store.getLastRotated()));
std::stringstream expectedRange;
expectedRange << minSeq << "-" << maxSeq;
BEAST_EXPECTS(lm.getCompleteLedgers() == expectedRange.str(), lm.getCompleteLedgers());
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
}
}
public:
void
run() override
@@ -110,6 +178,7 @@ public:
testWithFeats(FeatureBitset features)
{
testTxnIdFromIndex(features);
testCompleteLedgerRange(features);
}
};

View File

@@ -1,356 +0,0 @@
#include <test/shamap/common.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpl/beast/unit_test.h>
#include <xrpl/protocol/messages.h>
#include <xrpl/shamap/SHAMap.h>
#include <xrpl/shamap/SHAMapAccountStateLeafNode.h>
#include <xrpl/shamap/SHAMapInnerNode.h>
#include <xrpl/shamap/SHAMapItem.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <bit>
namespace xrpl {
namespace tests {
class LedgerNodeHelpers_test : public beast::unit_test::suite
{
// Helper function to create a simple SHAMapItem for testing.
static boost::intrusive_ptr<SHAMapItem>
makeTestItem(std::uint32_t seed)
{
Serializer s;
s.add32(seed);
s.add32(seed + 1);
s.add32(seed + 2);
return make_shamapitem(s.getSHA512Half(), s.slice());
}
// Helper function to serialize a tree node to wire format.
static std::string
serializeNode(intr_ptr::SharedPtr<SHAMapTreeNode> const& node)
{
Serializer s;
node->serializeForWire(s);
auto const slice = s.slice();
return std::string(std::bit_cast<char const*>(slice.data()), slice.size());
}
void
testValidateLedgerNode()
{
// In the tests below the validity of the content of the node data and ID fields is not
// checked - only that the fields have values when expected. The content of the fields is
// verified in the other tests in this file.
testcase("validateLedgerNode");
// Invalid: missing all fields.
{
protocol::TMLedgerNode const node;
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: missing `nodedata` field.
{
protocol::TMLedgerNode node;
node.set_nodeid("test_nodeid");
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: missing `nodedata` field.
{
protocol::TMLedgerNode node;
node.set_id("test_nodeid");
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: missing `nodedata` field.
{
protocol::TMLedgerNode node;
node.set_depth(1);
BEAST_EXPECT(!validateLedgerNode(node));
}
// Valid: legacy `nodeid` field.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_nodeid("test_nodeid");
BEAST_EXPECT(validateLedgerNode(node));
}
// Invalid: has both legacy `nodeid` and new `id` fields.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_nodeid("test_nodeid");
node.set_id("test_nodeid");
BEAST_EXPECT(!validateLedgerNode(node));
}
// Invalid: has both legacy `nodeid` and new `depth` fields.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_nodeid("test_nodeid");
node.set_depth(5);
BEAST_EXPECT(!validateLedgerNode(node));
}
// Valid: new `id` field.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_id("test_id");
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: new `depth` field.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(5);
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: `depth` at minimum depth.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(0);
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: `depth` at arbitrary depth between minimum and maximum.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(10);
BEAST_EXPECT(validateLedgerNode(node));
}
// Valid: `depth` at maximum depth.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(SHAMap::leafDepth);
BEAST_EXPECT(validateLedgerNode(node));
}
// Invalid: `depth` is greater than maximum depth.
{
protocol::TMLedgerNode node;
node.set_nodedata("test_data");
node.set_depth(SHAMap::leafDepth + 1);
BEAST_EXPECT(!validateLedgerNode(node));
}
}
void
testGetTreeNode()
{
testcase("getTreeNode");
// Valid: inner node. It must have at least one child for `serializeNode` to work.
{
auto const innerNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
auto const childNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
innerNode->setChild(0, childNode);
auto const innerData = serializeNode(innerNode);
auto const result = getTreeNode(innerData);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT((*result)->isInner());
}
// Valid: leaf node.
{
auto const leafItem = makeTestItem(12345);
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
auto const leafData = serializeNode(leafNode);
auto result = getTreeNode(leafData);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT((*result)->isLeaf());
}
// Invalid: empty data.
{
auto const result = getTreeNode("");
BEAST_EXPECT(!result.has_value());
}
// Invalid: garbage data.
{
auto const result = getTreeNode("invalid");
BEAST_EXPECT(!result.has_value());
}
// Invalid: truncated data.
{
auto const leafItem = makeTestItem(54321);
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
// Truncate the data to trigger an exception in SHAMapTreeNode::makeAccountState when
// the data is used to deserialize the node.
uint256 const tag;
auto const leafData = serializeNode(leafNode).substr(0, tag.bytes - 1);
auto const result = getTreeNode(leafData);
BEAST_EXPECT(!result.has_value());
}
}
void
testGetSHAMapNodeID()
{
testcase("getSHAMapNodeID");
{
// Tests using inner nodes at various depths.
auto const innerNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
auto const childNode = intr_ptr::make_shared<SHAMapInnerNode>(1);
innerNode->setChild(0, childNode);
auto const innerData = serializeNode(innerNode);
// Valid: legacy `nodeid` field at arbitrary depth.
{
auto const innerDepth = 3;
auto const innerID = SHAMapNodeID::createID(innerDepth, uint256{});
protocol::TMLedgerNode node;
node.set_nodedata(innerData);
node.set_nodeid(innerID.getRawString());
auto const result = getSHAMapNodeID(node, innerNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == innerID);
}
// Valid: new `id` field at minimum depth.
{
auto const innerDepth = 0;
auto const innerID = SHAMapNodeID::createID(innerDepth, uint256{});
protocol::TMLedgerNode node;
node.set_nodedata(innerData);
node.set_id(innerID.getRawString());
auto const result = getSHAMapNodeID(node, innerNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == innerID);
}
// Invalid: new `depth` field should not be used for inner nodes.
{
protocol::TMLedgerNode node;
node.set_nodedata(innerData);
node.set_depth(10);
auto const result = getSHAMapNodeID(node, innerNode);
BEAST_EXPECT(!result.has_value());
}
}
{
// Tests using leaf nodes at various depths.
auto const leafItem = makeTestItem(12345);
auto const leafNode = intr_ptr::make_shared<SHAMapAccountStateLeafNode>(leafItem, 1);
auto const leafData = serializeNode(leafNode);
auto const leafKey = leafItem->key();
// Valid: legacy `nodeid` field at arbitrary depth.
{
auto const leafDepth = 5;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(leafData);
ledgerNode.set_nodeid(leafID.getRawString());
auto result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == leafID);
}
// Invalid: new `id` field should not be used for leaf nodes.
{
auto const leafDepth = 5;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(leafData);
ledgerNode.set_id(leafID.getRawString());
auto result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(!result.has_value());
}
// Valid: new `depth` field at minimum depth.
{
auto const leafDepth = 0;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode node;
node.set_nodedata(leafData);
node.set_depth(leafDepth);
auto result = getSHAMapNodeID(node, leafNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == leafID);
}
// Valid: new `depth` field at arbitrary depth between minimum and maximum.
{
auto const leafDepth = 10;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(leafData);
ledgerNode.set_depth(leafDepth);
auto result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == leafID);
}
// Valid: new `depth` field at maximum depth.
// Note that we do not test a depth greater than the maximum depth, because the proto
// message is assumed to have been validated by the time the getSHAMapNodeID function is
// called.
{
auto const leafDepth = SHAMap::leafDepth;
auto const leafID = SHAMapNodeID::createID(leafDepth, leafKey);
protocol::TMLedgerNode node;
node.set_nodedata(leafData);
node.set_depth(leafDepth);
auto result = getSHAMapNodeID(node, leafNode);
BEAST_EXPECT(result.has_value());
BEAST_EXPECT(*result == leafID);
}
// Invalid: legacy `nodeid` field where the node ID is inconsistent with the key.
{
auto const otherItem = makeTestItem(54321);
auto const otherNode =
intr_ptr::make_shared<SHAMapAccountStateLeafNode>(otherItem, 1);
auto const otherData = serializeNode(otherNode);
auto const otherKey = otherItem->key();
auto const otherDepth = 1;
auto const otherID = SHAMapNodeID::createID(otherDepth, otherKey);
protocol::TMLedgerNode ledgerNode;
ledgerNode.set_nodedata(otherData);
ledgerNode.set_nodeid(otherID.getRawString());
auto result = getSHAMapNodeID(ledgerNode, leafNode);
BEAST_EXPECT(!result.has_value());
}
}
}
public:
void
run() override
{
testValidateLedgerNode();
testGetTreeNode();
testGetSHAMapNodeID();
}
};
BEAST_DEFINE_TESTSUITE(LedgerNodeHelpers, app, xrpl);
} // namespace tests
} // namespace xrpl

View File

@@ -1,6 +1,7 @@
#include <test/jtx.h>
#include <test/jtx/envconfig.h>
#include <xrpld/app/ledger/LedgerMaster.h>
#include <xrpld/app/main/Application.h>
#include <xrpld/app/main/NodeStoreScheduler.h>
#include <xrpld/app/misc/SHAMapStore.h>
@@ -10,6 +11,8 @@
#include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
#include <xrpl/protocol/jss.h>
#include <thread>
namespace xrpl {
namespace test {
@@ -20,10 +23,8 @@ class SHAMapStore_test : public beast::unit_test::suite
static auto
onlineDelete(std::unique_ptr<Config> cfg)
{
cfg->LEDGER_HISTORY = deleteInterval;
auto& section = cfg->section(ConfigSection::nodeDatabase());
section.set("online_delete", std::to_string(deleteInterval));
return cfg;
using namespace jtx;
return online_delete(std::move(cfg), deleteInterval);
}
static auto
@@ -563,6 +564,158 @@ public:
BEAST_EXPECT(dbr->getName() == "3");
}
void
testLedgerGaps()
{
// Note that this test is intentionally very similar to
// LedgerMaster_test::testCompleteLedgerRange, but has a different
// focus.
testcase("Wait for ledger gaps to fill in");
using namespace test::jtx;
Env env{*this, envconfig(onlineDelete)};
std::map<LedgerIndex, uint256> hashes;
auto failureMessage = [&](char const* label, auto expected, auto actual) {
std::stringstream ss;
ss << label << ": Expected: " << expected << ", Got: " << actual;
return ss.str();
};
auto const alice = Account("alice");
env.fund(XRP(1000), alice);
env.close();
auto& lm = env.app().getLedgerMaster();
LedgerIndex minSeq = 2;
LedgerIndex maxSeq = env.closed()->header().seq;
auto& store = env.app().getSHAMapStore();
store.rendezvous();
LedgerIndex lastRotated = store.getLastRotated();
BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
// Close enough ledgers to rotate a few times
while (maxSeq < 20)
{
for (int t = 0; t < 3; ++t)
{
env(noop(alice));
}
env.close();
store.rendezvous();
++maxSeq;
if (maxSeq + 1 == lastRotated + deleteInterval)
{
using namespace std::chrono_literals;
// The next ledger will trigger a rotation. Delete the
// current ledger from LedgerMaster.
std::this_thread::sleep_for(100ms);
LedgerIndex const deleteSeq = maxSeq;
while (!lm.haveLedger(deleteSeq))
{
std::this_thread::sleep_for(100ms);
}
lm.clearLedger(deleteSeq);
auto expectedRange = [](auto minSeq, auto deleteSeq, auto maxSeq) {
std::stringstream expectedRange;
expectedRange << minSeq << "-" << (deleteSeq - 1);
if (deleteSeq + 1 == maxSeq)
expectedRange << "," << maxSeq;
else if (deleteSeq < maxSeq)
expectedRange << "," << (deleteSeq + 1) << "-" << maxSeq;
return expectedRange.str();
};
BEAST_EXPECTS(
lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
failureMessage(
"Complete ledgers",
expectedRange(minSeq, deleteSeq, maxSeq),
lm.getCompleteLedgers()));
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 1);
// Close another ledger, which will trigger a rotation, but the
// rotation will be stuck until the missing ledger is filled in.
env.close();
// DO NOT CALL rendezvous()! You'll end up with a deadlock.
++maxSeq;
// Nothing has changed
BEAST_EXPECTS(
store.getLastRotated() == lastRotated,
failureMessage("lastRotated", lastRotated, store.getLastRotated()));
BEAST_EXPECTS(
lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
failureMessage(
"Complete ledgers",
expectedRange(minSeq, deleteSeq, maxSeq),
lm.getCompleteLedgers()));
// Close 5 more ledgers, waiting one second in between to
// simulate the ledger making progress while online delete waits
// for the missing ledger to be filled in.
// This ensures the healthWait check has time to run and
// detect the gap.
for (int l = 0; l < 5; ++l)
{
env.close();
// DO NOT CALL rendezvous()! You'll end up with a deadlock.
++maxSeq;
// Nothing has changed
BEAST_EXPECTS(
store.getLastRotated() == lastRotated,
failureMessage("lastRotated", lastRotated, store.getLastRotated()));
BEAST_EXPECTS(
lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
failureMessage(
"Complete Ledgers",
expectedRange(minSeq, deleteSeq, maxSeq),
lm.getCompleteLedgers()));
std::this_thread::sleep_for(1s);
}
// Put the missing ledger back in LedgerMaster
lm.setLedgerRangePresent(deleteSeq, deleteSeq);
// Wait for the rotation to finish
store.rendezvous();
minSeq = lastRotated;
lastRotated = deleteSeq + 1;
}
BEAST_EXPECT(maxSeq != lastRotated + deleteInterval);
BEAST_EXPECTS(
env.closed()->header().seq == maxSeq,
failureMessage("maxSeq", maxSeq, env.closed()->header().seq));
BEAST_EXPECTS(
store.getLastRotated() == lastRotated,
failureMessage("lastRotated", lastRotated, store.getLastRotated()));
std::stringstream expectedRange;
expectedRange << minSeq << "-" << maxSeq;
BEAST_EXPECTS(
lm.getCompleteLedgers() == expectedRange.str(),
failureMessage("CompleteLedgers", expectedRange.str(), lm.getCompleteLedgers()));
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
}
}
void
run() override
{
@@ -570,6 +723,7 @@ public:
testAutomatic();
testCanDelete();
testRotate();
testLedgerGaps();
}
};

View File

@@ -57,6 +57,17 @@ envconfig(F&& modfunc, Args&&... args)
return modfunc(envconfig(), std::forward<Args>(args)...);
}
/// @brief adjust config to enable online_delete
///
/// @param cfg config instance to be modified
///
/// @param deleteInterval how many new ledgers should be available before
/// rotating. Defaults to 8, because the standalone minimum is 8.
///
/// @return unique_ptr to Config instance
std::unique_ptr<Config>
online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval = 8);
/// @brief adjust config so no admin ports are enabled
///
/// this is intended for use with envconfig, as in

View File

@@ -53,6 +53,15 @@ setupConfigForUnitTests(Config& cfg)
namespace jtx {
std::unique_ptr<Config>
online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval)
{
cfg->LEDGER_HISTORY = deleteInterval;
auto& section = cfg->section(ConfigSection::nodeDatabase());
section.set("online_delete", std::to_string(deleteInterval));
return cfg;
}
std::unique_ptr<Config>
no_admin(std::unique_ptr<Config> cfg)
{

View File

@@ -60,8 +60,8 @@ public:
negotiateProtocolVersion("RTXP/1.2, XRPL/2.0, XRPL/2.1") == make_protocol(2, 1));
BEAST_EXPECT(negotiateProtocolVersion("XRPL/2.2") == make_protocol(2, 2));
BEAST_EXPECT(
negotiateProtocolVersion("RTXP/1.2, XRPL/2.3, XRPL/2.4, XRPL/999.999") ==
make_protocol(2, 3));
negotiateProtocolVersion("RTXP/1.2, XRPL/2.2, XRPL/2.3, XRPL/999.999") ==
make_protocol(2, 2));
BEAST_EXPECT(negotiateProtocolVersion("XRPL/999.999, WebSocket/1.0") == std::nullopt);
BEAST_EXPECT(negotiateProtocolVersion("") == std::nullopt);
}

View File

@@ -99,17 +99,14 @@ public:
destination.setSynching();
{
std::vector<std::tuple<SHAMapNodeID, Blob, bool>> a;
std::vector<std::pair<SHAMapNodeID, Blob>> a;
BEAST_EXPECT(source.getNodeFat(SHAMapNodeID(), a, rand_bool(eng_), rand_int(eng_, 2)));
unexpected(a.empty(), "NodeSize");
auto node = SHAMapTreeNode::makeFromWire(makeSlice(std::get<1>(a[0])));
if (!node)
fail("", __FILE__, __LINE__);
BEAST_EXPECT(
destination.addRootNode(source.getHash(), std::move(node), nullptr).isGood());
BEAST_EXPECT(destination.addRootNode(source.getHash(), makeSlice(a[0].second), nullptr)
.isGood());
}
do
@@ -123,7 +120,7 @@ public:
break;
// get as many nodes as possible based on this information
std::vector<std::tuple<SHAMapNodeID, Blob, bool>> b;
std::vector<std::pair<SHAMapNodeID, Blob>> b;
for (auto& it : nodesMissing)
{
@@ -145,10 +142,7 @@ public:
// Don't use BEAST_EXPECT here b/c it will be called a
// non-deterministic number of times and the number of tests run
// should be deterministic
auto node = SHAMapTreeNode::makeFromWire(makeSlice(std::get<1>(b[i])));
if (!node)
fail("", __FILE__, __LINE__);
if (!destination.addKnownNode(std::get<0>(b[i]), std::move(node), nullptr)
if (!destination.addKnownNode(b[i].first, makeSlice(b[i].second), nullptr)
.isUseful())
fail("", __FILE__, __LINE__);
}

View File

@@ -9,7 +9,6 @@
#include <mutex>
#include <set>
#include <string_view>
#include <utility>
namespace xrpl {
@@ -132,16 +131,16 @@ private:
processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& data);
bool
takeHeader(std::string_view data);
takeHeader(std::string const& data);
void
receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san);
receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode&);
bool
takeTxRootNode(std::string_view data, SHAMapAddNode& san);
takeTxRootNode(Slice const& data, SHAMapAddNode&);
bool
takeAsRootNode(std::string_view data, SHAMapAddNode& san);
takeAsRootNode(Slice const& data, SHAMapAddNode&);
std::vector<uint256>
neededTxHashes(int max, SHAMapSyncFilter* filter) const;

View File

@@ -105,7 +105,10 @@ public:
failedSave(std::uint32_t seq, uint256 const& hash);
std::string
getCompleteLedgers();
getCompleteLedgers() const;
std::size_t
missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const;
/** Apply held transactions to the open ledger
This is normally called as we close the ledger.
@@ -322,7 +325,7 @@ private:
// A set of transactions to replay during the next close
std::unique_ptr<LedgerReplay> replayData;
std::recursive_mutex mCompleteLock;
std::recursive_mutex mutable mCompleteLock;
RangeSet<std::uint32_t> mCompleteLedgers;
// Publish thread is running.

View File

@@ -3,7 +3,6 @@
#include <xrpld/app/ledger/InboundLedgers.h>
#include <xrpld/app/ledger/LedgerMaster.h>
#include <xrpld/app/ledger/TransactionStateSF.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpld/app/main/Application.h>
#include <xrpld/overlay/Overlay.h>
@@ -763,7 +762,7 @@ InboundLedger::filterNodes(
*/
// data must not have hash prefix
bool
InboundLedger::takeHeader(std::string_view data)
InboundLedger::takeHeader(std::string const& data)
{
// Return value: true=normal, false=bad data
JLOG(journal_.trace()) << "got header acquiring ledger " << hash_;
@@ -851,31 +850,20 @@ InboundLedger::receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san)
{
auto const f = filter.get();
for (auto const& ledger_node : packet.nodes())
for (auto const& node : packet.nodes())
{
auto treeNode = getTreeNode(ledger_node.nodedata());
if (!treeNode)
{
JLOG(journal_.warn()) << "Got invalid node data";
san.incInvalid();
return;
}
auto const nodeID = deserializeSHAMapNodeID(node.nodeid());
auto const nodeID = getSHAMapNodeID(ledger_node, *treeNode);
if (!nodeID)
{
JLOG(journal_.warn()) << "Got invalid node id";
san.incInvalid();
return;
}
throw std::runtime_error("data does not properly deserialize");
if (nodeID->isRoot())
{
san += map.addRootNode(rootHash, std::move(*treeNode), f);
san += map.addRootNode(rootHash, makeSlice(node.nodedata()), f);
}
else
{
san += map.addKnownNode(*nodeID, std::move(*treeNode), f);
san += map.addKnownNode(*nodeID, makeSlice(node.nodedata()), f);
}
if (!san.isGood())
@@ -915,7 +903,7 @@ InboundLedger::receiveNode(protocol::TMLedgerData& packet, SHAMapAddNode& san)
Call with a lock
*/
bool
InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
InboundLedger::takeAsRootNode(Slice const& data, SHAMapAddNode& san)
{
if (failed_ || mHaveState)
{
@@ -931,17 +919,9 @@ InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
// LCOV_EXCL_STOP
}
auto treeNode = getTreeNode(data);
if (!treeNode)
{
JLOG(journal_.warn()) << "Got invalid node data";
san.incInvalid();
return false;
}
AccountStateSF filter(mLedger->stateMap().family().db(), app_.getLedgerMaster());
san += mLedger->stateMap().addRootNode(
SHAMapHash{mLedger->header().accountHash}, std::move(*treeNode), &filter);
san +=
mLedger->stateMap().addRootNode(SHAMapHash{mLedger->header().accountHash}, data, &filter);
return san.isGood();
}
@@ -949,7 +929,7 @@ InboundLedger::takeAsRootNode(std::string_view data, SHAMapAddNode& san)
Call with a lock
*/
bool
InboundLedger::takeTxRootNode(std::string_view data, SHAMapAddNode& san)
InboundLedger::takeTxRootNode(Slice const& data, SHAMapAddNode& san)
{
if (failed_ || mHaveTransactions)
{
@@ -965,17 +945,8 @@ InboundLedger::takeTxRootNode(std::string_view data, SHAMapAddNode& san)
// LCOV_EXCL_STOP
}
auto treeNode = getTreeNode(data);
if (!treeNode)
{
JLOG(journal_.warn()) << "Got invalid node data";
san.incInvalid();
return false;
}
TransactionStateSF filter(mLedger->txMap().family().db(), app_.getLedgerMaster());
san += mLedger->txMap().addRootNode(
SHAMapHash{mLedger->header().txHash}, std::move(*treeNode), &filter);
san += mLedger->txMap().addRootNode(SHAMapHash{mLedger->header().txHash}, data, &filter);
return san.isGood();
}
@@ -1072,13 +1043,13 @@ InboundLedger::processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& p
}
if (!mHaveState && (packet.nodes().size() > 1) &&
!takeAsRootNode(packet.nodes(1).nodedata(), san))
!takeAsRootNode(makeSlice(packet.nodes(1).nodedata()), san))
{
JLOG(journal_.warn()) << "Included AS root invalid";
}
if (!mHaveTransactions && (packet.nodes().size() > 2) &&
!takeTxRootNode(packet.nodes(2).nodedata(), san))
!takeTxRootNode(makeSlice(packet.nodes(2).nodedata()), san))
{
JLOG(journal_.warn()) << "Included TX root invalid";
}
@@ -1109,13 +1080,13 @@ InboundLedger::processData(std::shared_ptr<Peer> peer, protocol::TMLedgerData& p
ScopedLockType const sl(mtx_);
// Verify nodes are complete
for (auto const& ledger_node : packet.nodes())
// Verify node IDs and data are complete
for (auto const& node : packet.nodes())
{
if (!validateLedgerNode(ledger_node))
if (!node.has_nodeid() || !node.has_nodedata())
{
JLOG(journal_.warn()) << "Got malformed ledger node";
peer->charge(Resource::feeMalformedRequest, "ledger_node");
JLOG(journal_.warn()) << "Got bad node";
peer->charge(Resource::feeMalformedRequest, "ledger_data bad node");
return -1;
}
}

View File

@@ -1,6 +1,5 @@
#include <xrpld/app/ledger/InboundLedgers.h>
#include <xrpld/app/ledger/LedgerMaster.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpld/app/main/Application.h>
#include <xrpl/basics/DecayingSample.h>
@@ -224,21 +223,23 @@ public:
Serializer s;
try
{
for (auto const& ledger_node : packet_ptr->nodes())
for (int i = 0; i < packet_ptr->nodes().size(); ++i)
{
if (!validateLedgerNode(ledger_node))
auto const& node = packet_ptr->nodes(i);
if (!node.has_nodeid() || !node.has_nodedata())
return;
auto const treeNode = getTreeNode(ledger_node.nodedata());
if (!treeNode)
auto newNode = SHAMapTreeNode::makeFromWire(makeSlice(node.nodedata()));
if (!newNode)
return;
auto const tn = *treeNode;
s.erase();
tn->serializeWithPrefix(s);
newNode->serializeWithPrefix(s);
app_.getLedgerMaster().addFetchPack(
tn->getHash().as_uint256(), std::make_shared<Blob>(s.begin(), s.end()));
newNode->getHash().as_uint256(), std::make_shared<Blob>(s.begin(), s.end()));
}
}
catch (std::exception const&) // NOLINT(bugprone-empty-catch)

View File

@@ -1,6 +1,5 @@
#include <xrpld/app/ledger/InboundLedgers.h>
#include <xrpld/app/ledger/InboundTransactions.h>
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpld/app/ledger/detail/TransactionAcquire.h>
#include <xrpld/app/main/Application.h>
@@ -132,35 +131,26 @@ public:
return;
}
std::vector<std::pair<SHAMapNodeID, intr_ptr::SharedPtr<SHAMapTreeNode>>> data;
std::vector<std::pair<SHAMapNodeID, Slice>> data;
data.reserve(packet.nodes().size());
for (auto const& ledger_node : packet.nodes())
for (auto const& node : packet.nodes())
{
if (!validateLedgerNode(ledger_node))
if (!node.has_nodeid() || !node.has_nodedata())
{
JLOG(j_.warn()) << "Got malformed ledger node";
peer->charge(Resource::feeMalformedRequest, "ledger_node");
peer->charge(Resource::feeMalformedRequest, "ledger_data");
return;
}
auto treeNode = getTreeNode(ledger_node.nodedata());
if (!treeNode)
auto const id = deserializeSHAMapNodeID(node.nodeid());
if (!id)
{
JLOG(j_.warn()) << "Got invalid node data";
peer->charge(Resource::feeInvalidData, "node_data");
peer->charge(Resource::feeInvalidData, "ledger_data");
return;
}
auto const nodeID = getSHAMapNodeID(ledger_node, *treeNode);
if (!nodeID)
{
JLOG(j_.warn()) << "Got invalid node id";
peer->charge(Resource::feeInvalidData, "node_id");
return;
}
data.emplace_back(std::make_pair(*nodeID, std::move(*treeNode)));
data.emplace_back(std::make_pair(*id, makeSlice(node.nodedata())));
}
if (!ta->takeNodes(data, peer).isUseful())

View File

@@ -1529,12 +1529,34 @@ LedgerMaster::getPublishedLedger()
}
std::string
LedgerMaster::getCompleteLedgers()
LedgerMaster::getCompleteLedgers() const
{
std::lock_guard const sl(mCompleteLock);
return to_string(mCompleteLedgers);
}
std::size_t
LedgerMaster::missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const
{
// Make a copy of the range to avoid holding the lock
auto const range = [&] {
std::lock_guard sl(mCompleteLock);
return mCompleteLedgers;
}();
std::size_t missing = 0;
for (LedgerIndex idx = first; idx <= last; ++idx)
{
if (!boost::icl::contains(range, idx))
{
++missing;
}
}
return missing;
}
std::optional<NetClock::time_point>
LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
{

View File

@@ -1,95 +0,0 @@
#include <xrpld/app/ledger/detail/LedgerNodeHelpers.h>
#include <xrpl/basics/IntrusivePointer.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/safe_cast.h>
#include <xrpl/beast/utility/instrumentation.h>
#include <xrpl/protocol/messages.h>
#include <xrpl/shamap/SHAMap.h>
#include <xrpl/shamap/SHAMapLeafNode.h>
#include <xrpl/shamap/SHAMapNodeID.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <optional>
#include <string>
namespace xrpl {
bool
validateLedgerNode(protocol::TMLedgerNode const& ledger_node)
{
if (!ledger_node.has_nodedata())
return false;
if (ledger_node.has_nodeid())
return !ledger_node.has_id() && !ledger_node.has_depth();
return ledger_node.has_id() ||
(ledger_node.has_depth() && ledger_node.depth() <= SHAMap::leafDepth);
}
std::optional<intr_ptr::SharedPtr<SHAMapTreeNode>>
getTreeNode(std::string_view data)
{
auto const slice = makeSlice(data);
try
{
auto treeNode = SHAMapTreeNode::makeFromWire(slice);
if (!treeNode)
return std::nullopt;
return treeNode;
}
catch (std::exception const&)
{
return std::nullopt;
}
}
std::optional<SHAMapNodeID>
getSHAMapNodeID(
protocol::TMLedgerNode const& ledger_node,
intr_ptr::SharedPtr<SHAMapTreeNode> const& treeNode)
{
if (ledger_node.has_id() || ledger_node.has_depth())
{
if (treeNode->isInner())
{
if (!ledger_node.has_id())
return std::nullopt;
return deserializeSHAMapNodeID(ledger_node.id());
}
if (treeNode->isLeaf())
{
if (!ledger_node.has_depth())
return std::nullopt;
auto const key =
safe_downcast<SHAMapLeafNode const*>(treeNode.get())->peekItem()->key();
return SHAMapNodeID::createID(ledger_node.depth(), key);
}
UNREACHABLE("xrpl::getSHAMapNodeID : tree node is neither inner nor leaf");
return std::nullopt;
}
if (!ledger_node.has_nodeid())
return std::nullopt;
auto nodeID = deserializeSHAMapNodeID(ledger_node.nodeid());
if (!nodeID.has_value())
return std::nullopt;
if (treeNode->isLeaf())
{
auto const key = safe_downcast<SHAMapLeafNode const*>(treeNode.get())->peekItem()->key();
auto const expected_id = SHAMapNodeID::createID(static_cast<int>(nodeID->getDepth()), key);
if (nodeID->getNodeID() != expected_id.getNodeID())
return std::nullopt;
}
return nodeID;
}
} // namespace xrpl

View File

@@ -1,75 +0,0 @@
#pragma once
#include <xrpl/basics/IntrusivePointer.h>
#include <xrpl/shamap/SHAMapNodeID.h>
#include <xrpl/shamap/SHAMapTreeNode.h>
#include <optional>
#include <string_view>
namespace protocol {
class TMLedgerNode;
} // namespace protocol
namespace xrpl {
/**
* @brief Validates a ledger node proto message.
*
* This function checks whether a ledger node has the expected fields (for non-ledger base data):
* - The node must have `nodedata`.
* - If the legacy `nodeid` field is present then the new `id` and `depth` fields must not be
* present.
* - If the new `id` or `depth` fields are present (it is a oneof field, so only one of the two can
* be set) then the legacy `nodeid` must not be present.
* - If the `depth` field is present then it must be between 0 and SHAMap::leafDepth (inclusive).
*
* @param ledger_node The ledger node to validate.
* @return true if the ledger node has the expected fields, false otherwise.
*/
[[nodiscard]] bool
validateLedgerNode(protocol::TMLedgerNode const& ledger_node);
/**
* @brief Deserializes a SHAMapTreeNode from wire format data.
*
* This function attempts to create a SHAMapTreeNode from the provided data string. If the data is
* malformed or deserialization fails, the function returns std::nullopt instead of throwing an
* exception.
*
* @param data The serialized node data in wire format.
* @return An optional containing the deserialized tree node if successful, or std::nullopt if
* deserialization fails.
*/
[[nodiscard]] std::optional<intr_ptr::SharedPtr<SHAMapTreeNode>>
getTreeNode(std::string_view data);
/**
* @brief Extracts or reconstructs the SHAMapNodeID from a ledger node proto message.
*
* This function retrieves the SHAMapNodeID for a tree node, with behavior that depends on which
* field is set and the node type (inner vs. leaf).
*
* When the legacy `nodeid` field is set in the message:
* - For all nodes: Deserializes the node ID from the field.
* - For leaf nodes: Validates that the node ID is consistent with the leaf's key.
*
* When the new `id` or `depth` field is set in the message:
* - For inner nodes: Deserializes the node ID from the `id` field. Note that root nodes are also
* inner nodes.
* - For leaf nodes: Reconstructs the node ID using both the depth from the `depth` field and the
* key from the leaf node's item.
*
* @param ledger_node The validated protocol message containing the ledger node data.
* @param treeNode The deserialized tree node (inner or leaf node).
* @return An optional containing the node ID if extraction/reconstruction succeeds, or std::nullopt
* if the required fields are missing or validation fails.
* @note This function expects that the caller has already validated the ledger node by calling the
* `validateLedgerNode` function and obtained a valid tree node by calling `getTreeNode`.
*/
[[nodiscard]] std::optional<SHAMapNodeID>
getSHAMapNodeID(
protocol::TMLedgerNode const& ledger_node,
intr_ptr::SharedPtr<SHAMapTreeNode> const& treeNode);
} // namespace xrpl

View File

@@ -158,7 +158,7 @@ TransactionAcquire::trigger(std::shared_ptr<Peer> const& peer)
SHAMapAddNode
TransactionAcquire::takeNodes(
std::vector<std::pair<SHAMapNodeID, intr_ptr::SharedPtr<SHAMapTreeNode>>> const& data,
std::vector<std::pair<SHAMapNodeID, Slice>> const& data,
std::shared_ptr<Peer> const& peer)
{
ScopedLockType const sl(mtx_);
@@ -182,7 +182,7 @@ TransactionAcquire::takeNodes(
ConsensusTransSetSF sf(app_, app_.getTempNodeCache());
for (auto& d : data)
for (auto const& d : data)
{
if (d.first.isRoot())
{

View File

@@ -20,8 +20,8 @@ public:
SHAMapAddNode
takeNodes(
std::vector<std::pair<SHAMapNodeID, intr_ptr::SharedPtr<SHAMapTreeNode>>> const& data,
std::shared_ptr<Peer> const& peer);
std::vector<std::pair<SHAMapNodeID, Slice>> const& data,
std::shared_ptr<Peer> const&);
void
init(int startPeers);

View File

@@ -263,6 +263,17 @@ SHAMapStoreImp::run()
bool const readyToRotate = validatedSeq >= lastRotated + deleteInterval_ &&
canDelete_ >= lastRotated - 1 && healthWait() == keepGoing;
JLOG(journal_.debug()) << "run: Setting lastGoodValidatedLedger_ to " << validatedSeq;
{
// Note that this is set after the healthWait() check, so that we
// don't start the rotation until the validated ledger is fully
// processed. It is not guaranteed to be done at this point. It also
// allows the testLedgerGaps unit test to work.
std::unique_lock<std::mutex> lock(mutex_);
lastGoodValidatedLedger_ = validatedSeq;
}
// will delete up to (not including) lastRotated
if (readyToRotate)
{
@@ -270,7 +281,8 @@ SHAMapStoreImp::run()
<< lastRotated << " deleteInterval " << deleteInterval_
<< " canDelete_ " << canDelete_ << " state "
<< app_.getOPs().strOperatingMode(false) << " age "
<< ledgerMaster_->getValidatedLedgerAge().count() << 's';
<< ledgerMaster_->getValidatedLedgerAge().count()
<< "s. Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
clearPrior(lastRotated);
if (healthWait() == stopping)
@@ -330,7 +342,9 @@ SHAMapStoreImp::run()
clearCaches(validatedSeq);
});
JLOG(journal_.warn()) << "finished rotation " << validatedSeq;
JLOG(journal_.warn()) << "finished rotation. validatedSeq: " << validatedSeq
<< ", lastRotated: " << lastRotated
<< ". Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
}
}
}
@@ -570,22 +584,40 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
SHAMapStoreImp::HealthResult
SHAMapStoreImp::healthWait()
{
auto index = ledgerMaster_->getValidLedgerIndex();
auto age = ledgerMaster_->getValidatedLedgerAge();
OperatingMode mode = netOPs_->getOperatingMode();
std::unique_lock lock(mutex_);
while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_))
auto numMissing =
ledgerMaster_->missingFromCompleteLedgerRange(lastGoodValidatedLedger_, index);
while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_ || numMissing > 0))
{
// this value shouldn't change, so grab it while we have the
// lock
auto const lastGood = lastGoodValidatedLedger_;
lock.unlock();
JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count()
<< "s for node to stabilize. state: "
<< app_.getOPs().strOperatingMode(mode, false) << ". age "
<< age.count() << 's';
auto const stream =
mode != OperatingMode::FULL || age > ageThreshold_ ? journal_.warn() : journal_.info();
JLOG(stream) << "Waiting " << recoveryWaitTime_.count()
<< "s for node to stabilize. state: "
<< app_.getOPs().strOperatingMode(mode, false) << ". age " << age.count()
<< "s. Missing ledgers: " << numMissing << ". Expect: " << lastGood << "-"
<< index << ". Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
std::this_thread::sleep_for(recoveryWaitTime_);
index = ledgerMaster_->getValidLedgerIndex();
age = ledgerMaster_->getValidatedLedgerAge();
mode = netOPs_->getOperatingMode();
numMissing = ledgerMaster_->missingFromCompleteLedgerRange(lastGood, index);
lock.lock();
}
JLOG(journal_.debug()) << "healthWait: Setting lastGoodValidatedLedger_ to " << index;
lastGoodValidatedLedger_ = index;
return stop_ ? stopping : keepGoing;
}

View File

@@ -70,6 +70,11 @@ private:
std::thread thread_;
bool stop_ = false;
bool healthy_ = true;
// Used to prevent ledger gaps from forming during online deletion. Keeps
// track of the last validated ledger that was processed without gaps. There
// are no guarantees about gaps while online delete is not running. For
// that, use advisory_delete and check for gaps externally.
LedgerIndex lastGoodValidatedLedger_ = 0;
mutable std::condition_variable cond_;
mutable std::condition_variable rendezvous_;
mutable std::mutex mutex_;
@@ -83,11 +88,11 @@ private:
std::uint32_t deleteBatch_ = 100;
std::chrono::milliseconds backOff_{100};
std::chrono::seconds ageThreshold_{60};
/// If the node is out of sync during an online_delete healthWait()
/// call, sleep the thread for this time, and continue checking until
/// recovery.
/// If the node is out of sync, or any recent ledgers are not
/// available during an online_delete healthWait() call, sleep
/// the thread for this time, and continue checking until recovery.
/// See also: "recovery_wait_seconds" in xrpld-example.cfg
std::chrono::seconds recoveryWaitTime_{5};
std::chrono::seconds recoveryWaitTime_{1};
// these do not exist upon SHAMapStore creation, but do exist
// as of run() or before
@@ -204,6 +209,8 @@ private:
enum HealthResult { stopping, keepGoing };
[[nodiscard]] HealthResult
healthWait();
bool
hasCompleteRange(LedgerIndex first, LedgerIndex last);
public:
void

View File

@@ -17,7 +17,6 @@ enum class ProtocolFeature {
ValidatorListPropagation,
ValidatorList2Propagation,
LedgerReplay,
LedgerNodeDepth,
};
/** Represents a peer connection in the overlay. */

View File

@@ -30,7 +30,6 @@
#include <mutex>
#include <numeric>
#include <sstream>
#include <tuple>
using namespace std::chrono_literals;
@@ -499,8 +498,6 @@ PeerImp::supportsFeature(ProtocolFeature f) const
return protocol_ >= make_protocol(2, 1);
case ProtocolFeature::ValidatorList2Propagation:
return protocol_ >= make_protocol(2, 2);
case ProtocolFeature::LedgerNodeDepth:
return protocol_ >= make_protocol(2, 3);
case ProtocolFeature::LedgerReplay:
return ledgerReplayEnabled_;
}
@@ -3391,19 +3388,13 @@ PeerImp::processLedgerRequest(std::shared_ptr<protocol::TMGetLedger> const& m)
std::uint32_t const defaultDepth = isHighLatency() ? 2 : 1;
auto const queryDepth{m->has_querydepth() ? m->querydepth() : defaultDepth};
std::vector<std::tuple<SHAMapNodeID, Blob, bool>> data;
auto const useLedgerNodeDepth = supportsFeature(ProtocolFeature::LedgerNodeDepth);
std::vector<std::pair<SHAMapNodeID, Blob>> data;
for (int i = 0;
i < m->nodeids_size() && ledgerData.nodes_size() < Tuning::softMaxReplyNodes;
++i)
{
auto const shaMapNodeId{deserializeSHAMapNodeID(m->nodeids(i))};
if (!shaMapNodeId.has_value())
{
JLOG(p_journal_.error()) << "processLedgerRequest: Invalid SHAMap node ID";
return;
}
data.clear();
data.reserve(Tuning::softMaxReplyNodes);
@@ -3419,28 +3410,9 @@ PeerImp::processLedgerRequest(std::shared_ptr<protocol::TMGetLedger> const& m)
{
if (ledgerData.nodes_size() >= Tuning::hardMaxReplyNodes)
break;
protocol::TMLedgerNode* node{ledgerData.add_nodes()};
auto const& nodeData = std::get<1>(d);
node->set_nodedata(nodeData.data(), nodeData.size());
// When the LedgerNodeDepth protocol feature is not supported by the peer,
// we always set the `nodeid` field. However, when it is supported then we
// set the `id` field for inner nodes and the `depth` field for leaf nodes.
auto const& nodeID = std::get<0>(d);
if (!useLedgerNodeDepth)
{
node->set_nodeid(nodeID.getRawString());
}
else if (std::get<2>(d))
{
node->set_depth(nodeID.getDepth());
}
else
{
node->set_id(nodeID.getRawString());
}
node->set_nodeid(d.first.getRawString());
node->set_nodedata(d.second.data(), d.second.size());
}
}
else

View File

@@ -20,7 +20,6 @@ namespace xrpl {
constexpr ProtocolVersion const supportedProtocolList[]{
{2, 1},
{2, 2},
{2, 3},
};
// This ugly construct ensures that supportedProtocolList is sorted in strictly