mirror of
https://github.com/XRPLF/rippled.git
synced 2026-10-21 19:18:22 +00:00
Compare commits
128
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3c6279b735 | ||
|
|
55b265a8c2 | ||
|
|
02e40e7a00 | ||
|
|
f14395427e | ||
|
|
b707bac74f | ||
|
|
da2b5d5eaf | ||
|
|
4de4902200 | ||
|
|
2ea1f09dcb | ||
|
|
68b4b894de | ||
|
|
defd83e5a6 | ||
|
|
452e31a05f | ||
|
|
1798262d7a | ||
|
|
cd633b9ac9 | ||
|
|
5c5f7d315a | ||
|
|
0f7493ce50 | ||
|
|
40f61f828a | ||
|
|
108277f4b7 | ||
|
|
651bb207b4 | ||
|
|
c45363fd8b | ||
|
|
f1744cb76e | ||
|
|
bcbaa4df07 | ||
|
|
3cd357949c | ||
|
|
a9027bb997 | ||
|
|
0a6da4de74 | ||
|
|
6a05339c6c | ||
|
|
0c41a87604 | ||
|
|
f5938e4097 | ||
|
|
c7d10c1b60 | ||
|
|
84e2a155b9 | ||
|
|
cbade49976 | ||
|
|
97fbea23cc | ||
|
|
e6055ddbe1 | ||
|
|
a5c76fde2d | ||
|
|
eed6527946 | ||
|
|
ddbc5f1a24 | ||
|
|
97a1824537 | ||
|
|
b44c87613c | ||
|
|
b3bbf50d40 | ||
|
|
53246e5b15 | ||
|
|
7e82b0660f | ||
|
|
8ef1bdd346 | ||
|
|
3a2d19f980 | ||
|
|
04eca6d6c3 | ||
|
|
768aef30dc | ||
|
|
97cddd6721 | ||
|
|
e4dbd7de3a | ||
|
|
646d2ce620 | ||
|
|
6641896e9f | ||
|
|
d6df33ff03 | ||
|
|
1d7669f528 | ||
|
|
30640a626f | ||
|
|
5a5ad8673a | ||
|
|
8f4e9c25d8 | ||
|
|
d8870162eb | ||
|
|
8b1a2282c3 | ||
|
|
0219c01b33 | ||
|
|
00606bec1a | ||
|
|
2bc17c3e73 | ||
|
|
0229c294a9 | ||
|
|
184fe173fb | ||
|
|
16f7b263fd | ||
|
|
f6b51f0b8b | ||
|
|
07b36871c7 | ||
|
|
638ae08eeb | ||
|
|
24be48ee84 | ||
|
|
e20f448a71 | ||
|
|
f6c80fef68 | ||
|
|
a0c12420b5 | ||
|
|
1245254cac | ||
|
|
b60636169a | ||
|
|
d6022fbc4d | ||
|
|
295b74da1c | ||
|
|
faa2bf583f | ||
|
|
da260fa42b | ||
|
|
53788b193d | ||
|
|
227f1b4d9c | ||
|
|
26b66957ec | ||
|
|
54e62a621f | ||
|
|
53628b70c0 | ||
|
|
6fec2c11bf | ||
|
|
04108a030c | ||
|
|
4a4fded2eb | ||
|
|
c8e767afa4 | ||
|
|
00eeb0a005 | ||
|
|
a18839d92d | ||
|
|
8c594c7ed9 | ||
|
|
ebd810b184 | ||
|
|
3e4e56d6bb | ||
|
|
76da5d4475 | ||
|
|
c0d0fd0d97 | ||
|
|
9aebb5ebea | ||
|
|
796f2f8f1e | ||
|
|
b190f2b14f | ||
|
|
551a19b10d | ||
|
|
b0a940a383 | ||
|
|
e302e4eeed | ||
|
|
1a4a40ebb8 | ||
|
|
7f55dd390c | ||
|
|
9403736199 | ||
|
|
028783661d | ||
|
|
21890d9daf | ||
|
|
060957ed39 | ||
|
|
1381483c7a | ||
|
|
3e4bdf2782 | ||
|
|
e3c8996e44 | ||
|
|
6099940c2c | ||
|
|
0db7b766e6 | ||
|
|
ea6226b8b9 | ||
|
|
d5bfe94f15 | ||
|
|
eae0a35415 | ||
|
|
2ad4def35f | ||
|
|
f7f50caa6e | ||
|
|
3e54e7d00b | ||
|
|
827b50f169 | ||
|
|
58a59c37ed | ||
|
|
986065c16f | ||
|
|
37cb4cdbe3 | ||
|
|
2ddf6ee148 | ||
|
|
8ce4f71427 | ||
|
|
6e1eb88e6e | ||
|
|
49cdc105de | ||
|
|
636d2d4851 | ||
|
|
7d7275847d | ||
|
|
f0fd6ad85e | ||
|
|
5d8fd9824e | ||
|
|
346ea40f69 | ||
|
|
8809bdf3f0 | ||
|
|
deaf596494 |
No files matched your search
+8
-1
@@ -90,6 +90,13 @@ CheckOptions:
|
||||
readability-identifier-naming.ClassCase: CamelCase
|
||||
readability-identifier-naming.StructCase: CamelCase
|
||||
readability-identifier-naming.UnionCase: CamelCase
|
||||
readability-identifier-naming.TypeAliasCase: CamelCase
|
||||
readability-identifier-naming.TypedefCase: CamelCase
|
||||
# Member type names mandated by std/Boost concepts. Subtrees that need more
|
||||
# than these carry their own .clang-tidy (include/xrpl/{beast,basics,protocol,
|
||||
# peerfinder}, src/test/beast); one-off cases use an inline NOLINT.
|
||||
readability-identifier-naming.TypeAliasIgnoredRegexp: "^(const_iterator|const_reference|difference_type|duration|iterator|iterator_category|key_type|mapped_type|period|pointer|reference|rep|result_type|size_type|time_point|value_type)$"
|
||||
readability-identifier-naming.TypedefIgnoredRegexp: "^(const_iterator|const_reference|difference_type|duration|iterator|iterator_category|key_type|mapped_type|period|pointer|reference|rep|result_type|size_type|time_point|value_type)$"
|
||||
readability-identifier-naming.EnumCase: CamelCase
|
||||
readability-identifier-naming.EnumConstantCase: CamelCase
|
||||
readability-identifier-naming.ScopedEnumConstantCase: CamelCase
|
||||
@@ -122,5 +129,5 @@ CheckOptions:
|
||||
readability-identifier-naming.GlobalFunctionIgnoredRegexp: "^(to_string|hash_append|tuple_hash)$"
|
||||
|
||||
HeaderFilterRegex: '^.*/(tests?|xrpl|xrpld)/.*\.(h|hpp|ipp)$'
|
||||
ExcludeHeaderFilterRegex: '^.*/protocol_autogen/.*\.(h|hpp)$'
|
||||
ExcludeHeaderFilterRegex: '^.*/protocol_autogen/.*\.(h|hpp)$|^.*\.pb\.h$'
|
||||
WarningsAsErrors: "*"
|
||||
+3
-1
@@ -64,6 +64,7 @@ words:
|
||||
- blindings
|
||||
- bookdir
|
||||
- Bougalis
|
||||
- bthomee
|
||||
- Britto
|
||||
- Btrfs
|
||||
- Buildx
|
||||
@@ -141,6 +142,7 @@ words:
|
||||
- hwrap
|
||||
- ifndef
|
||||
- inequation
|
||||
- Injectivity
|
||||
- insuf
|
||||
- insuff
|
||||
- invasively
|
||||
@@ -253,6 +255,7 @@ words:
|
||||
- queuable
|
||||
- Raphson
|
||||
- rcflags
|
||||
- reencrypted
|
||||
- replayer
|
||||
- repodata
|
||||
- repomd
|
||||
@@ -366,7 +369,6 @@ words:
|
||||
- venv
|
||||
- vfalco
|
||||
- vinnie
|
||||
- vkeylet
|
||||
- wasmi
|
||||
- wextra
|
||||
- wptr
|
||||
|
||||
@@ -8,3 +8,7 @@ watch_file rust-toolchain.toml
|
||||
watch_dir conan
|
||||
|
||||
use flake
|
||||
|
||||
# Optional, untracked local overrides. To use a different shell, put e.g.
|
||||
# `use flake .#formal-verification` in .envrc.local.
|
||||
source_env_if_exists .envrc.local
|
||||
@@ -5,6 +5,18 @@
|
||||
# This file is sorted in reverse chronological order, with the most recent commits at the top.
|
||||
# The commits listed here are ignored by git blame, which is useful for formatting-only commits that would otherwise obscure the history of changes to a file.
|
||||
|
||||
# chore: CamelCase for typedef/using in `clang-tidy` (#8177)
|
||||
97a1824537d20d82d25bf151a37a7a4532ebf085
|
||||
# chore: Rename CamelCase namespaces to snake_case (#7933)
|
||||
06488c1318d96f56d0536251bee08ac85fa7fdd3
|
||||
# style: Unify style for all Doxygen comments (#7776)
|
||||
73b6852a122854140336e6e6bc30a3a4b41aa5fd
|
||||
# style: More clang-tidy identifier renaming (#7290)
|
||||
a830ab10efed8d3e59ef2fc15d66efdf9c6bb0d8
|
||||
# refactor: Rename static constants (#7120)
|
||||
5b6e8b6f93b19c1e3f6a3467a25639031d9d9a53
|
||||
# chore: More fixes for bad renames (#7092)
|
||||
7afdd71a54d562b32a50b29a5aa00bb997dc9053
|
||||
# refactor: Enable clang-tidy `readability-identifier-naming` check (#6571)
|
||||
8995564ed6b9e453e144bb663303072a3c1ba305
|
||||
# refactor: Enable remaining clang-tidy `cppcoreguidelines` checks (#6538)
|
||||
|
||||
@@ -15,30 +15,25 @@ outputs:
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
# A tag names its own version. Anything else takes it from BuildInfo.cpp and
|
||||
# appends the commit hash as build metadata, joined with a plus sign because a
|
||||
# Conan version cannot contain two hyphens.
|
||||
# A tag names its own version. Anything else is a development build named by
|
||||
# its commit hash, matching what cmake/XrplVersion.cmake derives: the head of
|
||||
# a pull request rather than the merge commit GitHub creates for it.
|
||||
- name: Determine version
|
||||
id: version
|
||||
shell: bash
|
||||
env:
|
||||
IS_TAG: ${{ startsWith(github.ref, 'refs/tags/') }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
SHA: ${{ github.sha }}
|
||||
SHA: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
run: |
|
||||
if [[ "${IS_TAG}" == "true" ]]; then
|
||||
version="${REF_NAME}"
|
||||
else
|
||||
version="$(awk -F'"' '/versionString =/ { print $2 }' src/libxrpl/protocol/BuildInfo.cpp)"
|
||||
if [[ -z "${version}" ]]; then
|
||||
echo "Unable to read versionString from BuildInfo.cpp." >&2
|
||||
exit 1
|
||||
fi
|
||||
version="${version}+${SHA:0:7}"
|
||||
version="0.0.0-dev+${SHA:0:7}"
|
||||
fi
|
||||
|
||||
echo "version=${version}" | tee -a "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Determine release channel and package release
|
||||
id: release_info
|
||||
uses: XRPLF/actions/release-info@7cc0e4a8d9d0b838f92c48d312856b190341bbba
|
||||
uses: XRPLF/actions/release-info@a9f2eeca6fb3980ba3a84cf68566f1c69ad30674
|
||||
@@ -40,10 +40,11 @@ runs:
|
||||
# Unlike the Linux nix images, macOS needs no SSL_CERT_FILE: it has its
|
||||
# own trust store, and pinning would break TLS to hosts relying on it.
|
||||
|
||||
# Workspace-local, so `cleanup-workspace` clears it, but not the
|
||||
# `.conan2` prepare-runner hands the system toolchain: that Conan is a
|
||||
# different version, and the two would migrate each other's cache.
|
||||
echo "CONAN_HOME=${{ github.workspace }}/.conan2-nix" >>"${GITHUB_ENV}"
|
||||
# In RUNNER_TEMP, which the runner empties per job, like the `.conan2`
|
||||
# prepare-runner hands the system toolchain - but under its own name:
|
||||
# that Conan is a different version, and the two would migrate each
|
||||
# other's cache.
|
||||
echo "CONAN_HOME=${RUNNER_TEMP}/.conan2-nix" >>"${GITHUB_ENV}"
|
||||
|
||||
# Config, profiles and remote, exactly as the dev shell sets them up on
|
||||
# entry; the `setup-conan` action is skipped for this toolchain.
|
||||
|
||||
@@ -18,7 +18,7 @@ If too broad, please consider splitting into multiple PRs.
|
||||
If there is a relevant task or issue, please link it here.
|
||||
-->
|
||||
|
||||
### Context of Change
|
||||
## Context of Change
|
||||
|
||||
<!--
|
||||
Please include the context of a change.
|
||||
@@ -29,7 +29,7 @@ If a refactor, how is this better than the previous implementation?
|
||||
If there is a spec or design document for this feature, please link it here.
|
||||
-->
|
||||
|
||||
### API Impact
|
||||
## API Impact
|
||||
|
||||
<!--
|
||||
Please check [x] relevant options, delete irrelevant ones.
|
||||
|
||||
@@ -1,19 +1,34 @@
|
||||
# Levelization
|
||||
|
||||
Levelization is the term used to describe efforts to prevent xrpld from having or creating cyclic dependencies.
|
||||
Levelization is the term used to describe efforts to prevent xrpld from
|
||||
having or creating cyclic dependencies.
|
||||
|
||||
xrpld code is organized into directories under `src/xrpld`, `src/libxrpl` (and `src/test`) representing modules. The modules are intended to be organized into "tiers" or "levels" such that a module from one level can only include code from lower levels. Additionally, a module in one level should never include code in an `impl` or `detail` folder of any level other than its own.
|
||||
xrpld code is organized into directories under `src/xrpld`, `src/libxrpl` (and
|
||||
`src/test`) representing modules. The modules are intended to be
|
||||
organized into "tiers" or "levels" such that a module from one level can
|
||||
only include code from lower levels. Additionally, a module
|
||||
in one level should never include code in an `impl` or `detail` folder of any level
|
||||
other than its own.
|
||||
|
||||
The codebase is split into two main areas:
|
||||
|
||||
- **libxrpl** (`src/libxrpl`, `include/xrpl`): Reusable library modules with public interfaces
|
||||
- **xrpld** (`src/xrpld`): Application-specific implementation code
|
||||
|
||||
Unfortunately, over time, enforcement of levelization has been inconsistent, so the current state of the code doesn't necessarily reflect these rules. Whenever possible, developers should refactor any levelization violations they find (by moving files or individual classes). At the very least, don't make things worse.
|
||||
Unfortunately, over time, enforcement of levelization has been
|
||||
inconsistent, so the current state of the code doesn't necessarily
|
||||
reflect these rules. Whenever possible, developers should refactor any
|
||||
levelization violations they find (by moving files or individual
|
||||
classes). At the very least, don't make things worse.
|
||||
|
||||
The table below summarizes the _desired_ division of modules, based on the current state of the xrpld code. The levels are numbered from the bottom up with the lower level, lower numbered, more independent modules listed first, and the higher level, higher numbered modules with more dependencies listed later.
|
||||
The table below summarizes the _desired_ division of modules, based on the current
|
||||
state of the xrpld code. The levels are numbered from
|
||||
the bottom up with the lower level, lower numbered, more independent
|
||||
modules listed first, and the higher level, higher numbered modules with
|
||||
more dependencies listed later.
|
||||
|
||||
**tl;dr:** The modules listed first are more independent than the modules listed later.
|
||||
**tl;dr:** The modules listed first are more independent than the modules
|
||||
listed later.
|
||||
|
||||
## libxrpl Modules (Reusable Libraries)
|
||||
|
||||
@@ -40,37 +55,80 @@ The table below summarizes the _desired_ division of modules, based on the curre
|
||||
|
||||
## Test Modules
|
||||
|
||||
| Level / Tier | Module(s) |
|
||||
| --- | --- |
|
||||
| 11 | test/jtx test/beast test/csf |
|
||||
| 12 | test/unit_test |
|
||||
| 13 | test/crypto test/conditions test/json test/resource test/shamap test/peerfinder test/basics test/overlay |
|
||||
| 14 | test |
|
||||
| 15 | test/net test/protocol test/ledger test/consensus test/core test/server test/nodestore |
|
||||
| 16 | test/rpc test/app |
|
||||
| Level / Tier | Module(s) |
|
||||
| ------------ | -------------------------------------------------------------------------------------------------------- |
|
||||
| 11 | test/jtx test/beast test/csf |
|
||||
| 12 | test/unit_test |
|
||||
| 13 | test/crypto test/conditions test/json test/resource test/shamap test/peerfinder test/basics test/overlay |
|
||||
| 14 | test |
|
||||
| 15 | test/net test/protocol test/ledger test/consensus test/core test/server test/nodestore |
|
||||
| 16 | test/rpc test/app |
|
||||
|
||||
(Note that `test` levelization is _much_ less important and _much_ less strictly enforced than `xrpl`/`xrpld` levelization, other than the requirement that `test` code should _never_ be included in `xrpl` or `xrpld` code.)
|
||||
(Note that `test` levelization is _much_ less important and _much_ less
|
||||
strictly enforced than `xrpl`/`xrpld` levelization, other than the requirement
|
||||
that `test` code should _never_ be included in `xrpl` or `xrpld` code.)
|
||||
|
||||
## Validation
|
||||
|
||||
The [levelization](generate.py) script takes no parameters, reads no environment variables, and can be run from any directory, as long as it is in the expected location in the xrpld repo. It can be run at any time from within a checked out repo, and will do an analysis of all the `#include`s in the xrpld source. The only caveat is that it runs much slower under Windows than in Linux. It hasn't yet been tested under MacOS. It generates many files of [results](results):
|
||||
The [levelization](generate.py) script takes no parameters,
|
||||
reads no environment variables, and can be run from any directory,
|
||||
as long as it is in the expected location in the xrpld repo.
|
||||
It can be run at any time from within a checked out repo, and will
|
||||
do an analysis of all the `#include`s in
|
||||
the xrpld source. The only caveat is that it runs much slower
|
||||
under Windows than in Linux. It hasn't yet been tested under MacOS.
|
||||
It generates many files of [results](results):
|
||||
|
||||
- `rawincludes.txt`: The raw dump of the `#includes`
|
||||
- `paths.txt`: A second dump grouping the source module to the destination module, de-duped, and with frequency counts.
|
||||
- `includes/`: A directory where each file represents a module and contains a list of modules and counts that the module _includes_.
|
||||
- `included_by/`: Similar to `includes/`, but the other way around. Each file represents a module and contains a list of modules and counts that _include_ the module.
|
||||
- [`loops.txt`](results/loops.txt): A list of direct loops detected between modules as they actually exist, as opposed to how they are desired as described above. In a perfect repo, this file will be empty. This file is committed to the repo, and is used by the [levelization Github workflow](../../workflows/reusable-check-levelization.yml) to validate that nothing changed.
|
||||
- [`ordering.txt`](results/ordering.txt): A list showing relationships between modules where there are no loops as they actually exist, as opposed to how they are desired as described above. This file is committed to the repo, and is used by the [levelization Github workflow](../../workflows/reusable-check-levelization.yml) to validate that nothing changed.
|
||||
- [`levelization.yml`](../../workflows/reusable-check-levelization.yml) Github Actions workflow to test that levelization loops haven't changed. Unfortunately, if changes are detected, it can't tell if they are improvements or not, so if you have resolved any issues or done anything else to improve levelization, run `generate.py`, and commit the updated results.
|
||||
- `paths.txt`: A second dump grouping the source module
|
||||
to the destination module, de-duped, and with frequency counts.
|
||||
- `includes/`: A directory where each file represents a module and
|
||||
contains a list of modules and counts that the module _includes_.
|
||||
- `included_by/`: Similar to `includes/`, but the other way around. Each
|
||||
file represents a module and contains a list of modules and counts
|
||||
that _include_ the module.
|
||||
- [`loops.txt`](results/loops.txt): A list of direct loops detected
|
||||
between modules as they actually exist, as opposed to how they are
|
||||
desired as described above. In a perfect repo, this file will be
|
||||
empty.
|
||||
This file is committed to the repo, and is used by the [levelization
|
||||
Github workflow](../../workflows/reusable-check-levelization.yml) to validate
|
||||
that nothing changed.
|
||||
- [`ordering.txt`](results/ordering.txt): A list showing relationships
|
||||
between modules where there are no loops as they actually exist, as
|
||||
opposed to how they are desired as described above.
|
||||
This file is committed to the repo, and is used by the [levelization
|
||||
Github workflow](../../workflows/reusable-check-levelization.yml) to validate
|
||||
that nothing changed.
|
||||
- [`levelization.yml`](../../workflows/reusable-check-levelization.yml)
|
||||
Github Actions workflow to test that levelization loops haven't
|
||||
changed. Unfortunately, if changes are detected, it can't tell if
|
||||
they are improvements or not, so if you have resolved any issues or
|
||||
done anything else to improve levelization, run `generate.py`,
|
||||
and commit the updated results.
|
||||
|
||||
The `loops.txt` and `ordering.txt` files relate the modules using comparison signs, which indicate the number of times each module is included in the other.
|
||||
The `loops.txt` and `ordering.txt` files relate the modules
|
||||
using comparison signs, which indicate the number of times each
|
||||
module is included in the other.
|
||||
|
||||
- `A > B` means that A should probably be at a higher level than B, because B is included in A significantly more than A is included in B. These results can be included in both `loops.txt` and `ordering.txt`. Because `ordering.txt`only includes relationships where B is not included in A at all, it will only include these types of results.
|
||||
- `A ~= B` means that A and B are included in each other a different number of times, but the values are so close that the script can't definitively say that one should be above the other. These results will only be included in `loops.txt`.
|
||||
- `A == B` means that A and B include each other the same number of times, so the script has no clue which should be higher. These results will only be included in `loops.txt`.
|
||||
- `A > B` means that A should probably be at a higher level than B,
|
||||
because B is included in A significantly more than A is included in B.
|
||||
These results can be included in both `loops.txt` and `ordering.txt`.
|
||||
Because `ordering.txt`only includes relationships where B is not
|
||||
included in A at all, it will only include these types of results.
|
||||
- `A ~= B` means that A and B are included in each other a different
|
||||
number of times, but the values are so close that the script can't
|
||||
definitively say that one should be above the other. These results
|
||||
will only be included in `loops.txt`.
|
||||
- `A == B` means that A and B include each other the same number of
|
||||
times, so the script has no clue which should be higher. These results
|
||||
will only be included in `loops.txt`.
|
||||
|
||||
The committed files hide the detailed values intentionally, to prevent false alarms and merging issues, and because it's easy to get those details locally.
|
||||
The committed files hide the detailed values intentionally, to
|
||||
prevent false alarms and merging issues, and because it's easy to
|
||||
get those details locally.
|
||||
|
||||
1. Run `generate.py`
|
||||
2. Grep the modules in `paths.txt`.
|
||||
- For example, if a cycle is found `A ~= B`, simply `grep -w A .github/scripts/levelization/results/paths.txt | grep -w B`
|
||||
- For example, if a cycle is found `A ~= B`, simply `grep -w
|
||||
A .github/scripts/levelization/results/paths.txt | grep -w B`
|
||||
@@ -82,7 +82,6 @@ test.app > xrpl.tx
|
||||
test.basics > test.jtx
|
||||
test.basics > xrpl.basics
|
||||
test.basics > xrpl.core
|
||||
test.basics > xrpld.rpc
|
||||
test.basics > xrpl.json
|
||||
test.basics > xrpl.protocol
|
||||
test.beast > xrpl.basics
|
||||
@@ -113,6 +112,7 @@ test.jtx > xrpl.config
|
||||
test.jtx > xrpl.core
|
||||
test.jtx > xrpld.app
|
||||
test.jtx > xrpld.core
|
||||
test.jtx > xrpld.overlay
|
||||
test.jtx > xrpld.rpc
|
||||
test.jtx > xrpl.json
|
||||
test.jtx > xrpl.ledger
|
||||
@@ -194,6 +194,9 @@ tests.libxrpl > xrpl.resource
|
||||
tests.libxrpl > xrpl.server
|
||||
tests.libxrpl > xrpl.shamap
|
||||
tests.libxrpl > xrpl.tx
|
||||
tests.xrpld > xrpld.rpc
|
||||
tests.xrpld > xrpl.json
|
||||
tests.xrpld > xrpl.protocol
|
||||
xrpl.conditions > xrpl.basics
|
||||
xrpl.conditions > xrpl.protocol
|
||||
xrpl.config > xrpl.basics
|
||||
@@ -286,10 +289,10 @@ xrpld.perflog > xrpl.basics
|
||||
xrpld.perflog > xrpl.config
|
||||
xrpld.perflog > xrpl.core
|
||||
xrpld.perflog > xrpld.app
|
||||
xrpld.perflog > xrpld.rpc
|
||||
xrpld.perflog > xrpl.json
|
||||
xrpld.perflog > xrpl.nodestore
|
||||
xrpld.perflog > xrpl.protocol
|
||||
xrpld.perflog > xrpl.server
|
||||
xrpld.rpc > xrpl.basics
|
||||
xrpld.rpc > xrpl.config
|
||||
xrpld.rpc > xrpl.core
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Exit the script as soon as an error occurs.
|
||||
set -euo pipefail
|
||||
|
||||
# This script fails if <head> merges a release back into <base>,
|
||||
# but also adds commits that aren't on a release or staging branch, see RELEASING.md.
|
||||
# Merge commits are allowed.
|
||||
# Usage: .github/scripts/releasing/check-merge-back-commits.sh <base> <head>
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "Usage: $0 <base> <head>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BASE=$1
|
||||
HEAD=$2
|
||||
|
||||
SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
|
||||
# shellcheck source=.github/scripts/releasing/common.sh
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
load_release_branches
|
||||
|
||||
# A PR is a merge-back if some of its commits are on a release or staging branch.
|
||||
PR_COUNT=$(git rev-list --no-merges --count "${BASE}..${HEAD}")
|
||||
NEW_COUNT=$(git rev-list --no-merges --count "${BASE}..${HEAD}" --not "${BRANCHES[@]}")
|
||||
if ((NEW_COUNT == PR_COUNT)); then
|
||||
echo "This PR doesn't merge a release back."
|
||||
exit 0
|
||||
fi
|
||||
if ((NEW_COUNT == 0)); then
|
||||
echo "This merge-back adds no commits of its own."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "This PR merges a release back, but also adds commits that aren't on a release or staging branch:"
|
||||
git log --no-merges --format=' %h %s' "${BASE}..${HEAD}" --not "${BRANCHES[@]}"
|
||||
echo
|
||||
echo "Make these changes in a separate PR, see RELEASING.md."
|
||||
exit 1
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Exit the script as soon as an error occurs.
|
||||
set -euo pipefail
|
||||
|
||||
# This script fails if the commits in <base>..<head> copy commits from a release or staging branch
|
||||
# (e.g. by rebasing or cherry-picking them) instead of merging that branch, see RELEASING.md.
|
||||
# Commits are compared by patch-id,
|
||||
# and only against release and staging commits that <head> does not already contain.
|
||||
# Usage: .github/scripts/releasing/check-no-copied-release-commits.sh <base> <head>
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "Usage: $0 <base> <head>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BASE=$1
|
||||
HEAD=$2
|
||||
|
||||
patch_ids() {
|
||||
git log --no-merges --patch --no-color --no-ext-diff "$@" | git patch-id --stable | sort
|
||||
}
|
||||
|
||||
SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
|
||||
# shellcheck source=.github/scripts/releasing/common.sh
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
load_release_branches
|
||||
|
||||
# Each line is "<patch-id> <commit>".
|
||||
RELEASE_PATCHES=$(patch_ids "${BRANCHES[@]}" --not "${HEAD}")
|
||||
PR_PATCHES=$(patch_ids "${BASE}..${HEAD}")
|
||||
|
||||
# Each line is "<patch-id> <release commit> <PR commit>".
|
||||
COPIES=$(join <(echo "${RELEASE_PATCHES}") <(echo "${PR_PATCHES}"))
|
||||
if [ -z "${COPIES}" ]; then
|
||||
echo "No copied release commits found."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "These commits copy release commits instead of merging them:"
|
||||
while read -r _ RELEASE_COMMIT PR_COMMIT; do
|
||||
echo " $(git log -1 --format='%h %s' "${PR_COMMIT}") (copies ${RELEASE_COMMIT:0:10})"
|
||||
done <<<"${COPIES}"
|
||||
echo
|
||||
echo "Merge the release tag (or branch) instead, see RELEASING.md."
|
||||
exit 1
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Exit the script as soon as an error occurs.
|
||||
set -euo pipefail
|
||||
|
||||
# This script fails if <head> contains release or staging commits that <base> does not,
|
||||
# i.e. if <head> merges a release back into <base>.
|
||||
# Used in the merge queue, which squashes PRs and would drop the merge commit, see RELEASING.md.
|
||||
# Usage: .github/scripts/releasing/check-no-merge-back.sh <base> <head>
|
||||
|
||||
if [ "$#" -ne 2 ]; then
|
||||
echo "Usage: $0 <base> <head>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BASE=$1
|
||||
HEAD=$2
|
||||
|
||||
SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
|
||||
# shellcheck source=.github/scripts/releasing/common.sh
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
load_release_branches
|
||||
|
||||
RELEASE_COMMITS=$(git rev-list "${BRANCHES[@]}" --not "${BASE}")
|
||||
HEAD_COMMITS=$(git rev-list "${BASE}..${HEAD}")
|
||||
# The release commits in <head>, newest first.
|
||||
MERGED=$(grep -xF -f <(echo "${RELEASE_COMMITS}") <<<"${HEAD_COMMITS}" || true)
|
||||
if [ -z "${MERGED}" ]; then
|
||||
echo "No release commits are merged back."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "This PR merges $(wc -l <<<"${MERGED}" | tr -d ' ') release commits back, e.g.:"
|
||||
head -5 <<<"${MERGED}" | xargs git log --no-walk --format=' %h %s'
|
||||
echo
|
||||
echo "Merge-backs must not go through the merge queue, which squashes them."
|
||||
echo "Fast-forward develop to the PR branch instead, see RELEASING.md."
|
||||
exit 1
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Exit the script as soon as an error occurs.
|
||||
set -euo pipefail
|
||||
|
||||
# This script fails if a final release (a tag like 3.4.0) on a release branch
|
||||
# is not merged back into <develop> within a few days, see RELEASING.md.
|
||||
# Usage: .github/scripts/releasing/check-releases-merged.sh <develop>
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "Usage: $0 <develop>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DEVELOP=$1
|
||||
GRACE_DAYS=3
|
||||
|
||||
mapfile -t MERGED_ARGS < <(git for-each-ref --format='--merged=%(refname)' 'refs/remotes/*/release/*')
|
||||
if [ "${#MERGED_ARGS[@]}" -eq 0 ]; then
|
||||
echo "Error: No release branches found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Tags on a release branch that <develop> does not contain.
|
||||
TAGS=$(git for-each-ref --format='%(refname:short) %(creatordate:unix)' \
|
||||
"${MERGED_ARGS[@]}" --no-merged="${DEVELOP}" 'refs/tags/[0-9]*')
|
||||
|
||||
MISSING=0
|
||||
while read -r TAG CREATED; do
|
||||
if ! [[ "${TAG}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
continue
|
||||
fi
|
||||
if (($(date +%s) - CREATED < GRACE_DAYS * 86400)); then
|
||||
echo "${TAG}: not merged yet, still within the ${GRACE_DAYS}-day grace period."
|
||||
else
|
||||
echo "${TAG}: not merged into develop."
|
||||
MISSING=1
|
||||
fi
|
||||
done <<<"${TAGS}"
|
||||
|
||||
if [ "${MISSING}" -ne 0 ]; then
|
||||
echo
|
||||
echo "Merge the missing releases back into develop, see RELEASING.md."
|
||||
exit 1
|
||||
fi
|
||||
echo "No releases past the grace period are missing from develop."
|
||||
@@ -0,0 +1,12 @@
|
||||
# shellcheck shell=bash
|
||||
|
||||
# Helpers shared by the release checks in this directory, see RELEASING.md.
|
||||
|
||||
# Sets BRANCHES to all release and staging branches, and fails if there are none.
|
||||
load_release_branches() {
|
||||
mapfile -t BRANCHES < <(git for-each-ref --format='%(refname)' 'refs/remotes/*/release/*' 'refs/remotes/*/staging/*')
|
||||
if [ "${#BRANCHES[@]}" -eq 0 ]; then
|
||||
echo "Error: No release or staging branches found."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
@@ -1,20 +1,41 @@
|
||||
## Renaming ripple(d) to xrpl(d)
|
||||
|
||||
In the initial phases of development of the XRPL, the open source codebase was called "rippled" and it remains with that name even today. Today, over 1000 nodes run the application, and code contributions have been submitted by developers located around the world. The XRPL community is larger than ever. In light of the decentralized and diversified nature of XRPL, we will rename any references to `ripple` and `rippled` to `xrpl` and `xrpld`, when appropriate.
|
||||
In the initial phases of development of the XRPL, the open source codebase was
|
||||
called "rippled" and it remains with that name even today. Today, over 1000
|
||||
nodes run the application, and code contributions have been submitted by
|
||||
developers located around the world. The XRPL community is larger than ever.
|
||||
In light of the decentralized and diversified nature of XRPL, we will rename any
|
||||
references to `ripple` and `rippled` to `xrpl` and `xrpld`, when appropriate.
|
||||
|
||||
See [here](https://xls.xrpl.org/xls/XLS-0095-rename-rippled-to-xrpld.html) for more information.
|
||||
See [here](https://xls.xrpl.org/xls/XLS-0095-rename-rippled-to-xrpld.html) for
|
||||
more information.
|
||||
|
||||
### Scripts
|
||||
|
||||
To facilitate this transition, there will be multiple scripts that developers can run on their own PRs and forks to minimize conflicts. Each script should be run from the repository root.
|
||||
To facilitate this transition, there will be multiple scripts that developers
|
||||
can run on their own PRs and forks to minimize conflicts. Each script should be
|
||||
run from the repository root.
|
||||
|
||||
1. `.github/scripts/rename/definitions.sh`: This script will rename all definitions, such as include guards, from `RIPPLE_XXX` and `RIPPLED_XXX` to `XRPL_XXX`.
|
||||
2. `.github/scripts/rename/copyright.sh`: This script will remove superfluous copyright notices.
|
||||
3. `.github/scripts/rename/cmake.sh`: This script will rename all CMake files from `RippleXXX.cmake` or `RippledXXX.cmake` to `XrplXXX.cmake`, and any references to `ripple` and `rippled` (with or without capital letters) to `xrpl` and `xrpld`, respectively. The name of the binary will remain as-is, and will only be renamed to `xrpld` by a later script.
|
||||
4. `.github/scripts/rename/binary.sh`: This script will rename the binary from `rippled` to `xrpld`, and reverses the symlink so that `rippled` points to the `xrpld` binary.
|
||||
5. `.github/scripts/rename/namespace.sh`: This script will rename the C++ namespaces from `ripple` to `xrpl`.
|
||||
6. `.github/scripts/rename/config.sh`: This script will rename the config from `rippled.cfg` to `xrpld.cfg`, and updating the code accordingly. The old filename will still be accepted.
|
||||
7. `.github/scripts/rename/docs.sh`: This script will rename any lingering references of `ripple(d)` to `xrpl(d)` in code, comments, and documentation.
|
||||
1. `.github/scripts/rename/definitions.sh`: This script will rename all
|
||||
definitions, such as include guards, from `RIPPLE_XXX` and `RIPPLED_XXX` to
|
||||
`XRPL_XXX`.
|
||||
2. `.github/scripts/rename/copyright.sh`: This script will remove superfluous
|
||||
copyright notices.
|
||||
3. `.github/scripts/rename/cmake.sh`: This script will rename all CMake files
|
||||
from `RippleXXX.cmake` or `RippledXXX.cmake` to `XrplXXX.cmake`, and any
|
||||
references to `ripple` and `rippled` (with or without capital letters) to
|
||||
`xrpl` and `xrpld`, respectively. The name of the binary will remain as-is,
|
||||
and will only be renamed to `xrpld` by a later script.
|
||||
4. `.github/scripts/rename/binary.sh`: This script will rename the binary from
|
||||
`rippled` to `xrpld`, and reverses the symlink so that `rippled` points to
|
||||
the `xrpld` binary.
|
||||
5. `.github/scripts/rename/namespace.sh`: This script will rename the C++
|
||||
namespaces from `ripple` to `xrpl`.
|
||||
6. `.github/scripts/rename/config.sh`: This script will rename the config from
|
||||
`rippled.cfg` to `xrpld.cfg`, and updating the code accordingly. The old
|
||||
filename will still be accepted.
|
||||
7. `.github/scripts/rename/docs.sh`: This script will rename any lingering
|
||||
references of `ripple(d)` to `xrpl(d)` in code, comments, and documentation.
|
||||
|
||||
You can run all these scripts from the repository root as follows:
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ ${SED_COMMAND} -i 's@ripple/@xrpld/@g' src/test/core/Config_test.cpp
|
||||
${SED_COMMAND} -i 's/Rippled/File/g' src/test/core/Config_test.cpp
|
||||
|
||||
# Restore the old config file name in the code that maintains support for now.
|
||||
${SED_COMMAND} -i 's/kConfigLegacyName = "xrpld.cfg"/kConfigLegacyName = "rippled.cfg"/g' src/xrpld/core/detail/Config.cpp
|
||||
${SED_COMMAND} -i 's/kConfigLegacyName = "xrpld.cfg"/kConfigLegacyName = "rippled.cfg"/g' src/xrpld/core/Config.h
|
||||
|
||||
# Restore an URL.
|
||||
${SED_COMMAND} -i 's/connect-your-xrpld-to-the-xrp-test-net.html/connect-your-rippled-to-the-xrp-test-net.html/g' cfg/xrpld-example.cfg
|
||||
|
||||
@@ -90,7 +90,7 @@ ${SED_COMMAND} -i 's/www.ripple.com/www.xrpl.org/g' src/test/protocol/Seed_test.
|
||||
# Restore specific changes.
|
||||
${SED_COMMAND} -i 's@b5efcc/src/xrpld@b5efcc/src/ripple@' include/xrpl/protocol/README.md
|
||||
${SED_COMMAND} -i 's/dbPrefix_ = "xrpldb"/dbPrefix_ = "rippledb"/' src/xrpld/app/misc/SHAMapStoreImp.h # cspell: disable-line
|
||||
${SED_COMMAND} -i 's/kConfigLegacyName = "xrpld.cfg"/kConfigLegacyName = "rippled.cfg"/' src/xrpld/core/detail/Config.cpp
|
||||
${SED_COMMAND} -i 's/kConfigLegacyName = "xrpld.cfg"/kConfigLegacyName = "rippled.cfg"/' src/xrpld/core/Config.h
|
||||
|
||||
popd
|
||||
echo "Renaming complete."
|
||||
@@ -15,6 +15,14 @@ _BASE_CMAKE_ARGS = [
|
||||
"-Drust=ON",
|
||||
]
|
||||
|
||||
# The package formats a config can be packaged as, each with its own
|
||||
# install-test job in reusable-package.yml.
|
||||
PACKAGE_TYPES = ("deb", "rpm")
|
||||
|
||||
# The package name a variant suffixes, as build_pkg.py's BASE_NAME spells it:
|
||||
# the two have to agree, or the artifact globs miss what was built.
|
||||
BASE_NAME = "xrpld"
|
||||
|
||||
# Maps sanitizer names (as used in cmake) to short config-name suffixes.
|
||||
_SANITIZER_SUFFIX: dict[str, str] = {
|
||||
"address": "asan",
|
||||
@@ -62,10 +70,20 @@ def get_cmake_args(build_type: str, extra_args: str) -> str:
|
||||
class PackageConfig:
|
||||
"""The 'package' map of a config whose binaries are also packaged."""
|
||||
|
||||
type: str # "deb" or "rpm"; has to match what the image provides
|
||||
type: str # has to match what the image provides
|
||||
# The packaging container image: a vanilla distro image, not the nix image
|
||||
# the config itself builds in.
|
||||
image: str
|
||||
# A flavour of the package, named xrpld-<variant>, for a config whose
|
||||
# binaries are not the plain release build. A variant needs no counterpart
|
||||
# in the other format.
|
||||
variant: str = ""
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
assert self.type in PACKAGE_TYPES, (
|
||||
f"unsupported package type {self.type!r}: "
|
||||
f"use one of {', '.join(PACKAGE_TYPES)}."
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
@@ -178,6 +196,8 @@ class PackagingEntry:
|
||||
validator_keys_artifact_name: str
|
||||
image: str
|
||||
package_type: str # "deb" or "rpm"; drives the format-specific steps
|
||||
package_variant: str # passed to build_pkg.py --variant; empty for xrpld
|
||||
package_name: str # the name it builds under, which the artifact globs use
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -267,12 +287,32 @@ def expand_linux_packaging(linux: LinuxFile) -> list[PackagingEntry]:
|
||||
validator_keys_artifact_name=f"validator-keys-{name}",
|
||||
image=cfg.package.image,
|
||||
package_type=cfg.package.type,
|
||||
package_variant=cfg.package.variant,
|
||||
package_name=(
|
||||
f"{BASE_NAME}-{cfg.package.variant}"
|
||||
if cfg.package.variant
|
||||
else BASE_NAME
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def package_names_by_type(entries: list[PackagingEntry]) -> dict[str, list[str]]:
|
||||
"""The names of the packages in 'entries', keyed by format.
|
||||
|
||||
Derived from the packaging matrix rather than listed again, so the packages
|
||||
the install-test jobs look for are the packages that were built.
|
||||
"""
|
||||
return {
|
||||
package_type: sorted(
|
||||
{e.package_name for e in entries if e.package_type == package_type}
|
||||
)
|
||||
for package_type in PACKAGE_TYPES
|
||||
}
|
||||
|
||||
|
||||
def expand_platform_matrix(pf: PlatformFile, minimal: bool) -> list[MatrixEntry]:
|
||||
"""Expand a PlatformFile (macOS or Windows) into matrix entries.
|
||||
|
||||
@@ -341,6 +381,10 @@ if __name__ == "__main__":
|
||||
|
||||
if args.packaging:
|
||||
matrix = expand_linux_packaging(LinuxFile.load(THIS_DIR / "linux.json"))
|
||||
# One list per format, so each install-test job installs the packages its
|
||||
# own format produced.
|
||||
for package_type, names in package_names_by_type(matrix).items():
|
||||
print(f"{package_type}_package_names={json.dumps(names)}")
|
||||
else:
|
||||
if args.config in ("linux", None):
|
||||
matrix += expand_linux_matrix(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"image_tag": "sha-473fe44",
|
||||
"image_tag": "sha-060957e",
|
||||
"configs": {
|
||||
"ubuntu": [
|
||||
{
|
||||
@@ -74,7 +74,20 @@
|
||||
"extra_cmake_args": "-Dvalidator_keys=ON",
|
||||
"package": {
|
||||
"type": "deb",
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-b6a8995"
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-3a2d19f"
|
||||
}
|
||||
},
|
||||
{
|
||||
"compiler": ["gcc"],
|
||||
"build_type": ["Release"],
|
||||
"arch": ["amd64"],
|
||||
"minimal": false,
|
||||
"suffix": "assert",
|
||||
"extra_cmake_args": "-Dvalidator_keys=ON -Dassert=ON",
|
||||
"package": {
|
||||
"type": "deb",
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-debian:sha-3a2d19f",
|
||||
"variant": "assert"
|
||||
}
|
||||
}
|
||||
],
|
||||
@@ -88,7 +101,7 @@
|
||||
"extra_cmake_args": "-Dvalidator_keys=ON",
|
||||
"package": {
|
||||
"type": "rpm",
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-rhel:sha-b6a8995"
|
||||
"image": "ghcr.io/xrplf/xrpld/packaging-rhel:sha-3a2d19f"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -5,15 +5,13 @@ on:
|
||||
branches:
|
||||
- develop
|
||||
paths:
|
||||
- ".github/workflows/build-nix-images.yml"
|
||||
- "flake.nix"
|
||||
- "flake.lock"
|
||||
- "rust-toolchain.toml"
|
||||
- "nix/**"
|
||||
- "!nix/docker/README.md"
|
||||
- "!nix/devshell.nix"
|
||||
- "!nix/check-tools/*.txt"
|
||||
- "bin/check-tools.sh"
|
||||
- "!nix/check-tools/**"
|
||||
- "bin/default-loader-path.sh"
|
||||
- "bin/install-sanitizer-libs.sh"
|
||||
pull_request:
|
||||
@@ -25,7 +23,7 @@ on:
|
||||
- "nix/**"
|
||||
- "!nix/docker/README.md"
|
||||
- "!nix/devshell.nix"
|
||||
- "!nix/check-tools/*.txt"
|
||||
- "!nix/check-tools/**"
|
||||
- "bin/check-tools.sh"
|
||||
- "bin/default-loader-path.sh"
|
||||
- "bin/install-sanitizer-libs.sh"
|
||||
@@ -60,7 +58,7 @@ jobs:
|
||||
base_image: debian:bookworm
|
||||
- name: rhel
|
||||
base_image: registry.access.redhat.com/ubi9/ubi:latest
|
||||
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@65d5a0bd72be4ecea95cff0673a6e0672ab5243a
|
||||
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@696384b292577293292daed06af0306d1b83bd7d
|
||||
with:
|
||||
image_name: xrpld/nix-${{ matrix.distro.name }}
|
||||
dockerfile: nix/docker/Dockerfile
|
||||
|
||||
@@ -7,12 +7,12 @@ on:
|
||||
paths:
|
||||
- ".github/workflows/build-packaging-images.yml"
|
||||
- "bin/install-packaging-tools.sh"
|
||||
- "package/docker/**"
|
||||
- "package/images/packaging/**"
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/build-packaging-images.yml"
|
||||
- "bin/install-packaging-tools.sh"
|
||||
- "package/docker/**"
|
||||
- "package/images/packaging/**"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
@@ -41,9 +41,9 @@ jobs:
|
||||
# AlmaLinux rather than UBI, which does not ship rpm-sign.
|
||||
- name: rhel
|
||||
base_image: almalinux:10
|
||||
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@65d5a0bd72be4ecea95cff0673a6e0672ab5243a
|
||||
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@696384b292577293292daed06af0306d1b83bd7d
|
||||
with:
|
||||
image_name: xrpld/packaging-${{ matrix.distro.name }}
|
||||
dockerfile: package/docker/Dockerfile
|
||||
dockerfile: package/images/packaging/Dockerfile
|
||||
base_image: ${{ matrix.distro.base_image }}
|
||||
push: ${{ github.event_name == 'push' }}
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@65d5a0bd72be4ecea95cff0673a6e0672ab5243a
|
||||
uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@696384b292577293292daed06af0306d1b83bd7d
|
||||
with:
|
||||
image_name: xrpld/pre-commit
|
||||
dockerfile: bin/pre-commit/Dockerfile
|
||||
|
||||
@@ -34,7 +34,7 @@ permissions:
|
||||
jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-473fe44
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-060957e
|
||||
permissions:
|
||||
contents: read
|
||||
# Needed to open an issue on scheduled failures.
|
||||
|
||||
@@ -13,7 +13,6 @@ on:
|
||||
- ready_for_review
|
||||
branches:
|
||||
- develop
|
||||
- "release-*"
|
||||
- "release/*"
|
||||
- "staging/*"
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ on:
|
||||
- ready_for_review
|
||||
branches:
|
||||
- develop
|
||||
- "release-*"
|
||||
- "release/*"
|
||||
- "staging/*"
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# This workflow checks that every final release on a release branch
|
||||
# has been merged back into develop, see RELEASING.md.
|
||||
name: Check releases merged
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 06:47 UTC every Monday.
|
||||
- cron: "47 6 * * 1"
|
||||
push:
|
||||
branches:
|
||||
- "develop"
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
OUTPUT_FILE: /tmp/releases-merged.txt
|
||||
ISSUE_FILE: /tmp/releases-merged-issue.md
|
||||
|
||||
jobs:
|
||||
releases-merged:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
# Needed to open an issue on scheduled failures.
|
||||
issues: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
# The check needs the full history of develop, the release branches and tags.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Check releases are merged into develop
|
||||
run: |
|
||||
set -o pipefail
|
||||
.github/scripts/releasing/check-releases-merged.sh origin/develop | tee "${OUTPUT_FILE}"
|
||||
|
||||
- name: Prepare issue body
|
||||
if: ${{ failure() && github.event_name == 'schedule' && github.repository == 'XRPLF/rippled' }}
|
||||
run: |
|
||||
{
|
||||
echo '```'
|
||||
cat "${OUTPUT_FILE}"
|
||||
echo '```'
|
||||
} >"${ISSUE_FILE}"
|
||||
|
||||
- name: Create issue
|
||||
if: ${{ failure() && github.event_name == 'schedule' && github.repository == 'XRPLF/rippled' }}
|
||||
uses: XRPLF/actions/create-issue@2b8bc36af85b88bca0dd7bfac2e2dc05f94ad712
|
||||
with:
|
||||
title: "Releases not merged back into develop"
|
||||
body_file: ${{ env.ISSUE_FILE }}
|
||||
labels: "Bug"
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@7bf7ceca5932114abdd0d43493c3c30c5a654e13
|
||||
uses: XRPLF/actions/prepare-runner@b3e255d74d785d053e4903da8ac90983cd7d9e82
|
||||
with:
|
||||
enable_ccache: false
|
||||
|
||||
|
||||
@@ -85,6 +85,7 @@ jobs:
|
||||
.github/workflows/reusable-build-test.yml
|
||||
.github/workflows/reusable-check-autogen.yml
|
||||
.github/workflows/reusable-clang-tidy.yml
|
||||
.github/workflows/reusable-package-test-install.yml
|
||||
.github/workflows/reusable-package.yml
|
||||
.github/workflows/reusable-rust.yml
|
||||
.github/workflows/reusable-strategy-matrix.yml
|
||||
@@ -149,6 +150,48 @@ jobs:
|
||||
if: ${{ needs.should-run.outputs.go == 'true' }}
|
||||
uses: ./.github/workflows/reusable-check-rename.yml
|
||||
|
||||
# Runs regardless of the changed files.
|
||||
# PRs into staging branches are skipped, since fixes may be cherry-picked between release lines.
|
||||
check-release-commits:
|
||||
needs: should-run
|
||||
if: ${{ github.event.pull_request.base.ref == 'develop' || github.event.merge_group.base_ref == 'refs/heads/develop' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
# The check needs the full history of the release and staging branches.
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Check for copied release commits
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
env:
|
||||
BASE: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD: ${{ github.event.pull_request.head.sha }}
|
||||
run: .github/scripts/releasing/check-no-copied-release-commits.sh "${BASE}" "${HEAD}"
|
||||
# Runs even if the previous check fails, so that both problems are reported at once.
|
||||
- name: Check merge-back has no new commits
|
||||
if: ${{ !cancelled() && github.event_name == 'pull_request' }}
|
||||
env:
|
||||
BASE: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD: ${{ github.event.pull_request.head.sha }}
|
||||
run: .github/scripts/releasing/check-merge-back-commits.sh "${BASE}" "${HEAD}"
|
||||
# The queue squashes PRs, so check the PR's own branch, named in the queue branch.
|
||||
- name: Check the merge queue doesn't merge a release back
|
||||
if: ${{ github.event_name == 'merge_group' }}
|
||||
env:
|
||||
BASE: ${{ github.event.merge_group.base_sha }}
|
||||
HEAD_REF: ${{ github.event.merge_group.head_ref }}
|
||||
run: |
|
||||
if ! [[ "${HEAD_REF}" =~ /pr-([0-9]+)-[0-9a-f]+$ ]]; then
|
||||
echo "Error: Can't find the PR number in '${HEAD_REF}'."
|
||||
exit 1
|
||||
fi
|
||||
git fetch --no-tags origin "refs/pull/${BASH_REMATCH[1]}/head"
|
||||
.github/scripts/releasing/check-no-merge-back.sh "${BASE}" FETCH_HEAD
|
||||
|
||||
clang-tidy:
|
||||
needs: should-run
|
||||
if: ${{ needs.should-run.outputs.go == 'true' }}
|
||||
@@ -189,6 +232,12 @@ jobs:
|
||||
# matrix (i.e. not yet labeled "Ready to merge" or "Full CI build").
|
||||
if: ${{ needs.should-run.outputs.go == 'true' && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'Ready to merge') || contains(github.event.pull_request.labels.*.name, 'Full CI build')) }}
|
||||
uses: ./.github/workflows/reusable-package.yml
|
||||
with:
|
||||
# A pull request builds packages to prove they still build, and publishes
|
||||
# nothing. Stated rather than left to the input's default, so that changing
|
||||
# that default cannot start publishing from pull requests. No secrets are
|
||||
# passed either, which is the second reason a publish here cannot succeed.
|
||||
publish: false
|
||||
|
||||
upload-recipe:
|
||||
needs:
|
||||
@@ -223,6 +272,7 @@ jobs:
|
||||
- check-autogen
|
||||
- check-levelization
|
||||
- check-rename
|
||||
- check-release-commits
|
||||
- clang-tidy
|
||||
- build-test
|
||||
- rust
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
# When a versioned tag is pushed, this workflow:
|
||||
#
|
||||
# - uploads the libxrpl recipe to the Conan remote
|
||||
# - builds and tests the release binaries
|
||||
# - uploads the libxrpl recipe to the Conan remote
|
||||
# - builds the DEB and RPM packages
|
||||
# - publishes those packages to the XRPLF package repositories
|
||||
#
|
||||
# Nothing is published unless the build passes, which is also where CMake
|
||||
# rejects a tag that is not a valid version, e.g. 3.2.01.
|
||||
name: Tag
|
||||
|
||||
on:
|
||||
@@ -22,6 +25,7 @@ defaults:
|
||||
jobs:
|
||||
upload-recipe:
|
||||
if: ${{ github.repository == 'XRPLF/rippled' }}
|
||||
needs: build-test
|
||||
uses: ./.github/workflows/reusable-upload-recipe.yml
|
||||
secrets:
|
||||
remote_username: ${{ secrets.NEXUS_REMOTE_USERNAME }}
|
||||
@@ -50,3 +54,7 @@ jobs:
|
||||
remote_username: ${{ secrets.NEXUS_REMOTE_USERNAME }}
|
||||
remote_password: ${{ secrets.NEXUS_REMOTE_PASSWORD }}
|
||||
signing_key: ${{ secrets.NEXUS_PACKAGES_PRIVATE_KEY }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
antithesis_docker_host: ${{ secrets.ANTITHESIS_DOCKER_HOST }}
|
||||
antithesis_docker_path: ${{ secrets.ANTITHESIS_DOCKER_PATH }}
|
||||
antithesis_docker_credentials: ${{ secrets.ANTITHESIS_DOCKER_CREDENTIALS }}
|
||||
@@ -23,6 +23,7 @@ on:
|
||||
- ".github/workflows/reusable-build-test.yml"
|
||||
- ".github/workflows/reusable-check-autogen.yml"
|
||||
- ".github/workflows/reusable-clang-tidy.yml"
|
||||
- ".github/workflows/reusable-package-test-install.yml"
|
||||
- ".github/workflows/reusable-package.yml"
|
||||
- ".github/workflows/reusable-rust.yml"
|
||||
- ".github/workflows/reusable-strategy-matrix.yml"
|
||||
@@ -128,3 +129,7 @@ jobs:
|
||||
remote_username: ${{ secrets.NEXUS_REMOTE_USERNAME }}
|
||||
remote_password: ${{ secrets.NEXUS_REMOTE_PASSWORD }}
|
||||
signing_key: ${{ secrets.NEXUS_PACKAGES_PRIVATE_KEY }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
antithesis_docker_host: ${{ secrets.ANTITHESIS_DOCKER_HOST }}
|
||||
antithesis_docker_path: ${{ secrets.ANTITHESIS_DOCKER_PATH }}
|
||||
antithesis_docker_credentials: ${{ secrets.ANTITHESIS_DOCKER_CREDENTIALS }}
|
||||
@@ -14,7 +14,7 @@ on:
|
||||
jobs:
|
||||
# Call the workflow in the XRPLF/actions repo that runs the pre-commit hooks.
|
||||
run-hooks:
|
||||
uses: XRPLF/actions/.github/workflows/pre-commit.yml@f1952595d212e86169935135efc66294b4574131
|
||||
uses: XRPLF/actions/.github/workflows/pre-commit.yml@279ec358f4a1be4088be3e024b07916fa97c75b6
|
||||
with:
|
||||
runs_on: ubuntu-latest
|
||||
container: '{ "image": "ghcr.io/xrplf/xrpld/pre-commit:sha-473fe44" }'
|
||||
@@ -41,13 +41,13 @@ env:
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-473fe44
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-060957e
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@7bf7ceca5932114abdd0d43493c3c30c5a654e13
|
||||
uses: XRPLF/actions/prepare-runner@b3e255d74d785d053e4903da8ac90983cd7d9e82
|
||||
with:
|
||||
enable_ccache: false
|
||||
|
||||
@@ -91,4 +91,4 @@ jobs:
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deploy
|
||||
uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0
|
||||
uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1
|
||||
@@ -129,7 +129,7 @@ jobs:
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@7bf7ceca5932114abdd0d43493c3c30c5a654e13
|
||||
uses: XRPLF/actions/prepare-runner@b3e255d74d785d053e4903da8ac90983cd7d9e82
|
||||
with:
|
||||
enable_ccache: ${{ inputs.ccache_enabled }}
|
||||
|
||||
@@ -196,6 +196,19 @@ jobs:
|
||||
env:
|
||||
BUILD_TYPE: ${{ inputs.build_type }}
|
||||
CMAKE_ARGS: ${{ inputs.cmake_args }}
|
||||
# GitHub creates a merge commit for a PR
|
||||
# https://www.kenmuse.com/blog/the-many-shas-of-a-github-pull-request/
|
||||
#
|
||||
# We:
|
||||
# - explicitly provide branch name
|
||||
# - use `github.event.pull_request.head.sha` to get the SHA of last commit in the PR branch
|
||||
#
|
||||
# This way it works both for PRs and pushes to branches.
|
||||
GITHUB_BRANCH_NAME: "${{ github.head_ref || github.ref_name }}"
|
||||
GITHUB_HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
|
||||
#
|
||||
# If tag is being pushed, we use that version.
|
||||
FORCE_XRPLD_VERSION: ${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || '' }}
|
||||
run: |
|
||||
cmake \
|
||||
-G '${{ runner.os == 'Windows' && 'Visual Studio 18 2026' || 'Ninja' }}' \
|
||||
@@ -257,6 +270,7 @@ jobs:
|
||||
run: |
|
||||
./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpld"
|
||||
./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpl_tests"
|
||||
./bin/check-nix-store-refs.sh "${BUILD_DIR}/xrpld_tests"
|
||||
|
||||
- name: Show ccache statistics
|
||||
if: ${{ inputs.ccache_enabled }}
|
||||
@@ -338,7 +352,9 @@ jobs:
|
||||
- name: Run the separate tests
|
||||
if: ${{ !inputs.build_only }}
|
||||
working-directory: ${{ runner.os == 'Windows' && format('{0}/{1}', env.BUILD_DIR, inputs.build_type) || env.BUILD_DIR }}
|
||||
run: ./xrpl_tests
|
||||
run: |
|
||||
./xrpl_tests
|
||||
./xrpld_tests
|
||||
|
||||
- name: Run the embedded tests
|
||||
if: ${{ !inputs.build_only }}
|
||||
@@ -439,7 +455,7 @@ jobs:
|
||||
|
||||
- name: Upload coverage report
|
||||
if: ${{ github.repository_owner == 'XRPLF' && !inputs.build_only && env.COVERAGE_ENABLED == 'true' }}
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
|
||||
with:
|
||||
disable_search: true
|
||||
disable_telem: true
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
needs: [determine-files]
|
||||
if: ${{ needs.determine-files.outputs.cpp_changed_files != '' || needs.determine-files.outputs.need_full_run == 'true' }}
|
||||
runs-on: ["self-hosted", "Linux", "X64", "heavy"]
|
||||
container: "ghcr.io/xrplf/xrpld/nix-debian:sha-473fe44"
|
||||
container: "ghcr.io/xrplf/xrpld/nix-debian:sha-060957e"
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@7bf7ceca5932114abdd0d43493c3c30c5a654e13
|
||||
uses: XRPLF/actions/prepare-runner@b3e255d74d785d053e4903da8ac90983cd7d9e82
|
||||
with:
|
||||
enable_ccache: false
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# Install one package format on every distro family it targets, one job per
|
||||
# package name and image, and run the binaries there. Called once per format by
|
||||
# reusable-package.yml, which owns the names and the image lists.
|
||||
name: Install packages
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
package_type:
|
||||
description: 'The package format to install ("deb" or "rpm").'
|
||||
required: true
|
||||
type: string
|
||||
package_names:
|
||||
description: "JSON array of package names built for this format."
|
||||
required: true
|
||||
type: string
|
||||
images:
|
||||
description: "JSON array of container images to install in."
|
||||
required: true
|
||||
type: string
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
PACKAGE_DIR: packages
|
||||
|
||||
jobs:
|
||||
install:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
package_name: ${{ fromJson(inputs.package_names) }}
|
||||
image: ${{ fromJson(inputs.images) }}
|
||||
name: "${{ matrix.package_name }} on ${{ matrix.image }}"
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-latest
|
||||
container: ${{ matrix.image }}
|
||||
timeout-minutes: 5
|
||||
|
||||
steps:
|
||||
# Every package lands in one directory; the step below picks its own,
|
||||
# which keeps this independent of the artifact names.
|
||||
- name: Download package artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: "*-pkg"
|
||||
merge-multiple: true
|
||||
path: ${{ env.PACKAGE_DIR }}
|
||||
|
||||
- name: Find the package
|
||||
id: find
|
||||
env:
|
||||
PACKAGE_NAME: ${{ matrix.package_name }}
|
||||
PACKAGE_TYPE: ${{ inputs.package_type }}
|
||||
run: |
|
||||
# The version follows the name, separated by '_' in a DEB and '-' in an
|
||||
# RPM. Requiring a digit after it is what keeps 'xrpld' from picking up
|
||||
# another package, such as 'xrpld-assert'.
|
||||
pattern="${PACKAGE_NAME}[_-][0-9]*.${PACKAGE_TYPE}"
|
||||
package="$(find "${PACKAGE_DIR}" -type f -name "${pattern}" -print -quit)"
|
||||
test -n "${package}" || {
|
||||
echo "no ${pattern} found in ${PACKAGE_DIR}" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "package=${package}" >>"${GITHUB_OUTPUT}"
|
||||
|
||||
# Debian 11 went end-of-life on 2026-08-31
|
||||
# (https://www.debian.org/News/2026/20260831) and its packages are
|
||||
# already partly gone from deb.debian.org, so switch to the
|
||||
# snapshot.debian.org entries the image ships commented out in its
|
||||
# sources.list: they are pinned to the snapshot the image was built
|
||||
# from, so they serve every version it needs and never go away.
|
||||
# Snapshots keep their original, long-passed Valid-Until, hence the
|
||||
# disabled check; the retries absorb snapshot.debian.org's throttling.
|
||||
- name: Switch Debian 11 to snapshot.debian.org
|
||||
if: ${{ matrix.image == 'debian:11' }}
|
||||
run: |
|
||||
sed -i 's|^deb |# deb |; s|^# deb http://snapshot|deb http://snapshot|' /etc/apt/sources.list
|
||||
printf '%s\n' \
|
||||
'Acquire::Check-Valid-Until "false";' \
|
||||
'Acquire::Retries "3";' \
|
||||
>/etc/apt/apt.conf.d/99snapshot
|
||||
|
||||
- name: Install the DEB
|
||||
if: ${{ inputs.package_type == 'deb' }}
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
PACKAGE: ${{ steps.find.outputs.package }}
|
||||
run: |
|
||||
# Stock Debian and Ubuntu images carry no package lists, so apt has
|
||||
# nothing to resolve the systemd dependency from until it fetches them.
|
||||
apt-get update -qq
|
||||
apt-get install -y "./${PACKAGE}"
|
||||
|
||||
- name: Install the RPM
|
||||
if: ${{ inputs.package_type == 'rpm' }}
|
||||
env:
|
||||
PACKAGE: ${{ steps.find.outputs.package }}
|
||||
run: dnf install -y "./${PACKAGE}"
|
||||
|
||||
- name: Run xrpld
|
||||
run: xrpld --version
|
||||
|
||||
- name: Run validator-keys
|
||||
run: validator-keys --version
|
||||
|
||||
- name: Run rippled, the legacy compatibility symlink
|
||||
run: rippled --version
|
||||
|
||||
- name: Check the service account
|
||||
run: id xrpld
|
||||
|
||||
- name: Check the state directory
|
||||
run: test -d /var/lib/xrpld
|
||||
|
||||
- name: Check the log directory
|
||||
run: test -d /var/log/xrpld
|
||||
@@ -1,11 +1,19 @@
|
||||
# Build Linux packages from the pre-built xrpld and validator-keys artifacts:
|
||||
# Build, verify and publish Linux packages from the pre-built xrpld and
|
||||
# validator-keys artifacts, in these stages:
|
||||
#
|
||||
# - one job per config that carries a "package" map in linux.json
|
||||
# - that map names the container image and the format it builds there
|
||||
# - every job ends with the image's publish_pkg.py, uploading what it built
|
||||
# with 'publish: true' and doing a --dry-run otherwise
|
||||
# - 'package' builds and signs one format per config that carries a "package"
|
||||
# map in linux.json; that map names the container image and the format
|
||||
# - 'test-install-deb' and 'test-install-rpm' call
|
||||
# reusable-package-test-install.yml to install what was built on a range of
|
||||
# distros and run the binaries there, so a package that cannot be installed
|
||||
# never reaches Nexus
|
||||
# - 'publish' uploads with the image's publish_pkg.py, doing a --dry-run
|
||||
# unless 'publish: true'
|
||||
# - 'docker' builds an Ubuntu image from the tested DEB, and one with the
|
||||
# voidstar binary for Antithesis, pushing them to Docker Hub and to the
|
||||
# Antithesis registry only with 'publish: true'
|
||||
#
|
||||
# Only linux/amd64 is supported; the runner is hardcoded in the job below.
|
||||
# Only linux/amd64 is supported; the runner is hardcoded in the jobs below.
|
||||
name: Package
|
||||
|
||||
on:
|
||||
@@ -20,7 +28,7 @@ on:
|
||||
description: "The base URL of the Nexus instance hosting the deb and rpm repositories."
|
||||
required: false
|
||||
type: string
|
||||
default: https://packages.xrplf.org
|
||||
default: https://packages-upload.xrplf.org
|
||||
|
||||
secrets:
|
||||
remote_username:
|
||||
@@ -30,7 +38,19 @@ on:
|
||||
description: "The password or token for that Nexus account."
|
||||
required: false
|
||||
signing_key:
|
||||
description: "Armoured PGP private key used to sign the RPMs. Required when publishing."
|
||||
description: "Armoured PGP private key used to sign the RPMs."
|
||||
required: false
|
||||
dockerhub_token:
|
||||
description: "A Docker Hub organization access token for xrplf, with push access to xrplf/xrpld."
|
||||
required: false
|
||||
antithesis_docker_host:
|
||||
description: "The host of the Antithesis container registry, e.g. us-central1-docker.pkg.dev."
|
||||
required: false
|
||||
antithesis_docker_path:
|
||||
description: "The repository path in that registry, the image name excluded."
|
||||
required: false
|
||||
antithesis_docker_credentials:
|
||||
description: "The JSON key of a service account with push access to that repository."
|
||||
required: false
|
||||
|
||||
defaults:
|
||||
@@ -39,12 +59,15 @@ defaults:
|
||||
|
||||
env:
|
||||
BUILD_DIR: build
|
||||
PACKAGE_DIR: packages
|
||||
|
||||
jobs:
|
||||
generate-matrix:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.generate.outputs.matrix }}
|
||||
deb_package_names: ${{ steps.generate.outputs.deb_package_names }}
|
||||
rpm_package_names: ${{ steps.generate.outputs.rpm_package_names }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
@@ -70,14 +93,14 @@ jobs:
|
||||
contents: read
|
||||
runs-on: ["self-hosted", "Linux", "X64", "heavy"]
|
||||
container: ${{ matrix.image }}
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@7bf7ceca5932114abdd0d43493c3c30c5a654e13
|
||||
uses: XRPLF/actions/prepare-runner@b3e255d74d785d053e4903da8ac90983cd7d9e82
|
||||
with:
|
||||
enable_ccache: false
|
||||
|
||||
@@ -103,6 +126,7 @@ jobs:
|
||||
- name: Build package
|
||||
env:
|
||||
PACKAGE_TYPE: ${{ matrix.package_type }}
|
||||
PACKAGE_VARIANT: ${{ matrix.package_variant }}
|
||||
PKG_RELEASE: ${{ steps.release_info.outputs.pkg_release }}
|
||||
CHANNEL: ${{ steps.release_info.outputs.channel }}
|
||||
run: |
|
||||
@@ -110,26 +134,113 @@ jobs:
|
||||
--package-type "${PACKAGE_TYPE}" \
|
||||
--build-dir "${BUILD_DIR}" \
|
||||
--pkg-release "${PKG_RELEASE}" \
|
||||
--variant "${PACKAGE_VARIANT}" \
|
||||
--channel "${CHANNEL}"
|
||||
|
||||
# Before the upload, so the artifact and the published package are the
|
||||
# same bytes. DEBs are not signed, so the key is never set on that job.
|
||||
# Before the upload, so the artifact, the tested package and the published
|
||||
# package are the same bytes.
|
||||
- name: Sign RPM
|
||||
if: ${{ inputs.publish && matrix.package_type == 'rpm' }}
|
||||
env:
|
||||
PKG_SIGNING_KEY: ${{ secrets.signing_key }}
|
||||
run: ./package/sign_rpm.py --package-dir "${BUILD_DIR}"
|
||||
|
||||
# Split from the debug symbols, which are an order of magnitude larger, so
|
||||
# that test-install downloads only what it installs. In the globs below the
|
||||
# version follows the name, separated by '_' in a DEB and '-' in an RPM. A
|
||||
# version starts with a digit and a longer name does not, so that one digit
|
||||
# is what tells 'xrpld-3.4.1-...' from 'xrpld-assert-3.4.1-...'.
|
||||
- name: Upload package artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ matrix.xrpld_artifact_name }}-pkg
|
||||
path: |
|
||||
${{ env.BUILD_DIR }}/debbuild/*.deb
|
||||
${{ env.BUILD_DIR }}/debbuild/*.ddeb
|
||||
${{ env.BUILD_DIR }}/rpmbuild/RPMS/**/*.rpm
|
||||
${{ env.BUILD_DIR }}/debbuild/${{ matrix.package_name }}_[0-9]*.deb
|
||||
${{ env.BUILD_DIR }}/rpmbuild/RPMS/**/${{ matrix.package_name }}-[0-9]*.rpm
|
||||
if-no-files-found: error
|
||||
|
||||
- name: Upload debug symbol artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ matrix.xrpld_artifact_name }}-pkg-debug
|
||||
path: |
|
||||
${{ env.BUILD_DIR }}/debbuild/${{ matrix.package_name }}-dbgsym_[0-9]*.deb
|
||||
${{ env.BUILD_DIR }}/debbuild/${{ matrix.package_name }}-dbgsym_[0-9]*.ddeb
|
||||
${{ env.BUILD_DIR }}/rpmbuild/RPMS/**/${{ matrix.package_name }}-debuginfo-[0-9]*.rpm
|
||||
if-no-files-found: error
|
||||
|
||||
# One call per format, so a variant packaged for one format is installed for
|
||||
# that format alone. The images are every distro family that format targets,
|
||||
# oldest release first, so both ends of the dependency range the packages
|
||||
# declare are exercised.
|
||||
test-install-deb:
|
||||
needs: [generate-matrix, package]
|
||||
name: install deb
|
||||
uses: ./.github/workflows/reusable-package-test-install.yml
|
||||
with:
|
||||
package_type: deb
|
||||
package_names: ${{ needs.generate-matrix.outputs.deb_package_names }}
|
||||
images: |
|
||||
[
|
||||
"debian:11",
|
||||
"debian:12",
|
||||
"debian:13",
|
||||
"ubuntu:20.04",
|
||||
"ubuntu:22.04",
|
||||
"ubuntu:24.04",
|
||||
"ubuntu:26.04"
|
||||
]
|
||||
|
||||
test-install-rpm:
|
||||
needs: [generate-matrix, package]
|
||||
name: install rpm
|
||||
uses: ./.github/workflows/reusable-package-test-install.yml
|
||||
with:
|
||||
package_type: rpm
|
||||
package_names: ${{ needs.generate-matrix.outputs.rpm_package_names }}
|
||||
images: |
|
||||
[
|
||||
"almalinux:9",
|
||||
"almalinux:10",
|
||||
"rockylinux/rockylinux:9",
|
||||
"rockylinux/rockylinux:10",
|
||||
"registry.access.redhat.com/ubi9/ubi",
|
||||
"registry.access.redhat.com/ubi10/ubi"
|
||||
]
|
||||
|
||||
publish:
|
||||
needs: [generate-matrix, package, test-install-deb, test-install-rpm]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
|
||||
name: "publish ${{ matrix.xrpld_artifact_name }}${{ !inputs.publish && ' (dry run)' || '' }}"
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ["self-hosted", "Linux", "X64", "heavy"]
|
||||
container: ${{ matrix.image }}
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@b3e255d74d785d053e4903da8ac90983cd7d9e82
|
||||
with:
|
||||
enable_ccache: false
|
||||
|
||||
# Both artifacts, so the debug symbols are published alongside the package.
|
||||
- name: Download package artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: ${{ matrix.xrpld_artifact_name }}-pkg*
|
||||
merge-multiple: true
|
||||
path: ${{ env.PACKAGE_DIR }}
|
||||
|
||||
- name: Determine release info
|
||||
id: release_info
|
||||
uses: ./.github/actions/release-info
|
||||
|
||||
- name: Publish package
|
||||
env:
|
||||
CHANNEL: ${{ steps.release_info.outputs.channel }}
|
||||
@@ -140,6 +251,98 @@ jobs:
|
||||
run: |
|
||||
publish_pkg.py \
|
||||
--channel "${CHANNEL}" \
|
||||
--package-dir "${BUILD_DIR}" \
|
||||
--package-dir "${PACKAGE_DIR}" \
|
||||
--nexus-url "${NEXUS_URL}" \
|
||||
${DRY_RUN_OPTION}
|
||||
|
||||
docker:
|
||||
needs: [test-install-deb, test-install-rpm]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target: [xrpld, voidstar]
|
||||
name: "docker ${{ matrix.target }}${{ !inputs.publish && ' (dry run)' || '' }}"
|
||||
permissions:
|
||||
contents: read
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
CONTEXT: image-context
|
||||
IMAGE: ${{ matrix.target == 'voidstar' && 'xrpld-voidstar' || 'xrplf/xrpld' }}:${{ github.ref_type == 'tag' && github.ref_name || 'develop' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Determine release info
|
||||
id: release_info
|
||||
uses: ./.github/actions/release-info
|
||||
|
||||
# Docker Hub is public, so it only gets builds whose packages are public:
|
||||
# those of a public codebase, and stable releases.
|
||||
# The Antithesis registry is private, so it gets every build.
|
||||
- name: Decide whether to push
|
||||
env:
|
||||
PUSH: ${{ inputs.publish && (matrix.target == 'voidstar' || github.event.repository.visibility == 'public' || steps.release_info.outputs.channel == 'stable') }}
|
||||
run: echo "PUSH=${PUSH}" | tee -a "${GITHUB_ENV}"
|
||||
|
||||
- name: Download package artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: "*-pkg"
|
||||
merge-multiple: true
|
||||
path: ${{ env.PACKAGE_DIR }}
|
||||
|
||||
- name: Download voidstar binary
|
||||
if: ${{ matrix.target == 'voidstar' }}
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: xrpld-ubuntu-clang-debug-amd64-voidstar
|
||||
path: ${{ env.CONTEXT }}
|
||||
|
||||
- name: Build image
|
||||
env:
|
||||
TARGET: ${{ matrix.target }}
|
||||
run: |
|
||||
mkdir -p "${CONTEXT}"
|
||||
find "${PACKAGE_DIR}" -type f -name 'xrpld_[0-9]*.deb' -exec cp {} "${CONTEXT}/" \;
|
||||
docker build --pull --file package/images/xrpld/Dockerfile --target "${TARGET}" --tag "${IMAGE}" "${CONTEXT}"
|
||||
|
||||
- name: Start the server
|
||||
run: |
|
||||
container="$(docker run --detach "${IMAGE}" --standalone --silent --conf /etc/xrpld/xrpld.cfg)"
|
||||
trap 'docker rm --force "${container}" >/dev/null' EXIT
|
||||
|
||||
for _ in $(seq 30); do
|
||||
output="$(docker exec "${container}" xrpld --conf /etc/xrpld/xrpld.cfg server_info || true)"
|
||||
if [[ "${output}" == *'"status" : "success"'* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
docker logs "${container}"
|
||||
exit 1
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
if: ${{ env.PUSH == 'true' && matrix.target == 'xrpld' }}
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
username: xrplf
|
||||
password: ${{ secrets.dockerhub_token }}
|
||||
|
||||
- name: Log in to the Antithesis registry
|
||||
if: ${{ env.PUSH == 'true' && matrix.target == 'voidstar' }}
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ secrets.antithesis_docker_host }}
|
||||
username: _json_key
|
||||
password: ${{ secrets.antithesis_docker_credentials }}
|
||||
|
||||
- name: Push image
|
||||
if: ${{ env.PUSH == 'true' }}
|
||||
env:
|
||||
REGISTRY: ${{ matrix.target == 'voidstar' && format('{0}/{1}/', secrets.antithesis_docker_host, secrets.antithesis_docker_path) || '' }}
|
||||
run: |
|
||||
docker tag "${IMAGE}" "${REGISTRY}${IMAGE}"
|
||||
docker push "${REGISTRY}${IMAGE}"
|
||||
@@ -1,8 +1,9 @@
|
||||
# Clippy, coverage and documentation for the Rust crates in crates/. Each runs
|
||||
# as an independent job on a GitHub-hosted runner, but inside the same container
|
||||
# image used to build the crates in the C++/Corrosion path, so the toolchain
|
||||
# (and therefore the lints, coverage instrumentation and the cargo cache) matches
|
||||
# what production builds use.
|
||||
# (and therefore the lints and the cargo cache) matches what production builds
|
||||
# use. Coverage is the exception: it needs the nightly rustc that honours
|
||||
# #[coverage(off)], which the image carries alongside the pinned stable.
|
||||
#
|
||||
# Rust unit tests are deliberately NOT run here. They run as part of the C++
|
||||
# build (reusable-build-test-config.yml), which already compiles the crates on a
|
||||
@@ -27,7 +28,7 @@ permissions:
|
||||
jobs:
|
||||
clippy:
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-473fe44
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-060957e
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
@@ -40,11 +41,14 @@ jobs:
|
||||
|
||||
coverage:
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-473fe44
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-060957e
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Use the nightly Rust toolchain
|
||||
run: rust-nightly path >>"${GITHUB_PATH}"
|
||||
|
||||
- name: Use cargo artifacts cache
|
||||
uses: ./.github/actions/cargo-cache
|
||||
|
||||
@@ -53,7 +57,7 @@ jobs:
|
||||
|
||||
- name: Upload coverage report
|
||||
if: ${{ github.repository == 'XRPLF/rippled' }}
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1
|
||||
with:
|
||||
disable_search: true
|
||||
disable_telem: true
|
||||
@@ -66,7 +70,7 @@ jobs:
|
||||
|
||||
doc:
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-473fe44
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-060957e
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
@@ -40,7 +40,7 @@ defaults:
|
||||
jobs:
|
||||
upload:
|
||||
runs-on: ubuntu-latest
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-473fe44
|
||||
container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-060957e
|
||||
env:
|
||||
REMOTE_NAME: ${{ inputs.remote_name }}
|
||||
CONAN_LOGIN_USERNAME_XRPLF: ${{ secrets.remote_username }}
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@7bf7ceca5932114abdd0d43493c3c30c5a654e13
|
||||
uses: XRPLF/actions/prepare-runner@b3e255d74d785d053e4903da8ac90983cd7d9e82
|
||||
with:
|
||||
enable_ccache: false
|
||||
|
||||
@@ -68,33 +68,17 @@ jobs:
|
||||
run: conan remote login "${REMOTE_NAME}" "${CONAN_LOGIN_USERNAME_XRPLF}" --password "${CONAN_PASSWORD_XRPLF}"
|
||||
|
||||
- name: Upload Conan recipe (version)
|
||||
env:
|
||||
VERSION: ${{ steps.release_info.outputs.version }}
|
||||
run: |
|
||||
conan export . --version=${{ steps.release_info.outputs.version }}
|
||||
conan upload --confirm --check --remote="${REMOTE_NAME}" xrpl/${{ steps.release_info.outputs.version }}
|
||||
conan export . --version="${VERSION}"
|
||||
conan upload --confirm --check --remote="${REMOTE_NAME}" "xrpl/${VERSION}"
|
||||
|
||||
# When this workflow is triggered by a push event, it will always be when merging into the
|
||||
# 'develop' branch, see on-trigger.yml.
|
||||
- name: Upload Conan recipe (develop)
|
||||
if: ${{ github.event_name == 'push' }}
|
||||
if: ${{ github.ref == 'refs/heads/develop' }}
|
||||
run: |
|
||||
conan export . --version=develop
|
||||
conan upload --confirm --check --remote="${REMOTE_NAME}" xrpl/develop
|
||||
|
||||
# When this workflow is triggered by a pull request event, it will always be when merging into
|
||||
# one of the 'release' branches, see on-pr.yml.
|
||||
- name: Upload Conan recipe (rc)
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: |
|
||||
conan export . --version=rc
|
||||
conan upload --confirm --check --remote="${REMOTE_NAME}" xrpl/rc
|
||||
|
||||
# When this workflow is triggered by a push event, it will always be when tagging a final
|
||||
# release, see on-tag.yml.
|
||||
- name: Upload Conan recipe (release)
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/') }}
|
||||
run: |
|
||||
conan export . --version=release
|
||||
conan upload --confirm --check --remote="${REMOTE_NAME}" xrpl/release
|
||||
|
||||
outputs:
|
||||
ref: xrpl/${{ steps.release_info.outputs.version }}
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Prepare runner
|
||||
uses: XRPLF/actions/prepare-runner@7bf7ceca5932114abdd0d43493c3c30c5a654e13
|
||||
uses: XRPLF/actions/prepare-runner@b3e255d74d785d053e4903da8ac90983cd7d9e82
|
||||
with:
|
||||
enable_ccache: false
|
||||
|
||||
|
||||
+10
-2
@@ -72,11 +72,16 @@ DerivedData
|
||||
/.zed/
|
||||
|
||||
# AI tools.
|
||||
# Shared/committable AI agent config (AGENTS.md, CLAUDE.md, GEMINI.md, .claude/settings.json,
|
||||
# tool-specific rules files, etc.) should be checked in — see CONTRIBUTING.md. Only the
|
||||
# personal/local variants below are ignored.
|
||||
/.agent
|
||||
/.agents
|
||||
/.augment
|
||||
/.claude
|
||||
/CLAUDE.md
|
||||
/.claude/settings.local.json
|
||||
AGENTS.override.md
|
||||
CLAUDE.local.md
|
||||
GEMINI.local.md
|
||||
|
||||
# Python
|
||||
__pycache__
|
||||
@@ -87,6 +92,9 @@ target/
|
||||
# Direnv's directory
|
||||
/.direnv
|
||||
|
||||
# Direnv's local, per-developer overrides
|
||||
/.envrc.local
|
||||
|
||||
# clangd cache
|
||||
/.cache
|
||||
|
||||
|
||||
+17
-7
@@ -53,9 +53,14 @@ repos:
|
||||
entry: ./bin/pre-commit/check_doxygen_style.py
|
||||
language: python
|
||||
types_or: [c++, c]
|
||||
- id: fix-gtest-names
|
||||
name: "fix gtest names: CamelCase suite, snake_case test case"
|
||||
entry: ./bin/pre-commit/fix_gtest_names.py
|
||||
language: python
|
||||
types_or: [c++, c]
|
||||
|
||||
- repo: https://github.com/pre-commit/mirrors-clang-format
|
||||
rev: f4d7745e17a28aad7eed2f4874ca8d1568c11c4c # frozen: v22.1.8
|
||||
rev: e2b496dc2bd8340c2524cb9a2d2a943cde1bb6df # frozen: v23.1.1
|
||||
hooks:
|
||||
- id: clang-format
|
||||
args: [--style=file]
|
||||
@@ -70,14 +75,19 @@ repos:
|
||||
language: system
|
||||
types: [rust]
|
||||
pass_filenames: false # rustfmt formats the whole workspace
|
||||
- id: check-coverage-attrs
|
||||
name: check Rust coverage attributes
|
||||
entry: ./bin/pre-commit/check_rust_coverage_attrs.py
|
||||
language: python
|
||||
files: ^crates/.*\.rs$
|
||||
|
||||
- repo: https://github.com/BlankSpruce/gersemi-pre-commit
|
||||
rev: e98930bdc210d3387007f9252d8c1694ea7e410f # frozen: 0.27.7
|
||||
rev: f1c4833f8cf23c6d952673abc73525411a5719e8 # frozen: 0.29.1
|
||||
hooks:
|
||||
- id: gersemi
|
||||
|
||||
- repo: https://github.com/rbubley/mirrors-prettier
|
||||
rev: 9337a74165b178ae2c766f60bee7252a0f06f3e8 # frozen: v3.9.5
|
||||
rev: ef4a397f916211b4a39ccf9d3d9cbb6562157251 # frozen: v3.9.9
|
||||
hooks:
|
||||
- id: prettier
|
||||
args: [--end-of-line=auto]
|
||||
@@ -85,7 +95,7 @@ repos:
|
||||
# Scoped to package/: the rest of the repo's Python has pre-existing findings,
|
||||
# so widening these is its own change.
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: 7c55798a78262d14b2074abf623d8a992ebb70d4 # frozen: v0.16.2
|
||||
rev: a56c0b927e6465d37cae3e97d35d4d18ab2b96cd # frozen: v0.16.9
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
args: [--fix]
|
||||
@@ -97,14 +107,14 @@ repos:
|
||||
- id: black
|
||||
|
||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
||||
rev: 41e691678310dfd3833f7ab4e180ddb014310356 # frozen: v2.3.0
|
||||
rev: 7ff8d35ae36a7d2b968f2f90b4c723e292e594ee # frozen: v2.3.1
|
||||
hooks:
|
||||
- id: mypy
|
||||
args: [--strict]
|
||||
files: ^package/.*\.py$
|
||||
|
||||
- repo: https://github.com/scop/pre-commit-shfmt
|
||||
rev: 05c1426671b9237fb5e1444dd63aa5731bec0dfb # frozen: v3.13.1-1
|
||||
rev: 479be5958357ba5ab65ce47172117348516860e9 # frozen: v3.14.1-1
|
||||
hooks:
|
||||
- id: shfmt
|
||||
args: [--write, --indent=4, --case-indent=true]
|
||||
@@ -123,7 +133,7 @@ repos:
|
||||
files: \.md$
|
||||
|
||||
- repo: https://github.com/streetsidesoftware/cspell-cli
|
||||
rev: ea11f9efc0bec520073405bc30552da887ba71bc # frozen: v10.0.1
|
||||
rev: f5c5d72342f35643988a9ffa0705c05bd3ff8383 # frozen: v10.3.0
|
||||
hooks:
|
||||
- id: cspell
|
||||
name: check changed files spelling
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
# Collapse hard-wrapped Markdown prose to one line per paragraph instead of
|
||||
# preserving manual line breaks (the prettier default). Scoped to Markdown
|
||||
# only, since proseWrap also reflows YAML block scalars.
|
||||
overrides:
|
||||
- files: "*.md"
|
||||
options:
|
||||
proseWrap: never
|
||||
@@ -0,0 +1,42 @@
|
||||
# AGENTS.md
|
||||
|
||||
This file provides guidance to AI coding agents (Claude Code, and other AGENTS.md-compatible tools) when working with code in this repository.
|
||||
|
||||
## Build
|
||||
|
||||
Required on Linux/macOS: use the Nix devshell, which sets up the compiler, Conan, ccache, and (optionally) Rust automatically.
|
||||
|
||||
```bash
|
||||
nix develop
|
||||
```
|
||||
|
||||
For alternate devshell variants (specific compiler, no-compiler, coverage), see [docs/build/nix.md](./docs/build/nix.md). For the manual build steps, CMake options, and protocol codegen commands, see [BUILD.md](./BUILD.md) (`## Steps`, `## Options`, `## Code generation`).
|
||||
|
||||
Rust crate tests (independent of the CMake build): `cargo test --manifest-path crates/Cargo.toml --workspace` (CI uses `cargo nextest`).
|
||||
|
||||
## Testing
|
||||
|
||||
Unit tests are a custom framework built into the `xrpld` binary itself (not Boost.Test/GTest/Catch); see [CONTRIBUTING.md](./CONTRIBUTING.md#unit-tests) for the basic invocation. Notes not covered there:
|
||||
|
||||
- A suite's `--unittest` name is built from the arguments to its `BEAST_DEFINE_TESTSUITE`/`BEAST_DEFINE_TESTSUITE_PRIO` macro (usually at the bottom of the test file), in reverse order and joined with `.`: `BEAST_DEFINE_TESTSUITE(Credentials, app, xrpl)` → `xrpl.app.Credentials`.
|
||||
- `--unittest-arg` does nothing — don't use it.
|
||||
- Tests that run offline in under a minute should be automatic `--unittest` suites; anything else is a manual/integration test.
|
||||
- New tests should be written using `gtest` under `src/tests/` unless that isn't possible, in which case fall back to the legacy Beast framework under `src/test/`. `tests/` (top-level) holds integration tests exercised against `libxrpl`/`xrpld`.
|
||||
|
||||
## Lint/Format
|
||||
|
||||
See [CONTRIBUTING.md](./CONTRIBUTING.md#pre-commit-hooks) for `pre-commit` setup and [CONTRIBUTING.md](./CONTRIBUTING.md#clang-tidy) for `clang-tidy` (opt-in, needs local `clang-tidy` and generated headers).
|
||||
|
||||
## Code Style
|
||||
|
||||
New file placement and header levelization: see [CONTRIBUTING.md](./CONTRIBUTING.md#before-making-a-pull-request). Braces, whitespace, member order, and other conventions: see [docs/CodingStyle.md](./docs/CodingStyle.md). `XRPL_ASSERT`/`UNREACHABLE` contracts: see [CONTRIBUTING.md](./CONTRIBUTING.md#contracts-and-instrumentation). Commit messages: see [CONTRIBUTING.md](./CONTRIBUTING.md#good-commit-messages).
|
||||
|
||||
## Architecture
|
||||
|
||||
Paths below reflect the current layout; update this section if modularization moves a subsystem to a different directory.
|
||||
|
||||
- `include/xrpl/` + `src/libxrpl/` — the core protocol library: ledger, shamap, consensus, crypto, json, resource, nodestore, rdb, peerfinder, and `tx/` (transaction application: `Transactor.cpp`, `applySteps.cpp`, invariants, payment paths). `tx/transactors/` has one file per transaction type, grouped by subsystem: `escrow/`, `vault/`, `lending/`, `sponsor/`, `nft/`, `token/` (MPT), `payment_channel/`, `permissioned_domain/`, `dex/`, `oracle/`, `did/`, `credentials/`, `bridge/`, `check/`, `delegate/`, `account/`, `system/`. Any change to transaction-processing behavior must be gated behind an Amendment.
|
||||
- `src/xrpld/` — the server application built on top of `libxrpl`: `app`, `core`, `overlay` (P2P networking), `peerfinder`, `perflog`, `rpc`, `shamap`. `main` builds an `ApplicationImp` implementing `Application`; most components hold a reference to it (`app_`), giving broad cross-component access — expect to trace call chains through `Application&`.
|
||||
- `src/test/` — unit tests mirroring the subsystems above, plus `jtx/` (the transaction-building test DSL — e.g. `jtx/escrow.h`, `jtx/vault.h`, `jtx/sponsor.h`, `jtx/permissioned_dex.h`) and `unit_test/` (the custom test framework itself, derived from Beast).
|
||||
- `src/tests/` — unit tests for `libxrpl` written in `gtest`, gradually replacing the `src/test` equivalents.
|
||||
- `crates/` — a Rust workspace (only built with `-Dxrpld -Drust=ON`) bridged into C++ via `cxxbridge`/the `cxx` crate; currently just a `hello_world` interop scaffold. Requires the Rust toolchain pinned in `rust-toolchain.toml` (the Nix devshell provides it automatically).
|
||||
+78
-16
@@ -22,13 +22,60 @@ API version 2 is available in `xrpld` version 2.0.0 and later. See [API-VERSION-
|
||||
|
||||
This version is supported by all `xrpld` versions. For WebSocket and HTTP JSON-RPC requests, it is currently the default API version used when no `api_version` is specified.
|
||||
|
||||
## Unreleased
|
||||
## XRP Ledger server version 3.5.0
|
||||
|
||||
This section contains changes targeting a future version.
|
||||
Version 3.5.0 is not yet released.
|
||||
|
||||
### Additions
|
||||
### Additions in 3.5.0
|
||||
|
||||
- `subscribe`, `unsubscribe`: Added an optional `mpt_issuances` request field, an array of MPT issuance IDs (hex strings). Subscribers receive a message with `type` `mptTransaction` for each validated transaction whose metadata affects a subscribed issuance; the message has the same fields as the `transactions` stream. MPT issuance subscriptions count toward the per-connection subscription limit. An empty array, a non-array value, or an invalid ID returns `invalidParams`. ([#5671](https://github.com/XRPLF/rippled/pull/5671))
|
||||
|
||||
### Bugfixes in 3.5.0
|
||||
|
||||
- `channel_authorize`: The `channel_id` field now returns an `invalidParams` error if the value is not a string. [#7582](https://github.com/XRPLF/rippled/pull/7582)
|
||||
- `channel_verify`: The `channel_id` and `signature` fields now return an `invalidParams` error if the value is not a string. [#7582](https://github.com/XRPLF/rippled/pull/7582)
|
||||
|
||||
## XRP Ledger server version 3.4.0
|
||||
|
||||
Version 3.4.0 is not yet released. These changes are available in the 3.4.0 beta releases.
|
||||
|
||||
### Additions in 3.4.0
|
||||
|
||||
- `ledger`: `nftoken_id`, `nftoken_ids`, and `offer_id` are now included in transaction metadata when transactions are expanded (`expand`, or admin-only `full`), matching the `tx`, `account_tx`, and `subscribe` (`transactions` stream) responses. ([#5706](https://github.com/XRPLF/rippled/pull/5706))
|
||||
|
||||
### Bugfixes in 3.4.0
|
||||
|
||||
- `sign`, `sign_for`, `submit`: `signature_target` now returns `invalidParams` unless it names `CounterpartySignature` or `SponsorSignature`. It previously accepted any inner object field, such as `Book` or `NFToken`, and signed into it.
|
||||
- `sign`, `sign_for`, `submit`, `submit_multisigned`: With `fixCleanup3_4_0` enabled, a signature in `CounterpartySignature` or `SponsorSignature` covers a different prefix than the transaction's own signature, so a signature can no longer be moved from one of those roles into another. Clients that build these signatures themselves must use the new prefixes: `CPT` and `CPM` (single- and multi-signing) for `CounterpartySignature`, and `SPN` and `SPM` for `SponsorSignature`.
|
||||
- `get_aggregate_price`: Duplicate entries in the `oracles` request array are now ignored. [#6586](https://github.com/XRPLF/rippled/pull/6586)
|
||||
- `vault_info`: Errors now identify what the request got wrong instead of reporting every failure as the unregistered token `malformedRequest`, and the `error`, `error_code` and `error_message` fields now agree with each other. An invalid `vault_id` or `seq` returns `invalidParams`, an invalid `owner` returns `actMalformed`, and a request that mixes `vault_id` with `owner`/`seq` or supplies neither returns `invalidParams` with a message naming the accepted combinations. [#8015](https://github.com/XRPLF/rippled/pull/8015)
|
||||
- `vault_info`: A well-formed all-zero `vault_id` now returns `entryNotFound` instead of being rejected as malformed, and `entryNotFound` responses now include `error_code` and `error_message`. Clients that request `ripplerpc` 3.0 or above therefore receive HTTP 400 with that error rather than HTTP 200. [#8015](https://github.com/XRPLF/rippled/pull/8015)
|
||||
- `vault_info`: `vault_id` and `owner` must now be strings, matching how `ledger_entry` reads the same fields. An object or an array in either field previously produced an internal error, and a number was silently converted to its decimal text; `vault_id` now returns `invalidParams` and `owner` returns `actMalformed`. [#8015](https://github.com/XRPLF/rippled/pull/8015)
|
||||
- `gateway_balances`: The `account` and `ident` fields now return an `invalidParams` error if the value is not a string, instead of an `internal` error. [#7655](https://github.com/XRPLF/rippled/pull/7655)
|
||||
- `account_lines`: The `peer` field now returns an error if the value is not a string. [#7728](https://github.com/XRPLF/rippled/pull/7728)
|
||||
- `ledger`: `delivered_amount` is now included in the metadata of successful `AccountDelete` transactions when transactions are expanded (`expand`, or admin-only `full`). Previously it was only added for `Payment` and `CheckCash`, which made `ledger` inconsistent with `tx` and `account_tx`. [#5706](https://github.com/XRPLF/rippled/pull/5706)
|
||||
- `noripple_check`: The `transactions` field is no longer included in error responses; it is still returned (possibly as an empty array) whenever `transactions` is `true` and the request succeeds. A malformed `account` is now rejected before the ledger is looked up, so that error response no longer carries the `ledger_hash`, `ledger_index`, and `validated` fields ([#6303](https://github.com/XRPLF/rippled/pull/6303)).
|
||||
|
||||
## XRP Ledger server version 3.3.0
|
||||
|
||||
[Version 3.3.0](https://github.com/XRPLF/rippled/releases/tag/3.3.0) was released on Aug 6, 2026.
|
||||
|
||||
### Additions in 3.3.0
|
||||
|
||||
- `account_tx`: Added an optional `delegate` request object to filter delegated transactions. The object requires `delegate_filter`, which must be either `actor` for transactions owned by the requested account but signed by another account, or `authorizer` for transactions signed by the requested account on behalf of another account. The optional `counter_party` account narrows the results to a specific signer/delegate for `actor` or a specific owner/delegator for `authorizer`. Malformed `delegate`, `delegate_filter`, and `counter_party` values return standard invalid field errors, and invalid account IDs return `actMalformed`. When paginating delegate-filtered queries, a marker from a delegate-filtered query includes a `delegate` flag and is only valid for follow-up requests that also supply `delegate` (mixing marker conventions returns `invalidParams`). Because filtering is applied after the ledger scan, a page may contain fewer results than `limit` (possibly zero) while still returning a marker, so callers must continue until no marker is present. ([#6126](https://github.com/XRPLF/rippled/pull/6126))
|
||||
|
||||
## XRP Ledger server version 3.2.1
|
||||
|
||||
[Version 3.2.1](https://github.com/XRPLF/rippled/releases/tag/3.2.1) was released on Aug 1, 2026.
|
||||
|
||||
This release contains bug fixes only and no API changes.
|
||||
|
||||
## XRP Ledger server version 3.2.0
|
||||
|
||||
[Version 3.2.0](https://github.com/XRPLF/rippled/releases/tag/3.2.0) was released on Jun 16, 2026.
|
||||
|
||||
### Additions in 3.2.0
|
||||
|
||||
- `account_tx`: Added an optional `delegate` request object to filter delegated transactions. The object requires `delegate_filter`, which must be either `actor` for transactions owned by the requested account but signed by another account, or `authorizer` for transactions signed by the requested account on behalf of another account. The optional `counter_party` account narrows the results to a specific signer/delegate for `actor` or a specific owner/delegator for `authorizer`. Malformed `delegate`, `delegate_filter`, and `counter_party` values return standard invalid field errors, and invalid account IDs return `actMalformed`. When paginating delegate-filtered queries, a marker from a delegate-filtered query includes a `delegate` flag and is only valid for follow-up requests that also supply `delegate` (mixing marker conventions returns `invalidParams`). Because filtering is applied after the ledger scan, a page may contain fewer results than `limit` (possibly zero) while still returning a marker, so callers must continue until no marker is present.
|
||||
- `ledger_entry`, `account_objects`: The `Delegate` ledger entry now includes an optional `DestinationNode` field, which stores the index into the authorized account's owner directory. This field is present on entries created after bidirectional directory tracking was introduced and may appear in RPC responses for those entries. ([#6681](https://github.com/XRPLF/rippled/pull/6681))
|
||||
- `server_definitions`: Added the following new sections to the response ([#6321](https://github.com/XRPLF/rippled/pull/6321)):
|
||||
- `TRANSACTION_FORMATS`: Describes the fields and their optionality for each transaction type, including common fields shared across all transactions.
|
||||
@@ -36,11 +83,9 @@ This section contains changes targeting a future version.
|
||||
- `TRANSACTION_FLAGS`: Maps transaction type names to their supported flags and flag values.
|
||||
- `LEDGER_ENTRY_FLAGS`: Maps ledger entry type names to their flags and flag values.
|
||||
- `ACCOUNT_SET_FLAGS`: Maps AccountSet flag names (asf flags) to their numeric values.
|
||||
- `ledger`: `nftoken_id`, `nftoken_ids`, and `offer_id` are now included in transaction metadata when transactions are expanded (`expand`, or admin-only `full`), matching the `tx`, `account_tx`, and `subscribe` (`transactions` stream) responses. ([#5706](https://github.com/XRPLF/rippled/pull/5706))
|
||||
|
||||
### Bugfixes
|
||||
### Bugfixes in 3.2.0
|
||||
|
||||
- `get_aggregate_price`: Duplicate entries in the `oracles` request array are now ignored. [#6586](https://github.com/XRPLF/rippled/pull/6586)
|
||||
- Peer Crawler: The `port` field in `overlay.active[]` now consistently returns an integer instead of a string for outbound peers. [#6318](https://github.com/XRPLF/rippled/pull/6318)
|
||||
- `ping`: The `ip` field is no longer returned as an empty string for proxied connections without a forwarded-for header. It is now omitted, consistent with the behavior for identified connections. [#6730](https://github.com/XRPLF/rippled/pull/6730)
|
||||
- gRPC `GetLedgerDiff`: Fixed error message that incorrectly said "base ledger not validated" when the desired ledger was not validated. [#6730](https://github.com/XRPLF/rippled/pull/6730)
|
||||
@@ -52,12 +97,24 @@ This section contains changes targeting a future version.
|
||||
- `submit`: The `fail_hard` field now returns an error if the value is not a boolean. [#6529](https://github.com/XRPLF/rippled/pull/6529)
|
||||
- `subscribe`: The `taker` field in the `books` array now returns `actMalformed` instead of `badIssuer` if the value is not a valid account. [#6529](https://github.com/XRPLF/rippled/pull/6529)
|
||||
- Fixed a bug in `Forwarded` HTTP header parsing where the extracted IP address could be incorrect when no comma or semicolon delimiter follows the address. This could cause the server to misidentify a client's IP address when operating behind a reverse proxy. [#6529](https://github.com/XRPLF/rippled/pull/6529)
|
||||
- `vault_info`: Errors now identify what the request got wrong instead of reporting every failure as the unregistered token `malformedRequest`, and the `error`, `error_code` and `error_message` fields now agree with each other. An invalid `vault_id` or `seq` returns `invalidParams`, an invalid `owner` returns `actMalformed`, and a request that mixes `vault_id` with `owner`/`seq` or supplies neither returns `invalidParams` with a message naming the accepted combinations. [#8015](https://github.com/XRPLF/rippled/pull/8015)
|
||||
- `vault_info`: A well-formed all-zero `vault_id` now returns `entryNotFound` instead of being rejected as malformed, and `entryNotFound` responses now include `error_code` and `error_message`. Clients that request `ripplerpc` 3.0 or above therefore receive HTTP 400 with that error rather than HTTP 200. [#8015](https://github.com/XRPLF/rippled/pull/8015)
|
||||
- `vault_info`: `vault_id` and `owner` must now be strings, matching how `ledger_entry` reads the same fields. An object or an array in either field previously produced an internal error, and a number was silently converted to its decimal text; `vault_id` now returns `invalidParams` and `owner` returns `actMalformed`. [#8015](https://github.com/XRPLF/rippled/pull/8015)
|
||||
- `gateway_balances`: The `account` and `ident` fields now return an `invalidParams` error if the value is not a string, instead of an `internal` error. [#7655](https://github.com/XRPLF/rippled/pull/7655)
|
||||
- `account_lines`: The `peer` field now returns an error if the value is not a string. [#7728](https://github.com/XRPLF/rippled/pull/7728)
|
||||
- `ledger`: `delivered_amount` is now included in the metadata of successful `AccountDelete` transactions when transactions are expanded (`expand`, or admin-only `full`). Previously it was only added for `Payment` and `CheckCash`, which made `ledger` inconsistent with `tx` and `account_tx`. [#5706](https://github.com/XRPLF/rippled/pull/5706)
|
||||
|
||||
## XRP Ledger server version 3.1.3
|
||||
|
||||
[Version 3.1.3](https://github.com/XRPLF/rippled/releases/tag/3.1.3) was released on May 8, 2026.
|
||||
|
||||
This release contains bug fixes only and no API changes.
|
||||
|
||||
## XRP Ledger server version 3.1.2
|
||||
|
||||
[Version 3.1.2](https://github.com/XRPLF/rippled/releases/tag/3.1.2) was released on Mar 12, 2026.
|
||||
|
||||
This release contains bug fixes only and no API changes.
|
||||
|
||||
## XRP Ledger server version 3.1.1
|
||||
|
||||
[Version 3.1.1](https://github.com/XRPLF/rippled/releases/tag/3.1.1) was released on Feb 23, 2026.
|
||||
|
||||
This release contains bug fixes only and no API changes.
|
||||
|
||||
## XRP Ledger server version 3.1.0
|
||||
|
||||
@@ -207,7 +264,9 @@ This release contains bug fixes only and no API changes.
|
||||
- Adds `AMMDelete` transaction type to delete `AMM` instance.
|
||||
- Adds `sfAMMID` to `AccountRoot` to indicate that the account is `AMM`'s account. `AMMID` is used to fetch `ltAMM`.
|
||||
- Adds `lsfAMMNode` `TrustLine` flag to indicate that one side of the `TrustLine` is `AMM` account.
|
||||
- Adds `tfLPToken`, `tfSingleAsset`, `tfTwoAsset`, `tfOneAssetLPToken`, `tfLimitLPToken`, `tfTwoAssetIfEmpty`, `tfWithdrawAll`, `tfOneAssetWithdrawAll` which allow a trader to specify different fields combination for `AMMDeposit` and `AMMWithdraw` transactions.
|
||||
- Adds `tfLPToken`, `tfSingleAsset`, `tfTwoAsset`, `tfOneAssetLPToken`, `tfLimitLPToken`, `tfTwoAssetIfEmpty`,
|
||||
`tfWithdrawAll`, `tfOneAssetWithdrawAll` which allow a trader to specify different fields combination
|
||||
for `AMMDeposit` and `AMMWithdraw` transactions.
|
||||
- Adds new transaction result codes:
|
||||
- tecUNFUNDED_AMM: insufficient balance to fund AMM. The account does not have funds for liquidity provision.
|
||||
- tecAMM_BALANCE: AMM has invalid balance. Calculated balances greater than the current pool balances.
|
||||
@@ -250,11 +309,14 @@ This release contains bug fixes only and no API changes.
|
||||
|
||||
## XRP Ledger server version 1.10.0
|
||||
|
||||
[Version 1.10.0](https://github.com/XRPLF/rippled/releases/tag/1.10.0) was released on Mar 14, 2023.
|
||||
[Version 1.10.0](https://github.com/XRPLF/rippled/releases/tag/1.10.0)
|
||||
was released on Mar 14, 2023.
|
||||
|
||||
### Breaking changes in 1.10
|
||||
|
||||
- If the `XRPFees` feature is enabled, the `fee_ref` field will be removed from the [ledger subscription stream](https://xrpl.org/subscribe.html#ledger-stream), because it will no longer have any meaning.
|
||||
- If the `XRPFees` feature is enabled, the `fee_ref` field will be
|
||||
removed from the [ledger subscription stream](https://xrpl.org/subscribe.html#ledger-stream), because it will no longer
|
||||
have any meaning.
|
||||
|
||||
# Unit tests for API changes
|
||||
|
||||
|
||||
+2
-1
@@ -13,7 +13,8 @@ In API version 2, the following deprecated methods are no longer available: ([#4
|
||||
|
||||
## Modifications to JSON transaction element in API version 2
|
||||
|
||||
In API version 2, JSON elements for transaction output have been changed and made consistent for all methods which output transactions. ([#4775](https://github.com/XRPLF/rippled/pull/4775)) This helps to unify the JSON serialization format of transactions. ([clio#722](https://github.com/XRPLF/clio/issues/722), [#4727](https://github.com/XRPLF/rippled/issues/4727))
|
||||
In API version 2, JSON elements for transaction output have been changed and made consistent for all methods which output transactions. ([#4775](https://github.com/XRPLF/rippled/pull/4775))
|
||||
This helps to unify the JSON serialization format of transactions. ([clio#722](https://github.com/XRPLF/clio/issues/722), [#4727](https://github.com/XRPLF/rippled/issues/4727))
|
||||
|
||||
- JSON transaction element is named `tx_json`
|
||||
- Binary transaction element is named `tx_blob`
|
||||
|
||||
@@ -1,70 +1,95 @@
|
||||
| :warning: **WARNING** :warning: |
|
||||
| --- |
|
||||
| :warning: **WARNING** :warning: |
|
||||
| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| These instructions assume you have a C++ development environment ready with Git, Python, Conan, CMake, and a C++ compiler. For help setting one up on Linux, macOS, or Windows, [see this guide](./docs/build/environment.md).<br><br>These instructions also assume a basic familiarity with Conan and CMake. If you are unfamiliar with Conan, you can read our [crash course](./docs/build/conan.md) or the official [Getting Started][conan-getting-started] walkthrough. |
|
||||
|
||||
## Minimum Requirements
|
||||
|
||||
For the hardware needed to run a node, see [System Requirements](https://xrpl.org/system-requirements.html).
|
||||
For the hardware needed to run a node, see
|
||||
[System Requirements](https://xrpl.org/system-requirements.html).
|
||||
|
||||
For the software needed to build xrpld, see the [environment setup guide](./docs/build/environment.md).
|
||||
For the software needed to build xrpld, see the
|
||||
[environment setup guide](./docs/build/environment.md).
|
||||
|
||||
## Operating Systems
|
||||
|
||||
### Linux
|
||||
|
||||
The Ubuntu Linux distribution has received the highest level of quality assurance, testing, and support. We also support Red Hat and use Debian internally. Our Linux CI tooling is distro-independent and uses a Nix-based environment, so it should be possible to build on other Linux distributions as well, although we have not tested them.
|
||||
The Ubuntu Linux distribution has received the highest level of quality
|
||||
assurance, testing, and support. We also support Red Hat and use Debian
|
||||
internally.
|
||||
Our Linux CI tooling is distro-independent and uses a Nix-based environment, so it should be possible to build on other Linux distributions as well, although we have not tested them.
|
||||
|
||||
### macOS
|
||||
|
||||
Many `xrpld` engineers use macOS for development. The minimum supported version is macOS 15 (Sequoia). CI testing is done in macOS 26 (Tahoe), but the build defaults `CMAKE_OSX_DEPLOYMENT_TARGET` to 15.
|
||||
Many `xrpld` engineers use macOS for development.
|
||||
The minimum supported version is macOS 15 (Sequoia).
|
||||
CI testing is done in macOS 26 (Tahoe), but the build defaults `CMAKE_OSX_DEPLOYMENT_TARGET` to 15.
|
||||
|
||||
### Windows
|
||||
|
||||
Windows is used by some engineers for development only, and is not recommended for production use.
|
||||
Windows is used by some engineers for development only, and is not recommended
|
||||
for production use.
|
||||
|
||||
## Steps
|
||||
|
||||
### Branches
|
||||
|
||||
For the latest set of untested features, or to contribute, choose the `develop` branch.
|
||||
For the latest set of untested features, or to contribute, choose the `develop`
|
||||
branch.
|
||||
|
||||
```bash
|
||||
git checkout develop
|
||||
```
|
||||
|
||||
For a release candidate, choose the relevant release branch, e.g. `release/3.2.x`.
|
||||
For a release or release candidate, check out its [tag](https://github.com/XRPLF/rippled/releases), e.g.:
|
||||
|
||||
```bash
|
||||
git checkout release/3.2.x
|
||||
git checkout 3.4.0
|
||||
```
|
||||
|
||||
For a stable release, choose one of the [tagged releases](https://github.com/XRPLF/rippled/releases).
|
||||
See [RELEASING.md](./RELEASING.md) for how branches and releases are organized.
|
||||
|
||||
A build reports `0.0.0-dev` with its short commit hash as build metadata, e.g.
|
||||
`0.0.0-dev+0123abc`. Only builds of a release, from a tag in CI or a versioned
|
||||
Conan reference, report a release version. To use another version, set the
|
||||
`FORCE_XRPLD_VERSION` environment variable when running CMake, e.g.
|
||||
`FORCE_XRPLD_VERSION=3.4.0`.
|
||||
|
||||
### Set Up Conan
|
||||
|
||||
Once your [development environment](./docs/build/environment.md) is ready, set Conan up for this repository:
|
||||
Once your [development environment](./docs/build/environment.md) is ready, set
|
||||
Conan up for this repository:
|
||||
|
||||
```bash
|
||||
./conan/init.sh
|
||||
```
|
||||
|
||||
That installs our [`global.conf`](./conan/global.conf), our Conan [profiles](./conan/profiles), and the `xrplf` remote that hosts some of our dependencies. It honours `CONAN_HOME` and never deletes an existing Conan home, so it is safe to re-run — it only overwrites the files it manages.
|
||||
That installs our [`global.conf`](./conan/global.conf), our Conan
|
||||
[profiles](./conan/profiles), and the `xrplf` remote that hosts some of our
|
||||
dependencies. It honours `CONAN_HOME` and never deletes an existing Conan home,
|
||||
so it is safe to re-run — it only overwrites the files it manages.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!TIP]
|
||||
> In the [Nix development shell](./docs/build/nix.md#conan-configuration) this is already done for you: the script runs on entry.
|
||||
> In the [Nix development shell](./docs/build/nix.md#conan-configuration) this is
|
||||
> already done for you: the script runs on entry.
|
||||
|
||||
You can inspect the resulting profile with `conan profile show`. If it is not suitable for your environment, create a custom profile and pass it to Conan — see [Advanced Conan configuration](./docs/build/advanced_conan.md).
|
||||
You can inspect the resulting profile with `conan profile show`. If it is not
|
||||
suitable for your environment, create a custom profile and pass it to Conan — see
|
||||
[Advanced Conan configuration](./docs/build/advanced_conan.md).
|
||||
|
||||
### Set Up Ccache
|
||||
|
||||
To speed up repeated compilations, we recommend that you install [ccache](https://ccache.dev), a tool that wraps your compiler so that it can cache build objects locally.
|
||||
To speed up repeated compilations, we recommend that you install
|
||||
[ccache](https://ccache.dev), a tool that wraps your compiler so that it can
|
||||
cache build objects locally.
|
||||
|
||||
On Linux and macOS, `ccache` is included in the [Nix development shell](./docs/build/nix.md).
|
||||
|
||||
#### Windows
|
||||
|
||||
You can install it using Chocolatey, i.e. `choco install ccache`. If you already have Ccache installed, then `choco upgrade ccache` will update it to the latest version. However, if you see an error such as:
|
||||
You can install it using Chocolatey, i.e. `choco install ccache`. If you already
|
||||
have Ccache installed, then `choco upgrade ccache` will update it to the latest
|
||||
version. However, if you see an error such as:
|
||||
|
||||
```
|
||||
terminate called after throwing an instance of 'std::bad_alloc'
|
||||
@@ -72,7 +97,8 @@ terminate called after throwing an instance of 'std::bad_alloc'
|
||||
C:\Program Files\Microsoft Visual Studio\2022\Community\MSBuild\Microsoft\VC\v170\Microsoft.CppCommon.targets(617,5): error MSB6006: "cl.exe" exited with code 3.
|
||||
```
|
||||
|
||||
then please install a specific version of Ccache that we know works, via: `choco install ccache --version 4.11.3 --allow-downgrade`.
|
||||
then please install a specific version of Ccache that we know works, via: `choco
|
||||
install ccache --version 4.11.3 --allow-downgrade`.
|
||||
|
||||
### Build and Test
|
||||
|
||||
@@ -83,9 +109,14 @@ then please install a specific version of Ccache that we know works, via: `choco
|
||||
cd .build
|
||||
```
|
||||
|
||||
You can use any directory name. Conan treats your working directory as an install folder and generates files with implementation details. You don't need to worry about these files, but make sure to change your working directory to your build directory before calling Conan.
|
||||
You can use any directory name. Conan treats your working directory as an
|
||||
install folder and generates files with implementation details.
|
||||
You don't need to worry about these files, but make sure to change
|
||||
your working directory to your build directory before calling Conan.
|
||||
|
||||
**Note:** You can specify a directory for the installation files by adding the `install-folder` or `-if` option to every `conan install` command in the next step.
|
||||
**Note:** You can specify a directory for the installation files by adding
|
||||
the `install-folder` or `-if` option to every `conan install` command
|
||||
in the next step.
|
||||
|
||||
2. Use conan to generate CMake files for every configuration you want to build:
|
||||
|
||||
@@ -96,15 +127,25 @@ then please install a specific version of Ccache that we know works, via: `choco
|
||||
|
||||
To build Debug, in the next step, be sure to set `-DCMAKE_BUILD_TYPE=Debug`
|
||||
|
||||
For a single-configuration generator, e.g. `Unix Makefiles` or `Ninja`, you only need to run this command once. For a multi-configuration generator, e.g. `Visual Studio`, you may want to run it more than once.
|
||||
For a single-configuration generator, e.g. `Unix Makefiles` or `Ninja`,
|
||||
you only need to run this command once.
|
||||
For a multi-configuration generator, e.g. `Visual Studio`, you may want to
|
||||
run it more than once.
|
||||
|
||||
Each of these commands should also have a different `build_type` setting. A second command with the same `build_type` setting will overwrite the files generated by the first. You can pass the build type on the command line with `--settings build_type=$BUILD_TYPE` or in the profile itself, under the section `[settings]` with the key `build_type`.
|
||||
Each of these commands should also have a different `build_type` setting.
|
||||
A second command with the same `build_type` setting will overwrite the files
|
||||
generated by the first. You can pass the build type on the command line with
|
||||
`--settings build_type=$BUILD_TYPE` or in the profile itself,
|
||||
under the section `[settings]` with the key `build_type`.
|
||||
|
||||
3. Configure CMake and pass the toolchain file generated by Conan, located at `$OUTPUT_FOLDER/build/generators/conan_toolchain.cmake`.
|
||||
3. Configure CMake and pass the toolchain file generated by Conan, located at
|
||||
`$OUTPUT_FOLDER/build/generators/conan_toolchain.cmake`.
|
||||
|
||||
Single-config generators:
|
||||
|
||||
Pass the CMake variable [`CMAKE_BUILD_TYPE`][build_type] and make sure it matches the one of the `build_type` settings you chose in the previous step.
|
||||
Pass the CMake variable [`CMAKE_BUILD_TYPE`][build_type]
|
||||
and make sure it matches the one of the `build_type` settings
|
||||
you chose in the previous step.
|
||||
|
||||
For example, to build Debug, in the next command, replace "Release" with "Debug"
|
||||
|
||||
@@ -122,7 +163,9 @@ then please install a specific version of Ccache that we know works, via: `choco
|
||||
|
||||
4. Build `xrpld`.
|
||||
|
||||
For a single-configuration generator, it will build whatever configuration you passed for `CMAKE_BUILD_TYPE`. For a multi-configuration generator, you must pass the option `--config` to select the build configuration.
|
||||
For a single-configuration generator, it will build whatever configuration
|
||||
you passed for `CMAKE_BUILD_TYPE`. For a multi-configuration generator, you
|
||||
must pass the option `--config` to select the build configuration.
|
||||
|
||||
Single-config generators:
|
||||
|
||||
@@ -137,7 +180,8 @@ then please install a specific version of Ccache that we know works, via: `choco
|
||||
cmake --build . --config Debug --parallel N
|
||||
```
|
||||
|
||||
Replace the `--parallel` parameter N with the desired number of parallel jobs. A common starting point is half of the number of available CPU cores.
|
||||
Replace the `--parallel` parameter N with the desired number of parallel jobs. A common starting point is half of the number of available CPU
|
||||
cores.
|
||||
|
||||
5. Test xrpld.
|
||||
|
||||
@@ -154,20 +198,28 @@ then please install a specific version of Ccache that we know works, via: `choco
|
||||
./Debug/xrpld --unittest --unittest-jobs N
|
||||
```
|
||||
|
||||
Replace the `--unittest-jobs` parameter N with the desired unit tests concurrency. Recommended setting is half of the number of available CPU cores.
|
||||
Replace the `--unittest-jobs` parameter N with the desired unit tests
|
||||
concurrency. Recommended setting is half of the number of available CPU
|
||||
cores.
|
||||
|
||||
The location of `xrpld` binary in your build directory depends on your CMake generator. Pass `--help` to see the rest of the command line options.
|
||||
The location of `xrpld` binary in your build directory depends on your
|
||||
CMake generator. Pass `--help` to see the rest of the command line options.
|
||||
|
||||
## Code generation
|
||||
|
||||
The protocol wrapper classes in `include/xrpl/protocol_autogen/` are generated from macro definition files in `include/xrpl/protocol/detail/`. If you modify the macro files (e.g. `transactions.macro`, `ledger_entries.macro`) or the generation scripts/templates in `cmake/scripts/codegen/`, you need to regenerate the files:
|
||||
The protocol wrapper classes in `include/xrpl/protocol_autogen/` are generated
|
||||
from macro definition files in `include/xrpl/protocol/detail/`. If you modify
|
||||
the macro files (e.g. `transactions.macro`, `ledger_entries.macro`) or the
|
||||
generation scripts/templates in `cmake/scripts/codegen/`, you need to regenerate the
|
||||
files:
|
||||
|
||||
```
|
||||
cmake --build . --target setup_code_gen # create venv and install dependencies (once)
|
||||
cmake --build . --target code_gen # regenerate code
|
||||
```
|
||||
|
||||
The same targets are also available as a standalone project, which does not need the dependencies to be configured first:
|
||||
The same targets are also available as a standalone project, which does not
|
||||
need the dependencies to be configured first:
|
||||
|
||||
```
|
||||
cmake -S cmake/codegen -B build/codegen
|
||||
@@ -175,11 +227,15 @@ cmake --build build/codegen --target setup_code_gen
|
||||
cmake --build build/codegen --target code_gen
|
||||
```
|
||||
|
||||
The regenerated files should be committed alongside your changes. CI verifies that they are up-to-date.
|
||||
The regenerated files should be committed alongside your changes. CI verifies
|
||||
that they are up-to-date.
|
||||
|
||||
## Coverage report
|
||||
|
||||
The coverage report is intended for developers using compilers GCC or Clang (including Apple Clang). It is generated by the build target `coverage`, which is only enabled when the `coverage` option is set, e.g. with `--options coverage=True` in `conan` or `-Dcoverage=ON` variable in `cmake`
|
||||
The coverage report is intended for developers using compilers GCC
|
||||
or Clang (including Apple Clang). It is generated by the build target `coverage`,
|
||||
which is only enabled when the `coverage` option is set, e.g. with
|
||||
`--options coverage=True` in `conan` or `-Dcoverage=ON` variable in `cmake`
|
||||
|
||||
Prerequisites for the coverage report:
|
||||
|
||||
@@ -187,19 +243,34 @@ Prerequisites for the coverage report:
|
||||
- `gcov` for GCC or `llvm-cov` for Clang, usually installed with the compiler
|
||||
- `Debug` build type
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!NOTE]
|
||||
> Clang coverage is not available in the [Nix development shell](./docs/build/nix.md#building-xrpld-in-the-nix-shell): its `clang` shells do not ship `llvm-cov`. Use a `gcc` shell instead (`.#gcc`, or `.#gcc-plain` on Linux), which provides a `gcov` matching its compiler.
|
||||
> Clang coverage is not available in the [Nix development shell](./docs/build/nix.md#building-xrpld-in-the-nix-shell):
|
||||
> its `clang` shells do not ship `llvm-cov`. Use a `gcc` shell instead (`.#gcc`,
|
||||
> or `.#gcc-plain` on Linux), which provides a `gcov` matching its compiler.
|
||||
|
||||
A coverage report is created when the following steps are completed, in order:
|
||||
|
||||
1. `xrpld` binary built with instrumentation data, enabled by the `coverage` option mentioned above
|
||||
1. `xrpld` binary built with instrumentation data, enabled by the `coverage`
|
||||
option mentioned above
|
||||
2. completed one or more run of the unit tests, which populates coverage capture data
|
||||
3. completed run of the `gcovr` tool (which internally invokes either `gcov` or `llvm-cov`) to assemble both instrumentation data and the coverage capture data into a coverage report
|
||||
3. completed run of the `gcovr` tool (which internally invokes either `gcov` or `llvm-cov`)
|
||||
to assemble both instrumentation data and the coverage capture data into a coverage report
|
||||
|
||||
The last step of the above is automated into a single target `coverage`. The instrumented `xrpld` binary can also be used for regular development or testing work, at the cost of extra disk space utilization and a small performance hit (to store coverage capture data). Since `xrpld` binary is simply a dependency of the coverage report target, it is possible to re-run the `coverage` target without rebuilding the `xrpld` binary. Note, running of the unit tests before the `coverage` target is left to the developer. Each such run will append to the coverage data collected in the build directory.
|
||||
The last step of the above is automated into a single target `coverage`. The instrumented
|
||||
`xrpld` binary can also be used for regular development or testing work, at
|
||||
the cost of extra disk space utilization and a small performance hit
|
||||
(to store coverage capture data). Since `xrpld` binary is simply a dependency of the
|
||||
coverage report target, it is possible to re-run the `coverage` target without
|
||||
rebuilding the `xrpld` binary. Note, running of the unit tests before the `coverage`
|
||||
target is left to the developer. Each such run will append to the coverage data
|
||||
collected in the build directory.
|
||||
|
||||
The default coverage report format is `html-details`, but the user can override it to any of the formats listed in `Builds/CMake/CodeCoverage.cmake` by setting the `coverage_format` variable in `cmake`. It is also possible to generate more than one format at a time by setting the `coverage_extra_args` variable in `cmake`. The specific command line used to run the `gcovr` tool will be displayed if the `CODE_COVERAGE_VERBOSE` variable is set.
|
||||
The default coverage report format is `html-details`, but the user
|
||||
can override it to any of the formats listed in `Builds/CMake/CodeCoverage.cmake`
|
||||
by setting the `coverage_format` variable in `cmake`. It is also possible
|
||||
to generate more than one format at a time by setting the `coverage_extra_args`
|
||||
variable in `cmake`. The specific command line used to run the `gcovr` tool will be
|
||||
displayed if the `CODE_COVERAGE_VERBOSE` variable is set.
|
||||
|
||||
Example use with some cmake variables set:
|
||||
|
||||
@@ -210,14 +281,16 @@ cmake -DCMAKE_BUILD_TYPE=Debug -Dcoverage=ON -Dxrpld=ON -Dtests=ON -Dcoverage_te
|
||||
cmake --build . --target coverage
|
||||
```
|
||||
|
||||
After the `coverage` target is completed, the generated coverage report will be stored inside the build directory, as either of:
|
||||
After the `coverage` target is completed, the generated coverage report will be
|
||||
stored inside the build directory, as either of:
|
||||
|
||||
- file named `coverage.`_extension_, with a suitable extension for the report format, or
|
||||
- directory named `coverage`, with the `index.html` and other files inside, for the `html-details` or `html-nested` report formats.
|
||||
|
||||
## Sanitizers
|
||||
|
||||
To build dependencies and xrpld with sanitizer instrumentation, set the `SANITIZERS` environment variable when running `conan install` and use the `sanitizers` profile:
|
||||
To build dependencies and xrpld with sanitizer instrumentation, set the
|
||||
`SANITIZERS` environment variable when running `conan install` and use the `sanitizers` profile:
|
||||
|
||||
```bash
|
||||
export SANITIZERS=address,undefinedbehavior
|
||||
@@ -231,27 +304,42 @@ See [Sanitizers docs](./docs/build/sanitizers.md) for more details.
|
||||
|
||||
## Options
|
||||
|
||||
| Option | Default Value | Description |
|
||||
| --- | --- | --- |
|
||||
| `assert` | OFF | Force enabling assertions. |
|
||||
| `coverage` | OFF | Prepare the coverage report. |
|
||||
| `rust` | OFF | Build the Rust crates and the C++ code that depends on them. |
|
||||
| `tests` | OFF | Build tests. |
|
||||
| `unity` | OFF | Configure a unity build. |
|
||||
| `verify_headers` | ON | Make the `verify-headers` target available to compile each header on its own. |
|
||||
| `xrpld` | OFF | Build the xrpld application, and not just the libxrpl library. |
|
||||
| `werr` | OFF | Treat compilation warnings as errors |
|
||||
| `wextra` | OFF | Enable additional compilation warnings |
|
||||
| Option | Default Value | Description |
|
||||
| ---------------- | ------------- | ----------------------------------------------------------------------------- |
|
||||
| `assert` | OFF | Force enabling assertions. |
|
||||
| `coverage` | OFF | Prepare the coverage report. |
|
||||
| `rust` | OFF | Build the Rust crates and the C++ code that depends on them. |
|
||||
| `tests` | OFF | Build tests. |
|
||||
| `unity` | OFF | Configure a unity build. |
|
||||
| `verify_headers` | ON | Make the `verify-headers` target available to compile each header on its own. |
|
||||
| `xrpld` | OFF | Build the xrpld application, and not just the libxrpl library. |
|
||||
| `werr` | OFF | Treat compilation warnings as errors |
|
||||
| `wextra` | OFF | Enable additional compilation warnings |
|
||||
|
||||
[Unity builds][unity-build] may be faster for the first build (at the cost of much more memory) since they concatenate sources into fewer translation units. Non-unity builds may be faster for incremental builds, and can be helpful for detecting `#include` omissions.
|
||||
[Unity builds][unity-build] may be faster for the first build (at the cost of much more
|
||||
memory) since they concatenate sources into fewer translation units. Non-unity
|
||||
builds may be faster for incremental builds, and can be helpful for detecting
|
||||
`#include` omissions.
|
||||
|
||||
### Rust crates
|
||||
|
||||
The Rust crates in `crates/` are only part of the build when `rust` is ON. With `-Drust=OFF` (the default) the `crates` directory is not added to the build, no cxxbridge bindings are generated, and the C++ tests that exercise the Rust interop are not compiled — so no Rust toolchain is needed. CI builds always pass `-Drust=ON`.
|
||||
The Rust crates in `crates/` are only part of the build when `rust` is ON. With
|
||||
`-Drust=OFF` (the default) the `crates` directory is not added to the build, no
|
||||
cxxbridge bindings are generated, and the C++ tests that exercise the Rust
|
||||
interop are not compiled — so no Rust toolchain is needed. CI builds always pass
|
||||
`-Drust=ON`.
|
||||
|
||||
With `-Drust=ON` you need one extra dependency: a Rust toolchain (`cargo`, `rustc`) matching the channel pinned in [`rust-toolchain.toml`](./rust-toolchain.toml), which compiles the crates and generates the cxxbridge bindings. It is provided by the [Nix development shell](./docs/build/nix.md), so `-Drust=ON` works there without any extra setup; otherwise install it as described in [Rust](./docs/build/environment.md#rust).
|
||||
With `-Drust=ON` you need one extra dependency: a Rust toolchain (`cargo`,
|
||||
`rustc`) matching the channel pinned in
|
||||
[`rust-toolchain.toml`](./rust-toolchain.toml), which compiles the crates and
|
||||
generates the cxxbridge bindings. It is provided by the
|
||||
[Nix development shell](./docs/build/nix.md), so `-Drust=ON` works there without
|
||||
any extra setup; otherwise install it as described in
|
||||
[Rust](./docs/build/environment.md#rust).
|
||||
|
||||
The crates also have their own Rust unit tests. Those are run with `cargo` and need only the Rust toolchain, independently of CMake and of the `rust` option (CI runs them with `cargo nextest`):
|
||||
The crates also have their own Rust unit tests. Those are run with `cargo` and
|
||||
need only the Rust toolchain, independently of CMake and of the `rust` option
|
||||
(CI runs them with `cargo nextest`):
|
||||
|
||||
```bash
|
||||
cargo test --manifest-path crates/Cargo.toml --workspace
|
||||
@@ -259,13 +347,22 @@ cargo test --manifest-path crates/Cargo.toml --workspace
|
||||
|
||||
### Verifying headers
|
||||
|
||||
The regular build only compiles `.cpp` files, so a header is only ever checked through whatever translation unit happens to include it. A header that forgets an `#include` is not caught as long as every `.cpp` that uses it includes its missing dependency first. The `verify_headers` option (ON by default) adds a `verify-headers` target that compiles every header on its own, which fails if a header is not self-contained:
|
||||
The regular build only compiles `.cpp` files, so a header is only ever checked
|
||||
through whatever translation unit happens to include it. A header that forgets
|
||||
an `#include` is not caught as long as every `.cpp` that uses it includes its
|
||||
missing dependency first. The `verify_headers` option (ON by default) adds a
|
||||
`verify-headers` target that compiles every header on its own, which fails if a
|
||||
header is not self-contained:
|
||||
|
||||
```bash
|
||||
cmake --build . --target verify-headers
|
||||
```
|
||||
|
||||
The per-header objects are excluded from the `all` target, so a normal build never compiles them; they are built only through `verify-headers`. The generated translation units do appear in `compile_commands.json`, so clang-tidy (and clangd and IDEs) can lint each header on its own. Pass `-Dverify_headers=OFF` to omit them entirely.
|
||||
The per-header objects are excluded from the `all` target, so a normal build
|
||||
never compiles them; they are built only through `verify-headers`. The generated
|
||||
translation units do appear in `compile_commands.json`, so clang-tidy (and
|
||||
clangd and IDEs) can lint each header on its own. Pass `-Dverify_headers=OFF` to
|
||||
omit them entirely.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
@@ -292,15 +389,20 @@ After any updates or changes to dependencies, you may need to do the following:
|
||||
4. [Regenerate lockfile](./docs/build/advanced_conan.md#conan-lockfile).
|
||||
5. Re-run [conan install](#build-and-test).
|
||||
|
||||
If you are using the Nix development shell, whether prebuilt Conan binaries apply depends on your platform — see [Prebuilt packages](./docs/build/nix.md#prebuilt-packages).
|
||||
If you are using the Nix development shell, whether prebuilt Conan binaries apply
|
||||
depends on your platform — see
|
||||
[Prebuilt packages](./docs/build/nix.md#prebuilt-packages).
|
||||
|
||||
#### ERROR: Package not resolved
|
||||
|
||||
If you're seeing an error like `ERROR: Package 'snappy/1.1.10' not resolved: Unable to find 'snappy/1.1.10#968fef506ff261592ec30c574d4a7809%1756234314.246' in remotes.`, please [set Conan up](#set-up-conan) so the `xrplf` remote is configured, or re-run `conan export` for [patched recipes](./docs/build/advanced_conan.md#patched-recipes).
|
||||
If you're seeing an error like `ERROR: Package 'snappy/1.1.10' not resolved: Unable to find 'snappy/1.1.10#968fef506ff261592ec30c574d4a7809%1756234314.246' in remotes.`,
|
||||
please [set Conan up](#set-up-conan) so the `xrplf` remote is configured, or re-run `conan export` for [patched recipes](./docs/build/advanced_conan.md#patched-recipes).
|
||||
|
||||
### `protobuf/port_def.inc` file not found
|
||||
|
||||
If `cmake --build .` results in an error due to a missing a protobuf file, then you might have generated CMake files for a different `build_type` than the `CMAKE_BUILD_TYPE` you passed to Conan.
|
||||
If `cmake --build .` results in an error due to a missing a protobuf file, then
|
||||
you might have generated CMake files for a different `build_type` than the
|
||||
`CMAKE_BUILD_TYPE` you passed to Conan.
|
||||
|
||||
```
|
||||
/xrpld/.build/pb-xrpl.libpb/xrpl/proto/xrpl.pb.h:10:10: fatal error: 'google/protobuf/port_def.inc' file not found
|
||||
|
||||
+2
-1
@@ -77,7 +77,6 @@ endif()
|
||||
include(PatchNixBinary)
|
||||
|
||||
include(XrplSanity)
|
||||
include(XrplVersion)
|
||||
include(XrplSettings)
|
||||
# this check has to remain in the top-level cmake because of the early return statement
|
||||
if(packages_only)
|
||||
@@ -115,6 +114,7 @@ find_package(secp256k1 REQUIRED)
|
||||
find_package(SOCI REQUIRED)
|
||||
find_package(SQLite3 REQUIRED)
|
||||
find_package(xxHash REQUIRED)
|
||||
find_package(xrpl-rpc-spec REQUIRED)
|
||||
|
||||
target_link_libraries(
|
||||
xrpl_libs
|
||||
@@ -175,6 +175,7 @@ include(XrplPackaging)
|
||||
if(tests)
|
||||
include(CTest)
|
||||
add_subdirectory(src/tests/libxrpl)
|
||||
add_subdirectory(src/tests/xrpld)
|
||||
endif()
|
||||
|
||||
if(benchmark)
|
||||
|
||||
+302
-510
File diff suppressed because it is too large.
Load diff
+12
-3
@@ -1,7 +1,16 @@
|
||||
ISC License
|
||||
|
||||
Copyright (c) 2011, Arthur Britto, David Schwartz, Jed McCaleb, Vinnie Falco, Bob Way, Eric Lombrozo, Nikolaos D. Bougalis, Howard Hinnant. Copyright (c) 2012-present, the XRP Ledger developers.
|
||||
Copyright (c) 2011, Arthur Britto, David Schwartz, Jed McCaleb, Vinnie Falco, Bob Way, Eric Lombrozo, Nikolaos D. Bougalis, Howard Hinnant.
|
||||
Copyright (c) 2012-present, the XRP Ledger developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies.
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
@@ -56,7 +56,8 @@ Here are some good places to start learning the source code:
|
||||
| `./src` | Source code. |
|
||||
| `./crates` | Rust source code. |
|
||||
|
||||
Some of the directories under `src` are external repositories included using git-subtree. See those directories' README files for more details.
|
||||
Some of the directories under `src` are external repositories included using
|
||||
git-subtree. See those directories' README files for more details.
|
||||
|
||||
## Additional Documentation
|
||||
|
||||
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
# Branching and Release Management
|
||||
|
||||
This document describes how we branch, release, and merge releases back into `develop`.
|
||||
It does not define version naming
|
||||
(e.g., what constitutes a major-minor, patch, or beta release).
|
||||
|
||||
Examples use `X.Y` for a release line:
|
||||
`X` and `Y` are placeholders, while the trailing `x` is literal,
|
||||
e.g., `release/X.Y.x` is `release/3.4.x` for the `3.4` line.
|
||||
|
||||
## Branches
|
||||
|
||||
| Branch | Purpose |
|
||||
| :-------------- | :---------------------------------------------------------------------------------------- |
|
||||
| `develop` | Main development branch. Betas and the first RC of a major-minor release are tagged here. |
|
||||
| `staging/X.Y.x` | Where fixes for the `X.Y` line are developed and RCs are prepared. |
|
||||
| `release/X.Y.x` | The last public RC or release of the `X.Y` line. |
|
||||
|
||||
A release line is named `X.Y.x`
|
||||
because one branch serves every patch release of that line (`X.Y.0`, `X.Y.1`, `X.Y.2`, ...).
|
||||
The `/` groups branches hierarchically,
|
||||
so tools can filter them and protection rules can target `release/*` and `staging/*`.
|
||||
|
||||
## Principles
|
||||
|
||||
- **Releases are merged back into `develop`, never cherry-picked or rebased onto it.**
|
||||
Cherry-picked and rebased commits get new hashes,
|
||||
so Git can't tell that `develop` already has them.
|
||||
Future merges then replay them and produce artificial conflicts,
|
||||
and it's hard to verify that every fix actually reached `develop`.
|
||||
Merging keeps a single history:
|
||||
Git knows exactly which release commits `develop` contains,
|
||||
and no fix is left behind.
|
||||
- **Branches only move forward.**
|
||||
`develop`, `staging/X.Y.x`, and `release/X.Y.x` are never rewritten.
|
||||
- **Cherry-picking only goes from `develop` to a staging branch**,
|
||||
for fixes that must get into a release after the code freeze
|
||||
(see [Emergency Fixes From `develop`](#emergency-fixes-from-develop)).
|
||||
- **Security fixes are prepared privately and published with the release that contains them**,
|
||||
so vulnerabilities are not disclosed prematurely.
|
||||
|
||||
## Release Lifecycle
|
||||
|
||||
This diagram shows a major-minor release and its first patch release.
|
||||
The staging and release branches are drawn as one line.
|
||||
|
||||
```text
|
||||
develop staging/X.Y.x & release/X.Y.x
|
||||
│
|
||||
├── Tag: X.Y.0-b1
|
||||
├── Tag: X.Y.0-bN
|
||||
├── Tag: X.Y.0-rc1 ───────────────┐ (branches created)
|
||||
│ │
|
||||
│ (development continues) ├── Fixes
|
||||
│ ├── Tag: X.Y.0-rcN
|
||||
│ ├── Tag: X.Y.0 (final)
|
||||
◀──── (merge) ────────────────────┤
|
||||
│ ├── Fixes
|
||||
│ ├── Tag: X.Y.1-rcN
|
||||
│ ├── Tag: X.Y.1 (final)
|
||||
◀──── (merge) ────────────────────┤
|
||||
│
|
||||
▼
|
||||
```
|
||||
|
||||
### Betas, First RC & Branching
|
||||
|
||||
> [!NOTE]
|
||||
> This phase applies only to a new major-minor release (e.g., `X.Y.0`).
|
||||
> Patch releases work on the existing branches of the line.
|
||||
|
||||
1. **Betas:** All beta versions (e.g., `X.Y.0-b1`) are built and tagged directly on `develop`.
|
||||
2. **First RC:** We release the first RC (`X.Y.0-rc1`)
|
||||
once everything that should be included in the release has been merged.
|
||||
3. **Branches:** `staging/X.Y.x` and `release/X.Y.x` are created from `develop`
|
||||
at the commit tagged `X.Y.0-rc1`.
|
||||
The first RC also kicks off the QE process.
|
||||
4. **Code freeze:** No new features or unrelated changes are pulled from `develop`
|
||||
into `staging/X.Y.x` or `release/X.Y.x`.
|
||||
Only critical stabilization fixes go into the line.
|
||||
5. **No large changes on `develop`:** Until `X.Y.0` is [merged back](#merging-back-into-develop),
|
||||
large changes (e.g., big refactors, moving or renaming many files, mass reformatting)
|
||||
are not merged into `develop`,
|
||||
so that fixes on the line and the merge back don't run into conflicts.
|
||||
|
||||
### Release Candidates
|
||||
|
||||
1. Fixes are developed against `staging/X.Y.x`.
|
||||
2. When ready, a new RC is created on `staging/X.Y.x`,
|
||||
and `release/X.Y.x` is fast-forwarded to it.
|
||||
|
||||
RCs that contain unpublished security fixes are not published,
|
||||
and don't touch the public branches.
|
||||
|
||||
RCs are not merged back into `develop`:
|
||||
`X.Y.0-rc1` is tagged on `develop` itself,
|
||||
and all later changes reach `develop` with the [final release](#final-release).
|
||||
|
||||
### Final Release
|
||||
|
||||
Security fixes become public as soon as they reach the public repo,
|
||||
so these steps happen only once the release is ready to be published,
|
||||
one right after the other.
|
||||
|
||||
1. Unpublished security fixes, if any, are merged into `staging/X.Y.x`.
|
||||
2. `release/X.Y.x` is fast-forwarded to `staging/X.Y.x`,
|
||||
and the release is tagged on it.
|
||||
3. The release is immediately [merged back into `develop`](#merging-back-into-develop).
|
||||
For a major-minor release, this lifts the freeze on large changes in `develop`.
|
||||
|
||||
### Merging Back Into `develop`
|
||||
|
||||
The merge back is a regular PR into `develop`
|
||||
whose branch contains a real merge commit of the release tag:
|
||||
|
||||
1. Create a branch from `develop`, run `git merge --no-ff <tag>`, and resolve any conflicts.
|
||||
2. Once the PR is approved, `develop` is fast-forwarded to the PR branch,
|
||||
so the merge commit lands as it is.
|
||||
Never squash or rebase it.
|
||||
Never add it to the merge queue either: the queue squashes PRs, which would drop the merge commit.
|
||||
3. If `develop` moves while the PR is open, redo the merge on top of the new `develop`.
|
||||
|
||||
After the merge, `git log develop..<tag>` must be empty.
|
||||
|
||||
## Special Cases
|
||||
|
||||
### Emergency Fixes From `develop`
|
||||
|
||||
If a commit was merged to `develop`
|
||||
and needs to be included in a release after the code freeze:
|
||||
|
||||
1. Create a PR that cherry-picks the commit onto `staging/X.Y.x`.
|
||||
2. Leave `develop` as it is, with no reverts.
|
||||
3. Follow the [release candidates](#release-candidates) process as usual.
|
||||
|
||||
When the release is later merged back into `develop`,
|
||||
both sides already contain the same change,
|
||||
so the merge usually resolves it cleanly or with a trivial conflict.
|
||||
From Git's perspective, the cherry-picked commit then becomes part of `develop` too.
|
||||
|
||||
### Several Supported Lines
|
||||
|
||||
When a fix must ship in more than one supported line (e.g., `X.Y` and `X.(Y+1)`),
|
||||
the lines are merged upwards rather than cherry-picked between:
|
||||
|
||||
1. The fix goes into the oldest line first.
|
||||
2. After that line's release,
|
||||
its `release/X.Y.x` is merged into the staging branch of the next newer line,
|
||||
and so on up to the newest line.
|
||||
3. The newest line is merged back into `develop` as usual.
|
||||
|
||||
This way every newer line, and eventually `develop`, contains the history of the older lines.
|
||||
@@ -24,9 +24,13 @@
|
||||
# `g++-15`, ...) are probed under both names: a suffixed name can break while
|
||||
# the plain one still works (see mkVersionedToolLinks in nix/packages.nix).
|
||||
#
|
||||
# Tools scoped to a single dev shell rather than to commonPackages are checked
|
||||
# only in that shell, keyed off XRPL_DEVSHELL.
|
||||
#
|
||||
# Environment variables:
|
||||
# CI if set, skip the tools above when on macOS.
|
||||
# CHECK_TOOLS_SKIP_CLONE if set, skip the git-over-HTTPS connectivity check.
|
||||
# XRPL_DEVSHELL active dev shell; selects shell-specific tools.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
@@ -158,10 +162,19 @@ if [ "${os}" = "linux" ] || [ "${os}" = "macos" ]; then
|
||||
check cargo-nextest cargo nextest --version
|
||||
check clippy-driver
|
||||
check rust-analyzer
|
||||
check rust-nightly rust-nightly run rustc --version
|
||||
check rustc
|
||||
check rustfmt
|
||||
fi
|
||||
|
||||
# Lean4 is in the formal-verification shell only, not in commonPackages.
|
||||
if [ "${XRPL_DEVSHELL:-}" = "formal-verification" ]; then
|
||||
echo
|
||||
echo "Formal verification toolchain:"
|
||||
check lean
|
||||
check lake
|
||||
fi
|
||||
|
||||
# GCC is the default compiler on Linux. macOS uses the system Apple Clang
|
||||
# instead, so GCC/g++/gcov are not expected there.
|
||||
if [ "${os}" = "linux" ]; then
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [[ $# -ne 3 || "$1" == "--help" || "$1" = "-h" ]]; then
|
||||
name=$(basename $0)
|
||||
cat <<-USAGE
|
||||
Usage: $name workbranch base/branch version
|
||||
|
||||
* workbranch will be created locally from base/branch. If it exists,
|
||||
it will be reused, so make sure you don't overwrite any work.
|
||||
* base/branch may be specified as user:branch to allow easy copying
|
||||
from Github PRs.
|
||||
USAGE
|
||||
exit 0
|
||||
fi
|
||||
|
||||
work="$1"
|
||||
shift
|
||||
|
||||
base=$(echo "$1" | sed "s/:/\//")
|
||||
shift
|
||||
|
||||
version=$1
|
||||
shift
|
||||
|
||||
set -e
|
||||
|
||||
git fetch upstreams
|
||||
|
||||
git checkout -B "${work}" --no-track "${base}"
|
||||
|
||||
push=$(git rev-parse --abbrev-ref --symbolic-full-name '@{push}' \
|
||||
2>/dev/null) || true
|
||||
if [[ "${push}" != "" ]]; then
|
||||
echo "Warning: ${push} may already exist."
|
||||
fi
|
||||
|
||||
build=$(find -name BuildInfo.cpp)
|
||||
sed 's/\(^.*versionString =\).*$/\1 "'${version}'"/' ${build} >version.cpp &&
|
||||
diff "${build}" version.cpp && exit 1 ||
|
||||
mv -vi version.cpp ${build}
|
||||
|
||||
git diff
|
||||
|
||||
git add ${build}
|
||||
|
||||
git commit -S -m "Set version to ${version}"
|
||||
|
||||
git log --oneline --first-parent ${base}^..
|
||||
|
||||
cat <<PUSH
|
||||
|
||||
-------------------------------------------------------------------
|
||||
This script will not push. Verify everything is correct, then push
|
||||
to your repo, and create a PR as described in CONTRIBUTING.md.
|
||||
-------------------------------------------------------------------
|
||||
PUSH
|
||||
@@ -25,7 +25,9 @@ esac
|
||||
# Packaging runs in a vanilla distro image, so the tooling comes from the distro's
|
||||
# archive rather than from nixpkgs:
|
||||
#
|
||||
# - debhelper and dpkg-dev build the DEB
|
||||
# - debhelper and dpkg-dev build the DEB, and lintian checks it
|
||||
# - binutils gives debian/rules the readelf its glibc-floor check runs; it
|
||||
# already arrives via dpkg-dev, but that tool is called directly
|
||||
# - rpm-build builds the RPM, with systemd-rpm-macros and redhat-rpm-config
|
||||
# supplying the systemd and find-debuginfo macros the spec uses
|
||||
# - rpm-sign and gnupg2 sign the built RPM
|
||||
@@ -37,11 +39,13 @@ function install() {
|
||||
debian | ubuntu)
|
||||
apt-get update -y
|
||||
apt-get install -y --no-install-recommends \
|
||||
binutils \
|
||||
ca-certificates \
|
||||
debhelper \
|
||||
debhelper-compat \
|
||||
dpkg-dev \
|
||||
git \
|
||||
lintian \
|
||||
python3
|
||||
;;
|
||||
|
||||
|
||||
Executable
+149
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Check that Rust unit tests stay out of the coverage report.
|
||||
|
||||
cargo-llvm-cov instruments the test code along with everything else, so a test
|
||||
module that is not excluded counts its own body as covered and inflates the
|
||||
reported number. Excluding it takes two attributes:
|
||||
|
||||
* every `#[cfg(test)]` module carries
|
||||
`#[cfg_attr(coverage_nightly, coverage(off))]`;
|
||||
* every crate root (lib.rs, main.rs) carries
|
||||
`#![cfg_attr(coverage_nightly, feature(coverage_attribute))]`, which the
|
||||
attribute above needs in order to compile.
|
||||
|
||||
Both are inert outside the coverage job: cargo-llvm-cov defines
|
||||
`coverage_nightly` only when it runs on a nightly toolchain.
|
||||
|
||||
The crate-root gate is checked even in a crate that has no tests yet, because
|
||||
that is what lets the first test module added later carry the attribute without
|
||||
a build failure. Missing it is a hard error, so it cannot go unnoticed; a
|
||||
missing `coverage(off)` fails open, which is why this check exists.
|
||||
|
||||
Matching is on exact attribute text, which works because `cargo fmt` runs over
|
||||
the whole workspace in the hook ahead of this one: rustfmt puts every attribute
|
||||
on its own line and normalizes what is inside it, turning `#[cfg( test )]`
|
||||
and `#[cfg(test,)]` alike into `#[cfg(test)]`. So there is nothing here that
|
||||
parses Rust. The price is that a cfg this file does not spell out literally --
|
||||
`all(test, ...)`, `any(test, ...)`, `not(test)` -- is reported rather than
|
||||
classified, on the grounds that guessing at coverage semantics is how a check
|
||||
like this ends up quietly wrong.
|
||||
|
||||
Usage: ./bin/pre-commit/check_rust_coverage_attrs.py <file1> <file2> ...
|
||||
|
||||
Exit status is non-zero if any violation is found.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
CRATE_ROOTS = {"lib.rs", "main.rs"}
|
||||
|
||||
FEATURE_ATTR = "#![cfg_attr(coverage_nightly, feature(coverage_attribute))]"
|
||||
COVERAGE_OFF_ATTR = "#[cfg_attr(coverage_nightly, coverage(off))]"
|
||||
CFG_TEST_ATTR = "#[cfg(test)]"
|
||||
|
||||
# Any other cfg that mentions `test`. String literals are blanked before this
|
||||
# runs, so `feature = "test"` does not read as the `test` cfg.
|
||||
RE_CFG_MENTIONS_TEST = re.compile(r"^#\[cfg\(.*\btest\b.*\)\]$")
|
||||
RE_STRING = re.compile(r'"(?:[^"\\]|\\.)*"')
|
||||
RE_MOD = re.compile(r"^(?:pub(?:\([^)]*\))?\s+)?mod\s+([A-Za-z_]\w*)")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Finding:
|
||||
line: int
|
||||
label: str
|
||||
message: str
|
||||
|
||||
|
||||
def _check_module(attrs: list[str], line: int, name: str) -> list[Finding]:
|
||||
"""Findings for one module, given the attributes attached to it."""
|
||||
if COVERAGE_OFF_ATTR in attrs:
|
||||
return [] # excluded from coverage; which cfg gates it does not matter
|
||||
if CFG_TEST_ATTR in attrs:
|
||||
return [
|
||||
Finding(
|
||||
line,
|
||||
"missing-coverage-off",
|
||||
f"`mod {name}` is #[cfg(test)] but not excluded from coverage; "
|
||||
f"add {COVERAGE_OFF_ATTR}",
|
||||
)
|
||||
]
|
||||
unclassified = [
|
||||
attr for attr in attrs if RE_CFG_MENTIONS_TEST.match(RE_STRING.sub('""', attr))
|
||||
]
|
||||
if unclassified:
|
||||
return [
|
||||
Finding(
|
||||
line,
|
||||
"unclassified-cfg",
|
||||
f"`mod {name}` is gated on {unclassified[0]}, which this check "
|
||||
f"cannot tell apart from a module that ships in the library; "
|
||||
f"add {COVERAGE_OFF_ATTR} if it is test-only, or teach this "
|
||||
f"check the cfg if it is not",
|
||||
)
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def _check_test_modules(lines: list[str]) -> list[Finding]:
|
||||
"""Findings for every test module that is not excluded from coverage."""
|
||||
findings: list[Finding] = []
|
||||
attrs: list[str] = []
|
||||
attrs_line = 0
|
||||
for number, raw in enumerate(lines, start=1):
|
||||
stripped = raw.strip()
|
||||
# Blank lines and comments are allowed between an attribute and its item.
|
||||
if not stripped or stripped.startswith("//"):
|
||||
continue
|
||||
if stripped.startswith("#["):
|
||||
if not attrs:
|
||||
attrs_line = number
|
||||
attrs.append(stripped)
|
||||
continue
|
||||
module = RE_MOD.match(stripped)
|
||||
if module is not None and attrs:
|
||||
findings += _check_module(attrs, attrs_line, module.group(1))
|
||||
attrs = []
|
||||
return findings
|
||||
|
||||
|
||||
def _check_crate_root(name: str, lines: list[str]) -> list[Finding]:
|
||||
"""A finding if a crate root is missing the coverage_attribute feature gate."""
|
||||
if name not in CRATE_ROOTS:
|
||||
return []
|
||||
if any(line.strip() == FEATURE_ATTR for line in lines):
|
||||
return []
|
||||
return [
|
||||
Finding(
|
||||
1,
|
||||
"missing-feature-gate",
|
||||
f"crate root is missing {FEATURE_ATTR}",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def check_source(name: str, text: str) -> list[Finding]:
|
||||
"""Findings for one file's contents; `name` is its base name (lib.rs, ...)."""
|
||||
lines = text.splitlines()
|
||||
return _check_crate_root(name, lines) + _check_test_modules(lines)
|
||||
|
||||
|
||||
def check_file(path: Path) -> list[Finding]:
|
||||
return check_source(path.name, path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
total = 0
|
||||
for path in (Path(name) for name in sys.argv[1:]):
|
||||
for finding in check_file(path):
|
||||
total += 1
|
||||
print(f"{path}:{finding.line}: {finding.label}: {finding.message}")
|
||||
return 1 if total else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+144
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Rewrites gtest names to the required style in this project: the suite name is
|
||||
CamelCase, the test-case name is snake_case.
|
||||
|
||||
TEST(SuiteName, test_case_name)
|
||||
|
||||
The gtest `DISABLED_` prefix is kept verbatim on either name.
|
||||
|
||||
Both conversions fold acronyms the way a reader expects:
|
||||
`SetAndResetAccountTxnID` -> `set_and_reset_account_txn_id`, not
|
||||
`set_and_reset_account_txn_i_d`.
|
||||
|
||||
The first argument of `TEST_F`, `TEST_P`, `TYPED_TEST` and `TYPED_TEST_P` is a
|
||||
fixture class rather than a free identifier, so rewriting it here would leave
|
||||
the class it names behind. Those are reported for a human to rename (clang-tidy
|
||||
checks the class declaration itself, via readability-identifier-naming).
|
||||
|
||||
Usage: ./bin/pre-commit/fix_gtest_names.py <file1> <file2> ...
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
# A test-case definition, `MACRO(SuiteOrFixture, TestName)`, anchored at the
|
||||
# start of a line so that commented-out definitions and project macros that
|
||||
# merely look similar (`TEST_EXPECT(...)`) are left alone. The `\s*` between
|
||||
# arguments allows for a definition clang-format wrapped over several lines.
|
||||
PATTERN = re.compile(
|
||||
r"(?P<head>^[ \t]*(?P<macro>TYPED_TEST_P|TYPED_TEST|TEST_F|TEST_P|TEST)\s*\(\s*)"
|
||||
r"(?P<suite>\w+)(?P<mid>\s*,\s*)(?P<name>\w+)(?P<tail>\s*\))",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
# The macros whose first argument names a fixture class, not a free identifier.
|
||||
FIXTURE_MACROS = ("TEST_F", "TEST_P", "TYPED_TEST", "TYPED_TEST_P")
|
||||
|
||||
DISABLED = "DISABLED_"
|
||||
|
||||
ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])")
|
||||
WORD_BOUNDARY = re.compile(r"([a-z\d])([A-Z])")
|
||||
|
||||
|
||||
def _split_disabled(name: str) -> tuple[str, str]:
|
||||
"""Splits off gtest's `DISABLED_` prefix, which is kept verbatim."""
|
||||
if name.startswith(DISABLED):
|
||||
return DISABLED, name[len(DISABLED) :]
|
||||
return "", name
|
||||
|
||||
|
||||
def snake_case(name: str) -> str:
|
||||
"""Returns the name in snake_case, leaving acronyms whole.
|
||||
|
||||
`SetAndResetAccountTxnID` -> `set_and_reset_account_txn_id`,
|
||||
`parseStatRSSkB` -> `parse_stat_rs_sk_b`.
|
||||
"""
|
||||
prefix, core = _split_disabled(name)
|
||||
core = ACRONYM_BOUNDARY.sub(r"\1_\2", core)
|
||||
return prefix + WORD_BOUNDARY.sub(r"\1_\2", core).lower()
|
||||
|
||||
|
||||
def camel_case(name: str) -> str:
|
||||
"""Returns the name in CamelCase, capitalizing each underscored word.
|
||||
|
||||
Only the letters that have to change are touched, so acronyms survive: a
|
||||
conversion that went via snake_case would turn `SHAMapTest` into
|
||||
`ShaMapTest`, whereas here it is already CamelCase and stays put.
|
||||
`json_value` -> `JsonValue`, `parseStatRSSkB` -> `ParseStatRSSkB`.
|
||||
"""
|
||||
prefix, core = _split_disabled(name)
|
||||
return prefix + "".join(w[:1].upper() + w[1:] for w in core.split("_") if w)
|
||||
|
||||
|
||||
def _corrected(match: re.Match) -> tuple[str, str]:
|
||||
"""Returns the suite and test-case names this definition should end up with."""
|
||||
suite = match["suite"]
|
||||
return (
|
||||
suite if match["macro"] in FIXTURE_MACROS else camel_case(suite),
|
||||
snake_case(match["name"]),
|
||||
)
|
||||
|
||||
|
||||
def fix_source(text: str) -> tuple[str, list[str]]:
|
||||
"""Returns the corrected text and one `line: message` report per bad name."""
|
||||
# gtest joins the suite and test names into one class name, so two test
|
||||
# cases whose joined names agree cannot coexist: `TEST(a, b_c)` and
|
||||
# `TEST(a_b, c)` both define `a_b_c_Test`. A rename that would introduce
|
||||
# such a clash is reported for a human instead of applied.
|
||||
joined = Counter("_".join(_corrected(m)) for m in PATTERN.finditer(text))
|
||||
reports = []
|
||||
|
||||
def rewrite(match: re.Match) -> str:
|
||||
suite, name = match["suite"], match["name"]
|
||||
new_suite, new_name = _corrected(match)
|
||||
line = text.count("\n", 0, match.start()) + 1
|
||||
|
||||
if match["macro"] in FIXTURE_MACROS and camel_case(suite) != suite:
|
||||
reports.append(
|
||||
f"{line}: fixture '{suite}' is not CamelCase: rename the class "
|
||||
f"to '{camel_case(suite)}' by hand"
|
||||
)
|
||||
if (new_suite, new_name) == (suite, name):
|
||||
return match[0]
|
||||
if joined[f"{new_suite}_{new_name}"] > 1:
|
||||
reports.append(
|
||||
f"{line}: cannot rename '{suite}, {name}' to '{new_suite}, "
|
||||
f"{new_name}': another test case already generates that name"
|
||||
)
|
||||
return match[0]
|
||||
if new_suite != suite:
|
||||
reports.append(f"{line}: renamed suite '{suite}' to '{new_suite}'")
|
||||
if new_name != name:
|
||||
reports.append(f"{line}: renamed test case '{name}' to '{new_name}'")
|
||||
return match["head"] + new_suite + match["mid"] + new_name + match["tail"]
|
||||
|
||||
return PATTERN.sub(rewrite, text), reports
|
||||
|
||||
|
||||
def fix_names(path: Path) -> bool:
|
||||
"""Corrects one file's gtest names, reporting each on stdout."""
|
||||
original = path.read_text(encoding="utf-8")
|
||||
fixed, reports = fix_source(original)
|
||||
for report in reports:
|
||||
print(f"{path}:{report}")
|
||||
if fixed != original:
|
||||
path.write_text(fixed, encoding="utf-8")
|
||||
return not reports
|
||||
|
||||
|
||||
def main() -> int:
|
||||
files = [Path(f) for f in sys.argv[1:]]
|
||||
success = True
|
||||
|
||||
for path in files:
|
||||
success &= fix_names(path)
|
||||
|
||||
return 0 if success else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+259
@@ -0,0 +1,259 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tests for fix_gtest_names.py.
|
||||
|
||||
Run directly (no test framework needed):
|
||||
./bin/pre-commit/test_fix_gtest_names.py
|
||||
or under pytest:
|
||||
pytest bin/pre-commit/test_fix_gtest_names.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
from fix_gtest_names import camel_case, fix_source, snake_case
|
||||
|
||||
|
||||
def dedent(text: str) -> str:
|
||||
"""Removes a fixture's common indentation and its leading newline.
|
||||
|
||||
Lets fixtures be written as indented triple-quoted here-docs while keeping
|
||||
honest 1-based line numbers.
|
||||
"""
|
||||
return textwrap.dedent(text).lstrip("\n")
|
||||
|
||||
|
||||
def fixed(text: str) -> str:
|
||||
return fix_source(dedent(text))[0]
|
||||
|
||||
|
||||
def reports(text: str) -> list[str]:
|
||||
return fix_source(dedent(text))[1]
|
||||
|
||||
|
||||
# --- conversion --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_snake_case_conversion() -> None:
|
||||
assert snake_case("BadInputs") == "bad_inputs"
|
||||
assert snake_case("mulDiv") == "mul_div"
|
||||
assert snake_case("already_snake") == "already_snake"
|
||||
assert snake_case("base64") == "base64"
|
||||
|
||||
|
||||
def test_snake_case_keeps_acronyms_whole() -> None:
|
||||
assert snake_case("SetAndResetAccountTxnID") == "set_and_reset_account_txn_id"
|
||||
assert snake_case("XRPToIOU") == "xrp_to_iou"
|
||||
assert snake_case("STAmountMath") == "st_amount_math"
|
||||
|
||||
|
||||
def test_camel_case_conversion() -> None:
|
||||
assert camel_case("json_value") == "JsonValue"
|
||||
assert camel_case("mulDiv") == "MulDiv"
|
||||
assert camel_case("scope") == "Scope"
|
||||
assert camel_case("base64") == "Base64"
|
||||
|
||||
|
||||
def test_camel_case_leaves_acronyms_alone() -> None:
|
||||
# A snake_case round-trip would give `ShaMapTest` / `ParseStatmRsSkB` here.
|
||||
assert camel_case("SHAMapTest") == "SHAMapTest"
|
||||
assert camel_case("parseStatmRSSkB") == "ParseStatmRSSkB"
|
||||
assert camel_case("XRPAmount") == "XRPAmount"
|
||||
assert camel_case("CSPRNG") == "CSPRNG"
|
||||
|
||||
|
||||
def test_disabled_prefix_preserved() -> None:
|
||||
assert snake_case("DISABLED_FooBar") == "DISABLED_foo_bar"
|
||||
assert snake_case("DISABLED_foo_bar") == "DISABLED_foo_bar"
|
||||
assert snake_case("DISABLED_") == "DISABLED_"
|
||||
assert camel_case("DISABLED_foo_bar") == "DISABLED_FooBar"
|
||||
assert camel_case("DISABLED_") == "DISABLED_"
|
||||
|
||||
|
||||
# --- what counts as a test definition ---------------------------------------
|
||||
|
||||
|
||||
def test_all_macros_recognized() -> None:
|
||||
code = """
|
||||
TEST(Suite, oneName)
|
||||
TEST_F(Fixture, twoName)
|
||||
TEST_P(Fixture, threeName)
|
||||
TYPED_TEST(Fixture, fourName)
|
||||
TYPED_TEST_P(Fixture, fiveName)
|
||||
"""
|
||||
assert fixed(code) == dedent("""
|
||||
TEST(Suite, one_name)
|
||||
TEST_F(Fixture, two_name)
|
||||
TEST_P(Fixture, three_name)
|
||||
TYPED_TEST(Fixture, four_name)
|
||||
TYPED_TEST_P(Fixture, five_name)
|
||||
""")
|
||||
|
||||
|
||||
def test_conforming_definitions_untouched() -> None:
|
||||
code = """
|
||||
TEST(AccountSet, bad_inputs)
|
||||
TEST_F(MutexMakeTest, default_constructor)
|
||||
TEST(SHAMap, DISABLED_slow_path)
|
||||
"""
|
||||
assert reports(code) == []
|
||||
assert fixed(code) == dedent(code)
|
||||
|
||||
|
||||
def test_lookalikes_ignored() -> None:
|
||||
code = """
|
||||
// TEST(Suite, notATest)
|
||||
TEST_EXPECT(someCall())
|
||||
TEST_EXPECTS(amount == value, amount.getText())
|
||||
INSTANTIATE_TEST_SUITE_P(Prefix, Fixture, testValues());
|
||||
auto x = TEST(Suite, notATest);
|
||||
TYPED_TEST_SUITE(Fixture, MyTypes);
|
||||
"""
|
||||
assert reports(code) == []
|
||||
assert fixed(code) == dedent(code)
|
||||
|
||||
|
||||
def test_indented_and_wrapped_definitions() -> None:
|
||||
code = """
|
||||
namespace ripple {
|
||||
TEST(Suite, indentedName)
|
||||
}
|
||||
TEST_F(
|
||||
SomeVeryLongFixtureName,
|
||||
wrappedName)
|
||||
"""
|
||||
assert fixed(code) == dedent("""
|
||||
namespace ripple {
|
||||
TEST(Suite, indented_name)
|
||||
}
|
||||
TEST_F(
|
||||
SomeVeryLongFixtureName,
|
||||
wrapped_name)
|
||||
""")
|
||||
|
||||
|
||||
# --- rewriting --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_only_the_two_names_are_rewritten() -> None:
|
||||
code = """
|
||||
TEST(mulDiv, mulDiv)
|
||||
{
|
||||
auto const mulDiv = 1; // mulDiv stays
|
||||
}
|
||||
"""
|
||||
assert fixed(code) == dedent("""
|
||||
TEST(MulDiv, mul_div)
|
||||
{
|
||||
auto const mulDiv = 1; // mulDiv stays
|
||||
}
|
||||
""")
|
||||
|
||||
|
||||
def test_suite_name_camel_cased() -> None:
|
||||
code = """
|
||||
TEST(json_value, limits)
|
||||
TEST(scope, ScopeExit)
|
||||
"""
|
||||
assert reports(code) == [
|
||||
"1: renamed suite 'json_value' to 'JsonValue'",
|
||||
"2: renamed suite 'scope' to 'Scope'",
|
||||
"2: renamed test case 'ScopeExit' to 'scope_exit'",
|
||||
]
|
||||
assert fixed(code) == dedent("""
|
||||
TEST(JsonValue, limits)
|
||||
TEST(Scope, scope_exit)
|
||||
""")
|
||||
|
||||
|
||||
def test_fixture_reported_but_not_renamed() -> None:
|
||||
# The first argument names a class, so only a human (or clang-tidy) can
|
||||
# rename it; the test-case name is still fixed.
|
||||
code = """
|
||||
TEST_F(my_fixture, someTest)
|
||||
"""
|
||||
assert reports(code) == [
|
||||
"1: fixture 'my_fixture' is not CamelCase: rename the class to "
|
||||
"'MyFixture' by hand",
|
||||
"1: renamed test case 'someTest' to 'some_test'",
|
||||
]
|
||||
assert fixed(code) == dedent("""
|
||||
TEST_F(my_fixture, some_test)
|
||||
""")
|
||||
|
||||
|
||||
def test_reports_carry_line_numbers() -> None:
|
||||
code = """
|
||||
#include <foo.h>
|
||||
|
||||
TEST(Suite, firstName)
|
||||
|
||||
TEST(Suite, secondName)
|
||||
"""
|
||||
assert reports(code) == [
|
||||
"3: renamed test case 'firstName' to 'first_name'",
|
||||
"5: renamed test case 'secondName' to 'second_name'",
|
||||
]
|
||||
|
||||
|
||||
# --- collisions -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_collision_reported_and_not_applied() -> None:
|
||||
# Both would define `Suite_mul_div_Test`.
|
||||
code = """
|
||||
TEST(Suite, mulDiv)
|
||||
TEST(Suite, mul_div)
|
||||
"""
|
||||
assert reports(code) == [
|
||||
"1: cannot rename 'Suite, mulDiv' to 'Suite, mul_div': another test "
|
||||
"case already generates that name"
|
||||
]
|
||||
assert fixed(code) == dedent(code)
|
||||
|
||||
|
||||
def test_collision_between_converging_suites() -> None:
|
||||
# Both suites camel-case to `SuiteA`, so both would define
|
||||
# `SuiteA_one_test_Test`.
|
||||
code = """
|
||||
TEST(SuiteA, oneTest)
|
||||
TEST(Suite_a, one_test)
|
||||
"""
|
||||
assert [r.split(":")[1].strip() for r in reports(code)] == [
|
||||
"cannot rename 'SuiteA, oneTest' to 'SuiteA, one_test'",
|
||||
"cannot rename 'Suite_a, one_test' to 'SuiteA, one_test'",
|
||||
]
|
||||
assert fixed(code) == dedent(code)
|
||||
|
||||
|
||||
def test_same_name_in_different_suites_is_not_a_collision() -> None:
|
||||
code = """
|
||||
TEST(SuiteOne, mulDiv)
|
||||
TEST(SuiteTwo, mulDiv)
|
||||
"""
|
||||
assert fixed(code) == dedent("""
|
||||
TEST(SuiteOne, mul_div)
|
||||
TEST(SuiteTwo, mul_div)
|
||||
""")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
tests = sorted(
|
||||
(name, fn)
|
||||
for name, fn in globals().items()
|
||||
if name.startswith("test_") and callable(fn)
|
||||
)
|
||||
failed = 0
|
||||
for name, fn in tests:
|
||||
try:
|
||||
fn()
|
||||
print(f"PASS {name}")
|
||||
except AssertionError as exc:
|
||||
failed += 1
|
||||
print(f"FAIL {name}: {exc!r}")
|
||||
print(f"\n{len(tests) - failed}/{len(tests)} passed")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,28 +0,0 @@
|
||||
include_guard()
|
||||
|
||||
set(GIT_BUILD_BRANCH "")
|
||||
set(GIT_COMMIT_HASH "")
|
||||
|
||||
find_package(Git)
|
||||
if(NOT Git_FOUND)
|
||||
message(WARNING "Git not found. Git branch and commit hash will be empty.")
|
||||
return()
|
||||
endif()
|
||||
|
||||
set(GIT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.git)
|
||||
|
||||
execute_process(
|
||||
COMMAND
|
||||
${GIT_EXECUTABLE} --git-dir=${GIT_DIRECTORY} rev-parse --abbrev-ref HEAD
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
OUTPUT_VARIABLE GIT_BUILD_BRANCH
|
||||
)
|
||||
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} --git-dir=${GIT_DIRECTORY} rev-parse HEAD
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
OUTPUT_VARIABLE GIT_COMMIT_HASH
|
||||
)
|
||||
|
||||
message(STATUS "Git branch: ${GIT_BUILD_BRANCH}")
|
||||
message(STATUS "Git commit hash: ${GIT_COMMIT_HASH}")
|
||||
+14
-2
@@ -81,7 +81,7 @@ include(target_link_modules)
|
||||
add_module(xrpl beast)
|
||||
target_link_libraries(xrpl.libxrpl.beast PUBLIC xrpl.imports.main)
|
||||
|
||||
include(GitInfo)
|
||||
include(XrplVersion)
|
||||
add_module(xrpl git)
|
||||
target_compile_definitions(
|
||||
xrpl.libxrpl.git
|
||||
@@ -111,6 +111,11 @@ target_link_libraries(
|
||||
xrpl.libxrpl.protocol
|
||||
PUBLIC xrpl.libxrpl.crypto xrpl.libxrpl.git xrpl.libxrpl.json
|
||||
)
|
||||
# Only on BuildInfo.cpp, so a new version does not rebuild the whole module.
|
||||
set_source_files_properties(
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src/libxrpl/protocol/BuildInfo.cpp
|
||||
PROPERTIES COMPILE_DEFINITIONS XRPLD_VERSION="${XRPLD_VERSION}"
|
||||
)
|
||||
|
||||
# Level 05
|
||||
add_module(xrpl protocol_autogen)
|
||||
@@ -296,7 +301,14 @@ if(xrpld)
|
||||
target_sources(xrpld PRIVATE ${sources})
|
||||
endif()
|
||||
|
||||
target_link_libraries(xrpld Xrpl::boost Xrpl::opts Xrpl::libs xrpl.libxrpl)
|
||||
target_link_libraries(
|
||||
xrpld
|
||||
Xrpl::boost
|
||||
Xrpl::opts
|
||||
Xrpl::libs
|
||||
xrpl.libxrpl
|
||||
rpcspec::rpcspec
|
||||
)
|
||||
exclude_if_included(xrpld)
|
||||
# define a macro for tests that might need to
|
||||
# be excluded or run differently in CI environment
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ setup_target_for_coverage_gcovr(
|
||||
"include/xrpl/beast/test"
|
||||
"include/xrpl/beast/unit_test"
|
||||
"${CMAKE_BINARY_DIR}/pb-xrpl.libpb"
|
||||
DEPENDENCIES xrpld xrpl_tests
|
||||
DEPENDENCIES xrpld xrpl_tests xrpld_tests
|
||||
)
|
||||
|
||||
add_code_coverage_to_target(opts INTERFACE)
|
||||
@@ -44,12 +44,18 @@ else()
|
||||
set(pkg_type rpm)
|
||||
endif()
|
||||
|
||||
# Unquoted below, so an empty value adds no argument at all.
|
||||
set(pkg_variant_option "")
|
||||
if(assert)
|
||||
set(pkg_variant_option --variant=assert)
|
||||
endif()
|
||||
|
||||
add_custom_target(
|
||||
package
|
||||
COMMAND
|
||||
${CMAKE_SOURCE_DIR}/package/build_pkg.py --package-type=${pkg_type}
|
||||
--build-dir=${CMAKE_BINARY_DIR} --pkg-release=${pkg_release}
|
||||
--channel=UNRELEASED
|
||||
${pkg_variant_option} --channel=UNRELEASED
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
|
||||
DEPENDS xrpld validator-keys
|
||||
COMMENT "Building Linux ${pkg_type} package"
|
||||
|
||||
+73
-13
@@ -1,15 +1,75 @@
|
||||
#[===================================================================[
|
||||
read version from source
|
||||
#]===================================================================]
|
||||
find_package(Git)
|
||||
|
||||
file(STRINGS src/libxrpl/protocol/BuildInfo.cpp BUILD_INFO)
|
||||
foreach(line_ ${BUILD_INFO})
|
||||
if(line_ MATCHES "versionString[ ]*=[ ]*\"(.+)\"")
|
||||
set(xrpld_version ${CMAKE_MATCH_1})
|
||||
endif()
|
||||
endforeach()
|
||||
if(xrpld_version)
|
||||
message(STATUS "xrpld version: ${xrpld_version}")
|
||||
else()
|
||||
message(FATAL_ERROR "unable to determine xrpld version")
|
||||
set(GIT_BUILD_BRANCH "")
|
||||
set(GIT_COMMIT_HASH "")
|
||||
|
||||
if(DEFINED ENV{GITHUB_BRANCH_NAME})
|
||||
set(GIT_BUILD_BRANCH $ENV{GITHUB_BRANCH_NAME})
|
||||
set(GIT_COMMIT_HASH $ENV{GITHUB_HEAD_SHA})
|
||||
elseif(Git_FOUND AND EXISTS "${CMAKE_CURRENT_LIST_DIR}/../.git")
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/..
|
||||
OUTPUT_VARIABLE GIT_BUILD_BRANCH
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
COMMAND_ERROR_IS_FATAL ANY
|
||||
)
|
||||
|
||||
execute_process(
|
||||
COMMAND ${GIT_EXECUTABLE} rev-parse HEAD
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/..
|
||||
OUTPUT_VARIABLE GIT_COMMIT_HASH
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
COMMAND_ERROR_IS_FATAL ANY
|
||||
)
|
||||
endif()
|
||||
|
||||
message(STATUS "Git branch: ${GIT_BUILD_BRANCH}")
|
||||
message(STATUS "Git commit hash: ${GIT_COMMIT_HASH}")
|
||||
|
||||
if(
|
||||
DEFINED ENV{FORCE_XRPLD_VERSION}
|
||||
AND NOT "$ENV{FORCE_XRPLD_VERSION}" STREQUAL ""
|
||||
)
|
||||
message(
|
||||
STATUS
|
||||
"Using explicitly provided '$ENV{FORCE_XRPLD_VERSION}' as xrpld version"
|
||||
)
|
||||
|
||||
set(XRPLD_VERSION "$ENV{FORCE_XRPLD_VERSION}")
|
||||
|
||||
# The rules beast::SemanticVersion::parse applies, so that an invalid version
|
||||
# fails here rather than when xrpld starts. It reads each number as an int.
|
||||
set(SEMVER_NUMBER "(0|[1-9][0-9]*)")
|
||||
set(SEMVER_PRE_RELEASE "[A-Za-z1-9-][A-Za-z0-9-]*")
|
||||
set(SEMVER_METADATA "[A-Za-z0-9-]+")
|
||||
set(SEMVER_NUMBER_MAX 2147483647)
|
||||
if(
|
||||
NOT XRPLD_VERSION
|
||||
MATCHES
|
||||
"^${SEMVER_NUMBER}\\.${SEMVER_NUMBER}\\.${SEMVER_NUMBER}(-${SEMVER_PRE_RELEASE}(\\.${SEMVER_PRE_RELEASE})*)?(\\+${SEMVER_METADATA}(\\.${SEMVER_METADATA})*)?$"
|
||||
OR CMAKE_MATCH_1 GREATER SEMVER_NUMBER_MAX
|
||||
OR CMAKE_MATCH_2 GREATER SEMVER_NUMBER_MAX
|
||||
OR CMAKE_MATCH_3 GREATER SEMVER_NUMBER_MAX
|
||||
)
|
||||
message(
|
||||
FATAL_ERROR
|
||||
"FORCE_XRPLD_VERSION '${XRPLD_VERSION}' is not a semantic version xrpld accepts, see https://semver.org"
|
||||
)
|
||||
endif()
|
||||
else()
|
||||
message(STATUS "Using '0.0.0-dev+<git short rev>' as xrpld version")
|
||||
|
||||
if(GIT_COMMIT_HASH STREQUAL "")
|
||||
message(
|
||||
FATAL_ERROR
|
||||
"Unable to determine xrpld version without git, set FORCE_XRPLD_VERSION"
|
||||
)
|
||||
endif()
|
||||
|
||||
string(SUBSTRING ${GIT_COMMIT_HASH} 0 7 GIT_COMMIT_HASH_SHORT)
|
||||
|
||||
set(XRPLD_VERSION "0.0.0-dev+${GIT_COMMIT_HASH_SHORT}")
|
||||
endif()
|
||||
|
||||
message(STATUS "Build version: ${XRPLD_VERSION}")
|
||||
@@ -59,7 +59,7 @@ def create_transaction_parser():
|
||||
# the members that differ from these.
|
||||
SETTING_DEFAULTS = {
|
||||
"delegable": "Delegation::NotDelegable",
|
||||
"amendment": "uint256{}",
|
||||
"amendment": "UInt256{}",
|
||||
"privileges": "Privilege::NoPriv",
|
||||
}
|
||||
|
||||
|
||||
@@ -209,7 +209,7 @@ ${field['typeData']['setter_type']} ${field['paramName']}${',' if i < len(requir
|
||||
* @return The constructed ledger entry wrapper.
|
||||
*/
|
||||
${name}
|
||||
build(uint256 const& index)
|
||||
build(UInt256 const& index)
|
||||
{
|
||||
return ${name}{std::make_shared<SLE>(std::move(object_), index)};
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ namespace xrpl::ledger_entries {
|
||||
// builder's STObject and the wrapper's SLE.
|
||||
TEST(${name}Tests, BuilderSettersRoundTrip)
|
||||
{
|
||||
uint256 const index{1u};
|
||||
constexpr UInt256 index{1};
|
||||
|
||||
% for field in fields:
|
||||
auto const ${field["paramName"]}Value = ${canonical_expr(field)};
|
||||
@@ -85,7 +85,7 @@ TEST(${name}Tests, BuilderSettersRoundTrip)
|
||||
// from that SLE, build a new wrapper, and verify all fields (and validate()).
|
||||
TEST(${name}Tests, BuilderFromSleRoundTrip)
|
||||
{
|
||||
uint256 const index{2u};
|
||||
constexpr UInt256 index{2};
|
||||
|
||||
% for field in fields:
|
||||
auto const ${field["paramName"]}Value = ${canonical_expr(field)};
|
||||
@@ -146,7 +146,7 @@ TEST(${name}Tests, BuilderFromSleRoundTrip)
|
||||
// 3) Verify wrapper throws when constructed from wrong ledger entry type.
|
||||
TEST(${name}Tests, WrapperThrowsOnWrongEntryType)
|
||||
{
|
||||
uint256 const index{3u};
|
||||
constexpr UInt256 index{3};
|
||||
|
||||
// Build a valid ledger entry of a different type
|
||||
// Ticket requires: Account, OwnerNode, TicketSequence, PreviousTxnID, PreviousTxnLgrSeq
|
||||
@@ -177,7 +177,7 @@ TEST(${name}Tests, WrapperThrowsOnWrongEntryType)
|
||||
// 4) Verify builder throws when constructed from wrong ledger entry type.
|
||||
TEST(${name}Tests, BuilderThrowsOnWrongEntryType)
|
||||
{
|
||||
uint256 const index{4u};
|
||||
constexpr UInt256 index{4};
|
||||
|
||||
// Build a valid ledger entry of a different type
|
||||
% if wrong_le_include == "Ticket":
|
||||
@@ -207,7 +207,7 @@ TEST(${name}Tests, BuilderThrowsOnWrongEntryType)
|
||||
// 5) Build with only required fields and verify optional fields return nullopt.
|
||||
TEST(${name}Tests, OptionalFieldsReturnNullopt)
|
||||
{
|
||||
uint256 const index{3u};
|
||||
constexpr UInt256 index{3};
|
||||
|
||||
% for field in required_fields:
|
||||
auto const ${field["paramName"]}Value = ${canonical_expr(field)};
|
||||
|
||||
+7
-2
@@ -3,6 +3,7 @@
|
||||
"requires": [
|
||||
"zlib/1.3.2#1cb806da49011867778ffb6ac7190fcb%1782392402.122708",
|
||||
"xxhash/0.8.3#681d36a0a6111fc56e5e45ea182c19cc%1782392402.420688",
|
||||
"xrpl-rpc-spec/0.1.19#870b2d3abcfbbf13b61c2d3c69495060%1790348286.187549",
|
||||
"sqlite3/3.53.0#324ada52333108388a9a6108bfa96734%1782392403.185447",
|
||||
"soci/4.0.3#e726491a03468795453f7c83fc924a96%1782392402.679521",
|
||||
"snappy/1.1.10#968fef506ff261592ec30c574d4a7809%1782307151.633168",
|
||||
@@ -19,8 +20,8 @@
|
||||
"libarchive/3.8.7#c446109bd1f1d8ba7936c94189bc50e6%1782392403.066892",
|
||||
"jemalloc/5.3.1#1fc58d55316041f10fbc1e8a2eae632a%1776700028.228",
|
||||
"gtest/1.17.0#5224b3b3ff3b4ce1133cbdd27d53ee7d%1782392402.791979",
|
||||
"grpc/1.81.1#f729f6d75992d20f9c72828e9142d62f%1783945160.094135",
|
||||
"fast_float/8.2.10#f6f28d6bb22112078e7dbda611caf681%1782494504.298",
|
||||
"grpc/1.81.1#b87796a4269034856cbc1a2522db16eb%1788275071.530512",
|
||||
"fast_float/8.2.10#f6f28d6bb22112078e7dbda611caf681%1785888854.601666",
|
||||
"ed25519/2015.03#ae761bdc52730a843f0809bdf6c1b1f6%1782307148.15562",
|
||||
"date/3.0.4#862e11e80030356b53c2c38599ceb32b%1782392402.538492",
|
||||
"corrosion/0.6.1#bfa292df0a957bc70a450ff316cd9435%1786119416.131296",
|
||||
@@ -45,6 +46,10 @@
|
||||
],
|
||||
"python_requires": [],
|
||||
"overrides": {
|
||||
"boost/1.91.0#ea540ca2133d831b560036aa24dece3c": [
|
||||
null,
|
||||
"boost/1.91.0#ea540ca2133d831b560036aa24dece3c"
|
||||
],
|
||||
"protobuf/[>=5.27.0 <7]": [
|
||||
"protobuf/6.33.5"
|
||||
],
|
||||
|
||||
@@ -2,8 +2,11 @@
|
||||
|
||||
To achieve reproducible dependencies, we use a [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html).
|
||||
|
||||
The `conan.lock` file in the repository contains a "snapshot" of the current dependencies. It is implicitly used when running `conan` commands, so you don't need to specify it.
|
||||
The `conan.lock` file in the repository contains a "snapshot" of the current
|
||||
dependencies. It is implicitly used when running `conan` commands, so you don't
|
||||
need to specify it.
|
||||
|
||||
You have to update this file every time you add a new dependency or change a revision or version of an existing dependency.
|
||||
You have to update this file every time you add a new dependency or change a
|
||||
revision or version of an existing dependency.
|
||||
|
||||
To update a lockfile, run from the repository root: `./conan/lockfile/regenerate.sh`
|
||||
+19
-7
@@ -1,9 +1,13 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
|
||||
from conan.tools.env import Environment
|
||||
|
||||
from conan import ConanFile
|
||||
|
||||
DEV_VERSION = "0.0.0-dev"
|
||||
|
||||
|
||||
class Xrpl(ConanFile):
|
||||
name = "xrpl"
|
||||
@@ -36,6 +40,7 @@ class Xrpl(ConanFile):
|
||||
"nudb/2.0.9",
|
||||
"openssl/3.6.3",
|
||||
"soci/4.0.3",
|
||||
"xrpl-rpc-spec/0.1.19",
|
||||
"zlib/1.3.2",
|
||||
]
|
||||
|
||||
@@ -114,17 +119,12 @@ class Xrpl(ConanFile):
|
||||
"soci/*:shared": False,
|
||||
"soci/*:with_sqlite3": True,
|
||||
"soci/*:with_boost": True,
|
||||
"xrpl-rpc-spec/*:server": "xrpld",
|
||||
"xxhash/*:shared": False,
|
||||
}
|
||||
|
||||
def set_version(self):
|
||||
if self.version is None:
|
||||
path = f"{self.recipe_folder}/src/libxrpl/protocol/BuildInfo.cpp"
|
||||
regex = r"versionString\s?=\s?\"(.*)\""
|
||||
with open(path, encoding="utf-8") as file:
|
||||
matches = (re.search(regex, line) for line in file)
|
||||
match = next(m for m in matches if m)
|
||||
self.version = match.group(1)
|
||||
self.version = self.version or DEV_VERSION
|
||||
|
||||
def configure(self):
|
||||
if self.settings.compiler == "apple-clang":
|
||||
@@ -149,6 +149,7 @@ class Xrpl(ConanFile):
|
||||
self.requires("xxhash/0.8.3", transitive_headers=True)
|
||||
|
||||
exports_sources = (
|
||||
"bin/default-loader-path.sh",
|
||||
"CMakeLists.txt",
|
||||
"cfg/*",
|
||||
"cmake/*",
|
||||
@@ -166,6 +167,17 @@ class Xrpl(ConanFile):
|
||||
generators = "CMakeDeps"
|
||||
|
||||
def generate(self):
|
||||
# The sources in the Conan cache have no git history, so the version
|
||||
# comes from the reference, unless it is not one, like 'develop'.
|
||||
if not os.path.exists(os.path.join(self.source_folder, ".git")):
|
||||
version = str(self.version)
|
||||
env = Environment()
|
||||
env.define(
|
||||
"FORCE_XRPLD_VERSION",
|
||||
version if re.match(r"\d+\.\d+\.\d+", version) else DEV_VERSION,
|
||||
)
|
||||
env.vars(self).save_script("xrpld_version")
|
||||
|
||||
tc = CMakeToolchain(self)
|
||||
tc.variables["tests"] = self.options.tests
|
||||
tc.variables["benchmark"] = self.options.benchmark
|
||||
|
||||
Generated
+10
-10
@@ -57,9 +57,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxx"
|
||||
version = "1.0.199"
|
||||
version = "1.0.202"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824894a4a85dca76d4c95c2b9098c036f5a29f627b30c12780774f6654e60974"
|
||||
checksum = "13f6de320895f42e6e081abb5c7983bedcf0b6d0ff9323de0d33f620c8ac1199"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cxx-build",
|
||||
@@ -72,9 +72,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxx-build"
|
||||
version = "1.0.199"
|
||||
version = "1.0.202"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1ae0b651ea5b0000b19513aef5a03f194d7e3486f2d9258b658da8677fe9036"
|
||||
checksum = "4fde53ca86b9704a943fef0f1e1d836239a6aedca5de3c65fa9f97ec0bd46d39"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"codespan-reporting",
|
||||
@@ -87,9 +87,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-cmd"
|
||||
version = "1.0.199"
|
||||
version = "1.0.202"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb05f91d3fb8435d9bab6ac5ce6ac1868be774325fb7fb2a91be39393b21388e"
|
||||
checksum = "07bae89236c811fd4d08ed3441759ac4ac98d752cbfd3de341315ba16ad20ec3"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"codespan-reporting",
|
||||
@@ -101,15 +101,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-flags"
|
||||
version = "1.0.199"
|
||||
version = "1.0.202"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf293202e0e3e98495785745389e8d0755b217e66f19194a5c695c25e03282ef"
|
||||
checksum = "49045042e5fced01b80742aba5508de82aa4f13677ed3fa2b4cda709c40c5918"
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-macro"
|
||||
version = "1.0.199"
|
||||
version = "1.0.202"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca001d746947c7249ed9d332a10f7a59daedbafeb0ec68c5c18a7db7a93f6ccc"
|
||||
checksum = "b181252e2e3d3b5d183afbdc033a3958b445eb3e1a0ae65fc3d4f5259f5da6fd"
|
||||
dependencies = [
|
||||
"indexmap",
|
||||
"proc-macro2",
|
||||
|
||||
@@ -8,6 +8,9 @@ cxx = { version = "1.0.198", features = ["c++20"] }
|
||||
[workspace.package]
|
||||
edition = "2024"
|
||||
|
||||
[workspace.lints.rust]
|
||||
unexpected_cfgs = { level = "warn", check-cfg = [ 'cfg(coverage)', 'cfg(coverage_nightly)' ] }
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
overflow-checks = true
|
||||
|
||||
@@ -8,3 +8,6 @@ crate-type = ["staticlib"]
|
||||
|
||||
[dependencies]
|
||||
cxx.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -1,3 +1,5 @@
|
||||
#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
|
||||
|
||||
#[cxx::bridge(namespace = "rs::hello_world")]
|
||||
mod ffi {
|
||||
extern "Rust" {
|
||||
@@ -8,3 +10,14 @@ mod ffi {
|
||||
pub fn hello_world() -> String {
|
||||
"hello_world".to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg_attr(coverage_nightly, coverage(off))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn hello_world_returns_hello_world() {
|
||||
assert_eq!(hello_world(), "hello_world")
|
||||
}
|
||||
}
|
||||
@@ -2,21 +2,61 @@
|
||||
|
||||
## The Problem Statement
|
||||
|
||||
The moment-to-moment health of the XRP Ledger network depends on the health and connectivity of a small number of computers (nodes). The most important nodes are validators, specifically ones listed on the unique node list ([UNL](#Question-What-are-UNLs)). Ripple publishes a recommended UNL that most network nodes use to determine which peers in the network are trusted. Although most validators use the same list, they are not required to. The XRP Ledger network progresses to the next ledger when enough validators reach agreement (above the minimum quorum of 80%) about what transactions to include in the next ledger.
|
||||
The moment-to-moment health of the XRP Ledger network depends on the health and
|
||||
connectivity of a small number of computers (nodes). The most important nodes
|
||||
are validators, specifically ones listed on the unique node list
|
||||
([UNL](#Question-What-are-UNLs)). Ripple publishes a recommended UNL that most
|
||||
network nodes use to determine which peers in the network are trusted. Although
|
||||
most validators use the same list, they are not required to. The XRP Ledger
|
||||
network progresses to the next ledger when enough validators reach agreement
|
||||
(above the minimum quorum of 80%) about what transactions to include in the next
|
||||
ledger.
|
||||
|
||||
As an example, if there are 10 validators on the UNL, at least 8 validators have to agree with the latest ledger for it to become validated. But what if enough of those validators are offline to drop the network below the 80% quorum? The XRP Ledger network favors safety/correctness over advancing the ledger. Which means if enough validators are offline, the network will not be able to validate ledgers.
|
||||
As an example, if there are 10 validators on the UNL, at least 8 validators have
|
||||
to agree with the latest ledger for it to become validated. But what if enough
|
||||
of those validators are offline to drop the network below the 80% quorum? The
|
||||
XRP Ledger network favors safety/correctness over advancing the ledger. Which
|
||||
means if enough validators are offline, the network will not be able to validate
|
||||
ledgers.
|
||||
|
||||
Unfortunately validators can go offline at any time for many different reasons. Power outages, network connectivity issues, and hardware failures are just a few scenarios where a validator would appear "offline". Given that most of these events are temporary, it would make sense to temporarily remove that validator from the UNL. But the UNL is updated infrequently and not every node uses the same UNL. So instead of removing the unreliable validator from the Ripple recommended UNL, we can create a second negative UNL which is stored directly on the ledger (so the entire network has the same view). This will help the network see which validators are **currently** unreliable, and adjust their quorum calculation accordingly.
|
||||
Unfortunately validators can go offline at any time for many different reasons.
|
||||
Power outages, network connectivity issues, and hardware failures are just a few
|
||||
scenarios where a validator would appear "offline". Given that most of these
|
||||
events are temporary, it would make sense to temporarily remove that validator
|
||||
from the UNL. But the UNL is updated infrequently and not every node uses the
|
||||
same UNL. So instead of removing the unreliable validator from the Ripple
|
||||
recommended UNL, we can create a second negative UNL which is stored directly on
|
||||
the ledger (so the entire network has the same view). This will help the network
|
||||
see which validators are **currently** unreliable, and adjust their quorum
|
||||
calculation accordingly.
|
||||
|
||||
_Improving the liveness of the network is the main motivation for the negative UNL._
|
||||
|
||||
### Targeted Faults
|
||||
|
||||
In order to determine which validators are unreliable, we need clearly define what kind of faults to measure and analyze. We want to deal with the faults we frequently observe in the production network. Hence we will only monitor for validators that do not reliably respond to network messages or send out validations disagreeing with the locally generated validations. We will not target other byzantine faults.
|
||||
In order to determine which validators are unreliable, we need clearly define
|
||||
what kind of faults to measure and analyze. We want to deal with the faults we
|
||||
frequently observe in the production network. Hence we will only monitor for
|
||||
validators that do not reliably respond to network messages or send out
|
||||
validations disagreeing with the locally generated validations. We will not
|
||||
target other byzantine faults.
|
||||
|
||||
To track whether or not a validator is responding to the network, we could monitor them with a “heartbeat” protocol. Instead of creating a new heartbeat protocol, we can leverage some existing protocol messages to mimic the heartbeat. We picked validation messages because validators should send one and only one validation message per ledger. In addition, we only count the validation messages that agree with the local node's validations.
|
||||
To track whether or not a validator is responding to the network, we could
|
||||
monitor them with a “heartbeat” protocol. Instead of creating a new heartbeat
|
||||
protocol, we can leverage some existing protocol messages to mimic the
|
||||
heartbeat. We picked validation messages because validators should send one and
|
||||
only one validation message per ledger. In addition, we only count the
|
||||
validation messages that agree with the local node's validations.
|
||||
|
||||
With the negative UNL, the network could keep making forward progress safely even if the number of remaining validators gets to 60%. Say we have a network with 10 validators on the UNL and everything is operating correctly. The quorum required for this network would be 8 (80% of 10). When validators fail, the quorum required would be as low as 6 (60% of 10), which is the absolute **_minimum quorum_**. We need the absolute minimum quorum to be strictly greater than 50% of the original UNL so that there cannot be two partitions of well-behaved nodes headed in different directions. We arbitrarily choose 60% as the minimum quorum to give a margin of safety.
|
||||
With the negative UNL, the network could keep making forward progress safely
|
||||
even if the number of remaining validators gets to 60%. Say we have a network
|
||||
with 10 validators on the UNL and everything is operating correctly. The quorum
|
||||
required for this network would be 8 (80% of 10). When validators fail, the
|
||||
quorum required would be as low as 6 (60% of 10), which is the absolute
|
||||
**_minimum quorum_**. We need the absolute minimum quorum to be strictly greater
|
||||
than 50% of the original UNL so that there cannot be two partitions of
|
||||
well-behaved nodes headed in different directions. We arbitrarily choose 60% as
|
||||
the minimum quorum to give a margin of safety.
|
||||
|
||||
Consider these events in the absence of negative UNL:
|
||||
|
||||
@@ -25,7 +65,10 @@ Consider these events in the absence of negative UNL:
|
||||
1. 5:00pm - validator3 fails, votes vs. quorum: 7 < 8, we don’t have quorum
|
||||
- **network cannot validate new ledgers with 3 failed validators**
|
||||
|
||||
We're below 80% agreement, so new ledgers cannot be validated. This is how the XRP Ledger operates today, but if the negative UNL was enabled, the events would happen as follows. (Please note that the events below are from a simplified version of our protocol.)
|
||||
We're below 80% agreement, so new ledgers cannot be validated. This is how the
|
||||
XRP Ledger operates today, but if the negative UNL was enabled, the events would
|
||||
happen as follows. (Please note that the events below are from a simplified
|
||||
version of our protocol.)
|
||||
|
||||
1. 1:00pm - validator1 fails, votes vs. quorum: 9 >= 8, we have quorum
|
||||
1. 1:40pm - network adds validator1 to negative UNL, quorum changes to ceil(9 \* 0.8), or 8
|
||||
@@ -45,7 +88,8 @@ This proposal will:
|
||||
1. add a new pseudo-transaction type
|
||||
1. add the negative UNL to the ledger data structure.
|
||||
|
||||
Any tools or systems that rely on the format of this data will have to be updated.
|
||||
Any tools or systems that rely on the format of this data will have to be
|
||||
updated.
|
||||
|
||||
### Amendment
|
||||
|
||||
@@ -61,70 +105,122 @@ This section discusses the following topics about the Negative UNL design:
|
||||
- [Negative UNL maintenance](#Negative-UNL-Maintenance)
|
||||
- [Quorum size calculation](#Quorum-Size-Calculation)
|
||||
- [Filter validation messages](#Filter-Validation-Messages)
|
||||
- [High level sequence diagram of code changes](#High-Level-Sequence-Diagram-of-Code-Changes)
|
||||
- [High level sequence diagram of code
|
||||
changes](#High-Level-Sequence-Diagram-of-Code-Changes)
|
||||
|
||||
### Negative UNL Protocol Overview
|
||||
|
||||
Every ledger stores a list of zero or more unreliable validators. Updates to the list must be approved by the validators using the consensus mechanism that validators use to agree on the set of transactions. The list is used only when checking if a ledger is fully validated. If a validator V is in the list, nodes with V in their UNL adjust the quorum and V’s validation message is not counted when verifying if a ledger is fully validated. V’s flow of messages and network interactions, however, will remain the same.
|
||||
Every ledger stores a list of zero or more unreliable validators. Updates to the
|
||||
list must be approved by the validators using the consensus mechanism that
|
||||
validators use to agree on the set of transactions. The list is used only when
|
||||
checking if a ledger is fully validated. If a validator V is in the list, nodes
|
||||
with V in their UNL adjust the quorum and V’s validation message is not counted
|
||||
when verifying if a ledger is fully validated. V’s flow of messages and network
|
||||
interactions, however, will remain the same.
|
||||
|
||||
We define the **\*effective UNL** = original UNL - negative UNL\*, and the **_effective quorum_** as the quorum of the _effective UNL_. And we set _effective quorum = Ceiling(80% _ effective UNL)\*.
|
||||
We define the **\*effective UNL** = original UNL - negative UNL\*, and the
|
||||
**_effective quorum_** as the quorum of the _effective UNL_. And we set
|
||||
_effective quorum = Ceiling(80% _ effective UNL)\*.
|
||||
|
||||
### Validator Reliability Measurement
|
||||
|
||||
A node only measures the reliability of validators on its own UNL, and only proposes based on local observations. There are many metrics that a node can measure about its validators, but we have chosen ledger validation messages. This is because every validator shall send one and only one signed validation message per ledger. This keeps the measurement simple and removes timing/clock-sync issues. A node will measure the percentage of agreeing validation messages (_PAV_) received from each validator on the node's UNL. Note that the node will only count the validation messages that agree with its own validations.
|
||||
A node only measures the reliability of validators on its own UNL, and only
|
||||
proposes based on local observations. There are many metrics that a node can
|
||||
measure about its validators, but we have chosen ledger validation messages.
|
||||
This is because every validator shall send one and only one signed validation
|
||||
message per ledger. This keeps the measurement simple and removes
|
||||
timing/clock-sync issues. A node will measure the percentage of agreeing
|
||||
validation messages (_PAV_) received from each validator on the node's UNL. Note
|
||||
that the node will only count the validation messages that agree with its own
|
||||
validations.
|
||||
|
||||
We define the **PAV** as the Percentage of Agreed Validation messages received for the last N ledgers, where N = 256 by default.
|
||||
We define the **PAV** as the Percentage of Agreed Validation
|
||||
messages received for the last N ledgers, where N = 256 by default.
|
||||
|
||||
When the PAV drops below the **_low-water mark_**, the validator is considered unreliable, and is a candidate to be disabled by being added to the negative UNL. A validator must have a PAV higher than the **_high-water mark_** to be re-enabled. The validator is re-enabled by removing it from the negative UNL. In the implementation, we plan to set the low-water mark as 50% and the high-water mark as 80%.
|
||||
When the PAV drops below the **_low-water mark_**, the validator is considered
|
||||
unreliable, and is a candidate to be disabled by being added to the negative
|
||||
UNL. A validator must have a PAV higher than the **_high-water mark_** to be
|
||||
re-enabled. The validator is re-enabled by removing it from the negative UNL. In
|
||||
the implementation, we plan to set the low-water mark as 50% and the high-water
|
||||
mark as 80%.
|
||||
|
||||
### Format Changes
|
||||
|
||||
The negative UNL component in a ledger contains three fields.
|
||||
|
||||
- **_NegativeUNL_**: The current negative UNL, a list of unreliable validators.
|
||||
- **_ToDisable_**: The validator to be added to the negative UNL on the next flag ledger.
|
||||
- **_ToReEnable_**: The validator to be removed from the negative UNL on the next flag ledger.
|
||||
- **_ToDisable_**: The validator to be added to the negative UNL on the next
|
||||
flag ledger.
|
||||
- **_ToReEnable_**: The validator to be removed from the negative UNL on the
|
||||
next flag ledger.
|
||||
|
||||
All three fields are optional. When the _ToReEnable_ field exists, the _NegativeUNL_ field cannot be empty.
|
||||
All three fields are optional. When the _ToReEnable_ field exists, the
|
||||
_NegativeUNL_ field cannot be empty.
|
||||
|
||||
A new pseudo-transaction, **_UNLModify_**, is added. It has three fields
|
||||
|
||||
- **_Disabling_**: A flag indicating whether the modification is to disable or to re-enable a validator.
|
||||
- **_Disabling_**: A flag indicating whether the modification is to disable or
|
||||
to re-enable a validator.
|
||||
- **_Seq_**: The ledger sequence number.
|
||||
- **_Validator_**: The validator to be disabled or re-enabled.
|
||||
|
||||
There would be at most one _disable_ `UNLModify` and one _re-enable_ `UNLModify` transaction per flag ledger. The full machinery is described further on.
|
||||
There would be at most one _disable_ `UNLModify` and one _re-enable_ `UNLModify`
|
||||
transaction per flag ledger. The full machinery is described further on.
|
||||
|
||||
### Negative UNL Maintenance
|
||||
|
||||
The negative UNL can only be modified on the flag ledgers. If a validator's reliability status changes, it takes two flag ledgers to modify the negative UNL. Let's see an example of the algorithm:
|
||||
The negative UNL can only be modified on the flag ledgers. If a validator's
|
||||
reliability status changes, it takes two flag ledgers to modify the negative
|
||||
UNL. Let's see an example of the algorithm:
|
||||
|
||||
- Ledger seq = 100: A validator V goes offline.
|
||||
- Ledger seq = 256: This is a flag ledger, and V's reliability measurement _PAV_ is lower than the low-water mark. Other validators add `UNLModify` pseudo-transactions `{true, 256, V}` to the transaction set which goes through the consensus. Then the pseudo-transaction is applied to the negative UNL ledger component by setting `ToDisable = V`.
|
||||
- Ledger seq = 257 ~ 511: The negative UNL ledger component is copied from the parent ledger.
|
||||
- Ledger seq=512: This is a flag ledger, and the negative UNL is updated `NegativeUNL = NegativeUNL + ToDisable`.
|
||||
- Ledger seq = 256: This is a flag ledger, and V's reliability measurement _PAV_
|
||||
is lower than the low-water mark. Other validators add `UNLModify`
|
||||
pseudo-transactions `{true, 256, V}` to the transaction set which goes through
|
||||
the consensus. Then the pseudo-transaction is applied to the negative UNL
|
||||
ledger component by setting `ToDisable = V`.
|
||||
- Ledger seq = 257 ~ 511: The negative UNL ledger component is copied from the
|
||||
parent ledger.
|
||||
- Ledger seq=512: This is a flag ledger, and the negative UNL is updated
|
||||
`NegativeUNL = NegativeUNL + ToDisable`.
|
||||
|
||||
The negative UNL may have up to `MaxNegativeListed = floor(original UNL * 25%)` validators. The 25% is because of 75% \* 80% = 60%, where 75% = 100% - 25%, 80% is the quorum of the effective UNL, and 60% is the absolute minimum quorum of the original UNL. Adding more than 25% validators to the negative UNL does not improve the liveness of the network, because adding more validators to the negative UNL cannot lower the effective quorum.
|
||||
The negative UNL may have up to `MaxNegativeListed = floor(original UNL * 25%)`
|
||||
validators. The 25% is because of 75% \* 80% = 60%, where 75% = 100% - 25%, 80%
|
||||
is the quorum of the effective UNL, and 60% is the absolute minimum quorum of
|
||||
the original UNL. Adding more than 25% validators to the negative UNL does not
|
||||
improve the liveness of the network, because adding more validators to the
|
||||
negative UNL cannot lower the effective quorum.
|
||||
|
||||
The following is the detailed algorithm:
|
||||
|
||||
- **If** the ledger seq = x is a flag ledger
|
||||
1. Compute `NegativeUNL = NegativeUNL + ToDisable - ToReEnable` if they exist in the parent ledger
|
||||
1. Compute `NegativeUNL = NegativeUNL + ToDisable - ToReEnable` if they
|
||||
exist in the parent ledger
|
||||
|
||||
1. Try to find a candidate to disable if `sizeof NegativeUNL < MaxNegativeListed`
|
||||
|
||||
1. Find a validator V that has a _PAV_ lower than the low-water mark, but is not in `NegativeUNL`.
|
||||
1. Find a validator V that has a _PAV_ lower than the low-water
|
||||
mark, but is not in `NegativeUNL`.
|
||||
|
||||
1. If two or more are found, their public keys are XORed with the hash of the parent ledger and the one with the lowest XOR result is chosen.
|
||||
1. If V is found, create a `UNLModify` pseudo-transaction `TxDisableValidator = {true, x, V}`
|
||||
1. If two or more are found, their public keys are XORed with the hash
|
||||
of the parent ledger and the one with the lowest XOR result is chosen.
|
||||
1. If V is found, create a `UNLModify` pseudo-transaction
|
||||
`TxDisableValidator = {true, x, V}`
|
||||
1. Try to find a candidate to re-enable if `sizeof NegativeUNL > 0`:
|
||||
1. Find a validator U that is in `NegativeUNL` and has a _PAV_ higher than the high-water mark.
|
||||
1. If U is not found, try to find one in `NegativeUNL` but not in the local _UNL_.
|
||||
1. If two or more are found, their public keys are XORed with the hash of the parent ledger and the one with the lowest XOR result is chosen.
|
||||
1. If U is found, create a `UNLModify` pseudo-transaction `TxReEnableValidator = {false, x, U}`
|
||||
1. Find a validator U that is in `NegativeUNL` and has a _PAV_ higher
|
||||
than the high-water mark.
|
||||
1. If U is not found, try to find one in `NegativeUNL` but not in the
|
||||
local _UNL_.
|
||||
1. If two or more are found, their public keys are XORed with the hash
|
||||
of the parent ledger and the one with the lowest XOR result is chosen.
|
||||
1. If U is found, create a `UNLModify` pseudo-transaction
|
||||
`TxReEnableValidator = {false, x, U}`
|
||||
|
||||
1. If any `UNLModify` pseudo-transactions are created, add them to the transaction set. The transaction set goes through the consensus algorithm.
|
||||
1. If have enough support, the `UNLModify` pseudo-transactions remain in the transaction set agreed by the validators. Then the pseudo-transactions are applied to the ledger:
|
||||
1. If any `UNLModify` pseudo-transactions are created, add them to the
|
||||
transaction set. The transaction set goes through the consensus algorithm.
|
||||
1. If have enough support, the `UNLModify` pseudo-transactions remain in the
|
||||
transaction set agreed by the validators. Then the pseudo-transactions are
|
||||
applied to the ledger:
|
||||
|
||||
1. If have `TxDisableValidator`, set `ToDisable=TxDisableValidator.V`.
|
||||
Else clear `ToDisable`.
|
||||
@@ -135,33 +231,61 @@ The following is the detailed algorithm:
|
||||
- **Else** (not a flag ledger)
|
||||
1. Copy the negative UNL ledger component from the parent ledger
|
||||
|
||||
The negative UNL is stored on each ledger because we don't know when a validator may reconnect to the network. If the negative UNL was stored only on every flag ledger, then a new validator would have to wait until it acquires the latest flag ledger to know the negative UNL. So any new ledgers created that are not flag ledgers copy the negative UNL from the parent ledger.
|
||||
The negative UNL is stored on each ledger because we don't know when a validator
|
||||
may reconnect to the network. If the negative UNL was stored only on every flag
|
||||
ledger, then a new validator would have to wait until it acquires the latest
|
||||
flag ledger to know the negative UNL. So any new ledgers created that are not
|
||||
flag ledgers copy the negative UNL from the parent ledger.
|
||||
|
||||
Note that when we have a validator to disable and a validator to re-enable at the same flag ledger, we create two separate `UNLModify` pseudo-transactions. We want either one or the other or both to make it into the ledger on their own merits.
|
||||
Note that when we have a validator to disable and a validator to re-enable at
|
||||
the same flag ledger, we create two separate `UNLModify` pseudo-transactions. We
|
||||
want either one or the other or both to make it into the ledger on their own
|
||||
merits.
|
||||
|
||||
Readers may have noticed that we defined several rules of creating the `UNLModify` pseudo-transactions but did not describe how to enforce the rules. The rules are actually enforced by the existing consensus algorithm. Unless enough validators propose the same pseudo-transaction it will not be included in the transaction set of the ledger.
|
||||
Readers may have noticed that we defined several rules of creating the
|
||||
`UNLModify` pseudo-transactions but did not describe how to enforce the rules.
|
||||
The rules are actually enforced by the existing consensus algorithm. Unless
|
||||
enough validators propose the same pseudo-transaction it will not be included in
|
||||
the transaction set of the ledger.
|
||||
|
||||
### Quorum Size Calculation
|
||||
|
||||
The effective quorum is 80% of the effective UNL. Note that because at most 25% of the original UNL can be on the negative UNL, the quorum should not be lower than the absolute minimum quorum (i.e. 60%) of the original UNL. However, considering that different nodes may have different UNLs, to be safe we compute `quorum = Ceiling(max(60% * original UNL, 80% * effective UNL))`.
|
||||
The effective quorum is 80% of the effective UNL. Note that because at most 25%
|
||||
of the original UNL can be on the negative UNL, the quorum should not be lower
|
||||
than the absolute minimum quorum (i.e. 60%) of the original UNL. However,
|
||||
considering that different nodes may have different UNLs, to be safe we compute
|
||||
`quorum = Ceiling(max(60% * original UNL, 80% * effective UNL))`.
|
||||
|
||||
### Filter Validation Messages
|
||||
|
||||
If a validator V is in the negative UNL, it still participates in consensus sessions in the same way, i.e. V still follows the protocol and publishes proposal and validation messages. The messages from V are still stored the same way by everyone, used to calculate the new PAV for V, and could be used in future consensus sessions if needed. However V's ledger validation message is not counted when checking if the ledger is fully validated.
|
||||
If a validator V is in the negative UNL, it still participates in consensus
|
||||
sessions in the same way, i.e. V still follows the protocol and publishes
|
||||
proposal and validation messages. The messages from V are still stored the same
|
||||
way by everyone, used to calculate the new PAV for V, and could be used in
|
||||
future consensus sessions if needed. However V's ledger validation message is
|
||||
not counted when checking if the ledger is fully validated.
|
||||
|
||||
### High Level Sequence Diagram of Code Changes
|
||||
|
||||
The diagram below is the sequence of one round of consensus. Classes and components with non-trivial changes are colored green.
|
||||
The diagram below is the sequence of one round of consensus. Classes and
|
||||
components with non-trivial changes are colored green.
|
||||
|
||||
- The `ValidatorList` class is modified to compute the quorum of the effective UNL.
|
||||
- The `ValidatorList` class is modified to compute the quorum of the effective
|
||||
UNL.
|
||||
|
||||
- The `Validations` class provides an interface for querying the validation messages from trusted validators.
|
||||
- The `Validations` class provides an interface for querying the validation
|
||||
messages from trusted validators.
|
||||
|
||||
- The `ConsensusAdaptor` component:
|
||||
- The `RCLConsensus::Adaptor` class is modified for creating `UNLModify` Pseudo-Transactions.
|
||||
- The `Change` class is modified for applying `UNLModify` Pseudo-Transactions.
|
||||
- The `Ledger` class is modified for creating and adjusting the negative UNL ledger component.
|
||||
- The `LedgerMaster` class is modified for filtering out validation messages from negative UNL validators when verifying if a ledger is fully validated.
|
||||
- The `RCLConsensus::Adaptor` class is modified for creating `UNLModify`
|
||||
Pseudo-Transactions.
|
||||
- The `Change` class is modified for applying `UNLModify`
|
||||
Pseudo-Transactions.
|
||||
- The `Ledger` class is modified for creating and adjusting the negative UNL
|
||||
ledger component.
|
||||
- The `LedgerMaster` class is modified for filtering out validation messages
|
||||
from negative UNL validators when verifying if a ledger is fully
|
||||
validated.
|
||||
|
||||

|
||||
@@ -170,29 +294,63 @@ Changes")
|
||||
|
||||
### Use a Mechanism Like Fee Voting to Process UNLModify Pseudo-Transactions
|
||||
|
||||
The previous version of the negative UNL specification used the same mechanism as the [fee voting](https://xrpl.org/fee-voting.html#voting-process.) for creating the negative UNL, and used the negative UNL as soon as the ledger was fully validated. However the timing of fully validation can differ among nodes, so different negative UNLs could be used, resulting in different effective UNLs and different quorums for the same ledger. As a result, the network's safety is impacted.
|
||||
The previous version of the negative UNL specification used the same mechanism
|
||||
as the [fee voting](https://xrpl.org/fee-voting.html#voting-process.) for
|
||||
creating the negative UNL, and used the negative UNL as soon as the ledger was
|
||||
fully validated. However the timing of fully validation can differ among nodes,
|
||||
so different negative UNLs could be used, resulting in different effective UNLs
|
||||
and different quorums for the same ledger. As a result, the network's safety is
|
||||
impacted.
|
||||
|
||||
This updated version does not impact safety though operates a bit more slowly. The negative UNL modifications in the _UNLModify_ pseudo-transaction approved by the consensus will take effect at the next flag ledger. The extra time of the 256 ledgers should be enough for nodes to be in sync of the negative UNL modifications.
|
||||
This updated version does not impact safety though operates a bit more slowly.
|
||||
The negative UNL modifications in the _UNLModify_ pseudo-transaction approved by
|
||||
the consensus will take effect at the next flag ledger. The extra time of the
|
||||
256 ledgers should be enough for nodes to be in sync of the negative UNL
|
||||
modifications.
|
||||
|
||||
### Use an Expiration Approach to Re-enable Validators
|
||||
|
||||
After a validator disabled by the negative UNL becomes reliable, other validators explicitly vote for re-enabling it. An alternative approach to re-enable a validator is the expiration approach, which was considered in the previous version of the specification. In the expiration approach, every entry in the negative UNL has a fixed expiration time. One flag ledger interval was chosen as the expiration interval. Once expired, the other validators must continue voting to keep the unreliable validator on the negative UNL. The advantage of this approach is its simplicity. But it has a requirement. The negative UNL protocol must be able to vote multiple unreliable validators to be disabled at the same flag ledger. In this version of the specification, however, only one unreliable validator can be disabled at a flag ledger. So the expiration approach cannot be simply applied.
|
||||
After a validator disabled by the negative UNL becomes reliable, other
|
||||
validators explicitly vote for re-enabling it. An alternative approach to
|
||||
re-enable a validator is the expiration approach, which was considered in the
|
||||
previous version of the specification. In the expiration approach, every entry
|
||||
in the negative UNL has a fixed expiration time. One flag ledger interval was
|
||||
chosen as the expiration interval. Once expired, the other validators must
|
||||
continue voting to keep the unreliable validator on the negative UNL. The
|
||||
advantage of this approach is its simplicity. But it has a requirement. The
|
||||
negative UNL protocol must be able to vote multiple unreliable validators to be
|
||||
disabled at the same flag ledger. In this version of the specification, however,
|
||||
only one unreliable validator can be disabled at a flag ledger. So the
|
||||
expiration approach cannot be simply applied.
|
||||
|
||||
### Validator Reliability Measurement and Flag Ledger Frequency
|
||||
|
||||
If the ledger time is about 4.5 seconds and the low-water mark is 50%, then in the worst case, it takes 48 minutes _((0.5 _ 256 + 256 + 256) _ 4.5 / 60 = 48)_ to put an offline validator on the negative UNL. We considered lowering the flag ledger frequency so that the negative UNL can be more responsive. We also considered decoupling the reliability measurement and flag ledger frequency to be more flexible. In practice, however, their benefits are not clear.
|
||||
If the ledger time is about 4.5 seconds and the low-water mark is 50%, then in
|
||||
the worst case, it takes 48 minutes _((0.5 _ 256 + 256 + 256) _ 4.5 / 60 = 48)_
|
||||
to put an offline validator on the negative UNL. We considered lowering the flag
|
||||
ledger frequency so that the negative UNL can be more responsive. We also
|
||||
considered decoupling the reliability measurement and flag ledger frequency to
|
||||
be more flexible. In practice, however, their benefits are not clear.
|
||||
|
||||
## New Attack Vectors
|
||||
|
||||
A group of malicious validators may try to frame a reliable validator and put it on the negative UNL. But they cannot succeed. Because:
|
||||
A group of malicious validators may try to frame a reliable validator and put it
|
||||
on the negative UNL. But they cannot succeed. Because:
|
||||
|
||||
1. A reliable validator sends a signed validation message every ledger. A sufficient peer-to-peer network will propagate the validation messages to other validators. The validators will decide if another validator is reliable or not only by its local observation of the validation messages received. So an honest validator’s vote on another validator’s reliability is accurate.
|
||||
1. A reliable validator sends a signed validation message every ledger. A
|
||||
sufficient peer-to-peer network will propagate the validation messages to other
|
||||
validators. The validators will decide if another validator is reliable or not
|
||||
only by its local observation of the validation messages received. So an honest
|
||||
validator’s vote on another validator’s reliability is accurate.
|
||||
|
||||
1. Given the votes are accurate, and one vote per validator, an honest validator will not create a UNLModify transaction of a reliable validator.
|
||||
1. Given the votes are accurate, and one vote per validator, an honest validator
|
||||
will not create a UNLModify transaction of a reliable validator.
|
||||
|
||||
1. A validator can be added to a negative UNL only through a UNLModify transaction.
|
||||
1. A validator can be added to a negative UNL only through a UNLModify
|
||||
transaction.
|
||||
|
||||
Assuming the group of malicious validators is less than the quorum, they cannot frame a reliable validator.
|
||||
Assuming the group of malicious validators is less than the quorum, they cannot
|
||||
frame a reliable validator.
|
||||
|
||||
## Summary
|
||||
|
||||
@@ -200,13 +358,16 @@ The bullet points below briefly summarize the current proposal:
|
||||
|
||||
- The motivation of the negative UNL is to improve the liveness of the network.
|
||||
|
||||
- The targeted faults are the ones frequently observed in the production network.
|
||||
- The targeted faults are the ones frequently observed in the production
|
||||
network.
|
||||
|
||||
- Validators propose negative UNL candidates based on their local measurements.
|
||||
|
||||
- The absolute minimum quorum is 60% of the original UNL.
|
||||
|
||||
- The format of the ledger is changed, and a new _UNLModify_ pseudo-transaction is added. Any tools or systems that rely on the format of these data will have to be updated.
|
||||
- The format of the ledger is changed, and a new _UNLModify_ pseudo-transaction
|
||||
is added. Any tools or systems that rely on the format of these data will have
|
||||
to be updated.
|
||||
|
||||
- The negative UNL can only be modified on the flag ledgers.
|
||||
|
||||
@@ -214,39 +375,59 @@ The bullet points below briefly summarize the current proposal:
|
||||
|
||||
- At most one validator can be removed from the negative UNL at a flag ledger.
|
||||
|
||||
- If a validator's reliability status changes, it takes two flag ledgers to modify the negative UNL.
|
||||
- If a validator's reliability status changes, it takes two flag ledgers to
|
||||
modify the negative UNL.
|
||||
|
||||
- The quorum is the larger of 80% of the effective UNL and 60% of the original UNL.
|
||||
- The quorum is the larger of 80% of the effective UNL and 60% of the original
|
||||
UNL.
|
||||
|
||||
- If a validator is on the negative UNL, its validation messages are ignored when the local node verifies if a ledger is fully validated.
|
||||
- If a validator is on the negative UNL, its validation messages are ignored
|
||||
when the local node verifies if a ledger is fully validated.
|
||||
|
||||
## FAQ
|
||||
|
||||
### Question: What are UNLs?
|
||||
|
||||
Quote from the [Technical FAQ](https://xrpl.org/technical-faq.html): "They are the lists of transaction validators a given participant believes will not conspire to defraud them."
|
||||
Quote from the [Technical FAQ](https://xrpl.org/technical-faq.html): "They are
|
||||
the lists of transaction validators a given participant believes will not
|
||||
conspire to defraud them."
|
||||
|
||||
### Question: How does the negative UNL proposal affect network liveness?
|
||||
|
||||
The network can make forward progress when more than a quorum of the trusted validators agree with the progress. The lower the quorum size is, the easier for the network to progress. If the quorum is too low, however, the network is not safe because nodes may have different results. So the quorum size used in the consensus protocol is a balance between the safety and the liveness of the network. The negative UNL reduces the size of the effective UNL, resulting in a lower quorum size while keeping the network safe.
|
||||
The network can make forward progress when more than a quorum of the trusted
|
||||
validators agree with the progress. The lower the quorum size is, the easier for
|
||||
the network to progress. If the quorum is too low, however, the network is not
|
||||
safe because nodes may have different results. So the quorum size used in the
|
||||
consensus protocol is a balance between the safety and the liveness of the
|
||||
network. The negative UNL reduces the size of the effective UNL, resulting in a
|
||||
lower quorum size while keeping the network safe.
|
||||
|
||||
<h3> Question: How does a validator get into the negative UNL? How is a
|
||||
validator removed from the negative UNL? </h3>
|
||||
|
||||
A validator’s reliability is measured by other validators. If a validator becomes unreliable, at a flag ledger, other validators propose _UNLModify_ pseudo-transactions which vote the validator to add to the negative UNL during the consensus session. If agreed, the validator is added to the negative UNL at the next flag ledger. The mechanism of removing a validator from the negative UNL is the same.
|
||||
A validator’s reliability is measured by other validators. If a validator
|
||||
becomes unreliable, at a flag ledger, other validators propose _UNLModify_
|
||||
pseudo-transactions which vote the validator to add to the negative UNL during
|
||||
the consensus session. If agreed, the validator is added to the negative UNL at
|
||||
the next flag ledger. The mechanism of removing a validator from the negative
|
||||
UNL is the same.
|
||||
|
||||
### Question: Given a negative UNL, what happens if the UNL changes?
|
||||
|
||||
Answer: Let’s consider the cases:
|
||||
|
||||
1. A validator is added to the UNL, and it is already in the negative UNL. This case could happen when not all the nodes have the same UNL. Note that the negative UNL on the ledger lists unreliable nodes that are not necessarily the validators for everyone.
|
||||
1. A validator is added to the UNL, and it is already in the negative UNL. This
|
||||
case could happen when not all the nodes have the same UNL. Note that the
|
||||
negative UNL on the ledger lists unreliable nodes that are not necessarily the
|
||||
validators for everyone.
|
||||
|
||||
In this case, the liveness is affected negatively. Because the minimum
|
||||
quorum could be larger but the usable validators are not increased.
|
||||
|
||||
1. A validator is removed from the UNL, and it is in the negative UNL.
|
||||
|
||||
In this case, the liveness is affected positively. Because the quorum could be smaller but the usable validators are not reduced.
|
||||
In this case, the liveness is affected positively. Because the quorum could
|
||||
be smaller but the usable validators are not reduced.
|
||||
|
||||
1. A validator is added to the UNL, and it is not in the negative UNL.
|
||||
1. A validator is removed from the UNL, and it is not in the negative UNL.
|
||||
@@ -257,23 +438,59 @@ Answer: Let’s consider the cases:
|
||||
|
||||
Answer: No, because the negative UNL approach is safer.
|
||||
|
||||
First let’s compare the two approaches intuitively, (1) the _negative UNL_ approach, and (2) _lower quorum_: simply lowering the quorum from 80% to 60% without the negative UNL. The negative UNL approach uses consensus to come up with a list of unreliable validators, which are then removed from the effective UNL temporarily. With this approach, the list of unreliable validators is agreed to by a quorum of validators and will be used by every node in the network to adjust its UNL. The quorum is always 80% of the effective UNL. The lower quorum approach is a tradeoff between safety and liveness and against our principle of preferring safety over liveness. Note that different validators don't have to agree on which validation sources they are ignoring.
|
||||
First let’s compare the two approaches intuitively, (1) the _negative UNL_
|
||||
approach, and (2) _lower quorum_: simply lowering the quorum from 80% to 60%
|
||||
without the negative UNL. The negative UNL approach uses consensus to come up
|
||||
with a list of unreliable validators, which are then removed from the effective
|
||||
UNL temporarily. With this approach, the list of unreliable validators is agreed
|
||||
to by a quorum of validators and will be used by every node in the network to
|
||||
adjust its UNL. The quorum is always 80% of the effective UNL. The lower quorum
|
||||
approach is a tradeoff between safety and liveness and against our principle of
|
||||
preferring safety over liveness. Note that different validators don't have to
|
||||
agree on which validation sources they are ignoring.
|
||||
|
||||
Next we compare the two approaches quantitatively with examples, and apply Theorem 8 of [Analysis of the XRP Ledger Consensus Protocol](https://arxiv.org/abs/1802.07242) paper:
|
||||
Next we compare the two approaches quantitatively with examples, and apply
|
||||
Theorem 8 of [Analysis of the XRP Ledger Consensus
|
||||
Protocol](https://arxiv.org/abs/1802.07242) paper:
|
||||
|
||||
_XRP LCP guarantees fork safety if **O<sub>i,j</sub> > n<sub>j</sub> / 2 + n<sub>i</sub> − q<sub>i</sub> + t<sub>i,j</sub>** for every pair of nodes P<sub>i</sub>, P<sub>j</sub>,_
|
||||
_XRP LCP guarantees fork safety if **O<sub>i,j</sub> > n<sub>j</sub> / 2 +
|
||||
n<sub>i</sub> − q<sub>i</sub> + t<sub>i,j</sub>** for every pair of nodes
|
||||
P<sub>i</sub>, P<sub>j</sub>,_
|
||||
|
||||
where _O<sub>i,j</sub>_ is the overlapping requirement, n<sub>j</sub> and n<sub>i</sub> are UNL sizes, q<sub>i</sub> is the quorum size of P<sub>i</sub>, _t<sub>i,j</sub> = min(t<sub>i</sub>, t<sub>j</sub>, O<sub>i,j</sub>)_, and t<sub>i</sub> and t<sub>j</sub> are the number of faults can be tolerated by P<sub>i</sub> and P<sub>j</sub>.
|
||||
where _O<sub>i,j</sub>_ is the overlapping requirement, n<sub>j</sub> and
|
||||
n<sub>i</sub> are UNL sizes, q<sub>i</sub> is the quorum size of P<sub>i</sub>,
|
||||
_t<sub>i,j</sub> = min(t<sub>i</sub>, t<sub>j</sub>, O<sub>i,j</sub>)_, and
|
||||
t<sub>i</sub> and t<sub>j</sub> are the number of faults can be tolerated by
|
||||
P<sub>i</sub> and P<sub>j</sub>.
|
||||
|
||||
We denote _UNL<sub>i</sub>_ as _P<sub>i</sub>'s UNL_, and _|UNL<sub>i</sub>|_ as the size of _P<sub>i</sub>'s UNL_.
|
||||
We denote _UNL<sub>i</sub>_ as _P<sub>i</sub>'s UNL_, and _|UNL<sub>i</sub>|_ as
|
||||
the size of _P<sub>i</sub>'s UNL_.
|
||||
|
||||
Assuming _|UNL<sub>i</sub>| = |UNL<sub>j</sub>|_, let's consider the following three cases:
|
||||
Assuming _|UNL<sub>i</sub>| = |UNL<sub>j</sub>|_, let's consider the following
|
||||
three cases:
|
||||
|
||||
1. With 80% quorum and 20% faults, _O<sub>i,j</sub> > 100% / 2 + 100% - 80% + 20% = 90%_. I.e. fork safety requires > 90% UNL overlaps. This is one of the results in the analysis paper.
|
||||
1. With 80% quorum and 20% faults, _O<sub>i,j</sub> > 100% / 2 + 100% - 80% +
|
||||
20% = 90%_. I.e. fork safety requires > 90% UNL overlaps. This is one of the
|
||||
results in the analysis paper.
|
||||
|
||||
1. If the quorum is 60%, the relationship between the overlapping requirement and the faults that can be tolerated is _O<sub>i,j</sub> > 90% + t<sub>i,j</sub>_. Under the same overlapping condition (i.e. 90%), to guarantee the fork safety, the network cannot tolerate any faults. So under the same overlapping condition, if the quorum is simply lowered, the network can tolerate fewer faults.
|
||||
1. If the quorum is 60%, the relationship between the overlapping requirement
|
||||
and the faults that can be tolerated is _O<sub>i,j</sub> > 90% +
|
||||
t<sub>i,j</sub>_. Under the same overlapping condition (i.e. 90%), to guarantee
|
||||
the fork safety, the network cannot tolerate any faults. So under the same
|
||||
overlapping condition, if the quorum is simply lowered, the network can tolerate
|
||||
fewer faults.
|
||||
|
||||
1. With the negative UNL approach, we want to argue that the inequation _O<sub>i,j</sub> > n<sub>j</sub> / 2 + n<sub>i</sub> − q<sub>i</sub> + t<sub>i,j</sub>_ is always true to guarantee fork safety, while the negative UNL protocol runs, i.e. the effective quorum is lowered without weakening the network's fault tolerance. To make the discussion easier, we rewrite the inequation as _O<sub>i,j</sub> > n<sub>j</sub> / 2 + (n<sub>i</sub> − q<sub>i</sub>) + min(t<sub>i</sub>, t<sub>j</sub>)_, where O<sub>i,j</sub> is dropped from the definition of t<sub>i,j</sub> because _O<sub>i,j</sub> > min(t<sub>i</sub>, t<sub>j</sub>)_ always holds under the parameters we will use. Assuming a validator V is added to the negative UNL, now let's consider the 4 cases:
|
||||
1. With the negative UNL approach, we want to argue that the inequation
|
||||
_O<sub>i,j</sub> > n<sub>j</sub> / 2 + n<sub>i</sub> − q<sub>i</sub> +
|
||||
t<sub>i,j</sub>_ is always true to guarantee fork safety, while the negative UNL
|
||||
protocol runs, i.e. the effective quorum is lowered without weakening the
|
||||
network's fault tolerance. To make the discussion easier, we rewrite the
|
||||
inequation as _O<sub>i,j</sub> > n<sub>j</sub> / 2 + (n<sub>i</sub> −
|
||||
q<sub>i</sub>) + min(t<sub>i</sub>, t<sub>j</sub>)_, where O<sub>i,j</sub> is
|
||||
dropped from the definition of t<sub>i,j</sub> because _O<sub>i,j</sub> >
|
||||
min(t<sub>i</sub>, t<sub>j</sub>)_ always holds under the parameters we will
|
||||
use. Assuming a validator V is added to the negative UNL, now let's consider the
|
||||
4 cases:
|
||||
|
||||
1. V is not on UNL<sub>i</sub> nor UNL<sub>j</sub>
|
||||
|
||||
@@ -309,35 +526,64 @@ Assuming _|UNL<sub>i</sub>| = |UNL<sub>j</sub>|_, let's consider the following t
|
||||
<h3> Question: We have observed that occasionally a validator wanders off on its
|
||||
own chain. How is this case handled by the negative UNL algorithm? </h3>
|
||||
|
||||
Answer: The case that a validator wanders off on its own chain can be measured with the validations agreement. Because the validations by this validator must be different from other validators' validations of the same sequence numbers. When there are enough disagreed validations, other validators will vote this validator onto the negative UNL.
|
||||
Answer: The case that a validator wanders off on its own chain can be measured
|
||||
with the validations agreement. Because the validations by this validator must
|
||||
be different from other validators' validations of the same sequence numbers.
|
||||
When there are enough disagreed validations, other validators will vote this
|
||||
validator onto the negative UNL.
|
||||
|
||||
In general by measuring the agreement of validations, we also measured the "sanity". If two validators have too many disagreements, one of them could be insane. When enough validators think a validator is insane, that validator is put on the negative UNL.
|
||||
In general by measuring the agreement of validations, we also measured the
|
||||
"sanity". If two validators have too many disagreements, one of them could be
|
||||
insane. When enough validators think a validator is insane, that validator is
|
||||
put on the negative UNL.
|
||||
|
||||
<h3> Question: Why would there be at most one disable UNLModify and one
|
||||
re-enable UNLModify transaction per flag ledger? </h3>
|
||||
|
||||
Answer: It is a design choice so that the effective UNL does not change too quickly. A typical targeted scenario is several validators go offline slowly during a long weekend. The current design can handle this kind of cases well without changing the effective UNL too quickly.
|
||||
Answer: It is a design choice so that the effective UNL does not change too
|
||||
quickly. A typical targeted scenario is several validators go offline slowly
|
||||
during a long weekend. The current design can handle this kind of cases well
|
||||
without changing the effective UNL too quickly.
|
||||
|
||||
## Appendix
|
||||
|
||||
### Confidence Test
|
||||
|
||||
We will use two test networks, a single machine test network with multiple IP addresses and the QE test network with multiple machines. The single machine network will be used to test all the test cases and to debug. The QE network will be used after that. We want to see the test cases still pass with real network delay. A test case specifies:
|
||||
We will use two test networks, a single machine test network with multiple IP
|
||||
addresses and the QE test network with multiple machines. The single machine
|
||||
network will be used to test all the test cases and to debug. The QE network
|
||||
will be used after that. We want to see the test cases still pass with real
|
||||
network delay. A test case specifies:
|
||||
|
||||
1. a UNL with different number of validators for different test cases,
|
||||
1. a network with zero or more non-validator nodes,
|
||||
1. a sequence of validator reliability change events (by killing/restarting nodes, or by running modified xrpld that does not send all validation messages),
|
||||
1. a sequence of validator reliability change events (by killing/restarting
|
||||
nodes, or by running modified xrpld that does not send all validation
|
||||
messages),
|
||||
1. the correct outcomes.
|
||||
|
||||
For all the test cases, the correct outcomes are verified by examining logs. We will grep the log to see if the correct negative UNLs are generated, and whether or not the network is making progress when it should be. The ripdtop tool will be helpful for monitoring validators' states and ledger progress. Some of the timing parameters of xrpld will be changed to have faster ledger time. Most if not all test cases do not need client transactions.
|
||||
For all the test cases, the correct outcomes are verified by examining logs. We
|
||||
will grep the log to see if the correct negative UNLs are generated, and whether
|
||||
or not the network is making progress when it should be. The ripdtop tool will
|
||||
be helpful for monitoring validators' states and ledger progress. Some of the
|
||||
timing parameters of xrpld will be changed to have faster ledger time. Most if
|
||||
not all test cases do not need client transactions.
|
||||
|
||||
For example, the test cases for the prototype:
|
||||
|
||||
1. A 10-validator UNL.
|
||||
1. The network does not have other nodes.
|
||||
1. The validators will be started from the genesis. Once they start to produce ledgers, we kill five validators, one every flag ledger interval. Then we will restart them one by one.
|
||||
1. A sequence of events (or the lack of events) such as a killed validator is added to the negative UNL.
|
||||
1. The validators will be started from the genesis. Once they start to produce
|
||||
ledgers, we kill five validators, one every flag ledger interval. Then we
|
||||
will restart them one by one.
|
||||
1. A sequence of events (or the lack of events) such as a killed validator is
|
||||
added to the negative UNL.
|
||||
|
||||
#### Roads Not Taken: Test with Extended CSF
|
||||
|
||||
We considered testing with the current unit test framework, specifically the [Consensus Simulation Framework](https://github.com/XRPLF/rippled/blob/develop/src/test/csf/README.md) (CSF). However, the CSF currently can only test the generic consensus algorithm as in the paper: [Analysis of the XRP Ledger Consensus Protocol](https://arxiv.org/abs/1802.07242).
|
||||
We considered testing with the current unit test framework, specifically the
|
||||
[Consensus Simulation
|
||||
Framework](https://github.com/XRPLF/rippled/blob/develop/src/test/csf/README.md)
|
||||
(CSF). However, the CSF currently can only test the generic consensus algorithm
|
||||
as in the paper: [Analysis of the XRP Ledger Consensus
|
||||
Protocol](https://arxiv.org/abs/1802.07242).
|
||||
@@ -1,20 +1,63 @@
|
||||
# Ledger Replay
|
||||
|
||||
`LedgerReplayer` is a new `Stoppable` for replaying ledgers. Patterned after two other `Stoppable`s under `JobQueue`---`InboundLedgers` and `InboundTransactions`---it acts like a factory for creating state-machine workers, and a network message demultiplexer for those workers. Think of these workers like asynchronous functions. Like functions, they each take a set of parameters. The `Stoppable` memoizes these functions. It maintains a table for each worker type, mapping sets of arguments to the worker currently working on that argument set. Whenever the `Stoppable` is asked to construct a worker, it first searches its table to see if there is an existing worker with the same or overlapping argument set. If one exists, then it is used. If not, then a new one is created, initialized, and added to the table.
|
||||
`LedgerReplayer` is a new `Stoppable` for replaying ledgers.
|
||||
Patterned after two other `Stoppable`s under `JobQueue`---`InboundLedgers`
|
||||
and `InboundTransactions`---it acts like a factory for creating
|
||||
state-machine workers, and a network message demultiplexer for those workers.
|
||||
Think of these workers like asynchronous functions.
|
||||
Like functions, they each take a set of parameters.
|
||||
The `Stoppable` memoizes these functions. It maintains a table for each
|
||||
worker type, mapping sets of arguments to the worker currently working
|
||||
on that argument set.
|
||||
Whenever the `Stoppable` is asked to construct a worker, it first searches its
|
||||
table to see if there is an existing worker with the same or overlapping
|
||||
argument set.
|
||||
If one exists, then it is used. If not, then a new one is created,
|
||||
initialized, and added to the table.
|
||||
|
||||
For `LedgerReplayer`, there are three worker types: `LedgerReplayTask`, `SkipListAcquire`, and `LedgerDeltaAcquire`. Each is derived from `TimeoutCounter` to give it a timeout. For `LedgerReplayTask`, the parameter set is {reason, finish ledger ID, number of ledgers}. For `SkipListAcquire` and `LedgerDeltaAcquire`, there is just one parameter: a ledger ID.
|
||||
For `LedgerReplayer`, there are three worker types: `LedgerReplayTask`,
|
||||
`SkipListAcquire`, and `LedgerDeltaAcquire`.
|
||||
Each is derived from `TimeoutCounter` to give it a timeout.
|
||||
For `LedgerReplayTask`, the parameter set
|
||||
is {reason, finish ledger ID, number of ledgers}. For `SkipListAcquire` and
|
||||
`LedgerDeltaAcquire`, there is just one parameter: a ledger ID.
|
||||
|
||||
Each `Stoppable` has an entry point. For `LedgerReplayer`, it is `replay`. `replay` creates two workers: a `LedgerReplayTask` and a `SkipListAcquire`. `LedgerDeltaAcquire`s are created in the callback for when the skip list returns.
|
||||
Each `Stoppable` has an entry point. For `LedgerReplayer`, it is `replay`.
|
||||
`replay` creates two workers: a `LedgerReplayTask` and a `SkipListAcquire`.
|
||||
`LedgerDeltaAcquire`s are created in the callback for when the skip list
|
||||
returns.
|
||||
|
||||
For `SkipListAcquire` and `LedgerDeltaAcquire`, initialization fires off the underlying asynchronous network request and starts the timeout. The argument set identifying the worker is included in the network request, and copied to the network response. `SkipListAcquire` sends a request for a proof path for the skip list of the desired ledger. `LedgerDeltaAcquire` sends a request for the transaction set of the desired ledger.
|
||||
For `SkipListAcquire` and `LedgerDeltaAcquire`, initialization fires off the
|
||||
underlying asynchronous network request and starts the timeout. The argument
|
||||
set identifying the worker is included in the network request, and copied to
|
||||
the network response. `SkipListAcquire` sends a request for a proof path for
|
||||
the skip list of the desired ledger. `LedgerDeltaAcquire` sends a request for
|
||||
the transaction set of the desired ledger.
|
||||
|
||||
`LedgerReplayer` is also a network message demultiplexer. When a response arrives for a request that was sent by a `SkipListAcquire` or `LedgerDeltaAcquire` worker, the `Peer` object knows to send it to the `LedgerReplayer`, which looks up the worker waiting for that response based on the identifying argument set included in the response.
|
||||
`LedgerReplayer` is also a network message demultiplexer.
|
||||
When a response arrives for a request that was sent by a `SkipListAcquire` or
|
||||
`LedgerDeltaAcquire` worker, the `Peer` object knows to send it to the
|
||||
`LedgerReplayer`, which looks up the worker waiting for that response based on
|
||||
the identifying argument set included in the response.
|
||||
|
||||
`LedgerReplayTask` may ask `InboundLedgers` to send requests to acquire the start ledger, but there is no way to attach a callback or be notified when the `InboundLedger` worker completes. All the responses for its messages will be directed to `InboundLedgers`, not `LedgerReplayer`. Instead, `LedgerReplayTask` checks whether the start ledger has arrived every time its timeout expires.
|
||||
`LedgerReplayTask` may ask `InboundLedgers` to send requests to acquire
|
||||
the start ledger, but there is no way to attach a callback or be notified when
|
||||
the `InboundLedger` worker completes. All the responses for its messages will
|
||||
be directed to `InboundLedgers`, not `LedgerReplayer`. Instead,
|
||||
`LedgerReplayTask` checks whether the start ledger has arrived every time its
|
||||
timeout expires.
|
||||
|
||||
Like a promise, each worker keeps track of whether it is pending (`!isDone()`) or whether it has resolved successfully (`complete_ == true`) or unsuccessfully (`failed_ == true`). It will never exist in both resolved states at once, nor will it return to a pending state after reaching a resolved state.
|
||||
Like a promise, each worker keeps track of whether it is pending (`!isDone()`)
|
||||
or whether it has resolved successfully (`complete_ == true`) or unsuccessfully
|
||||
(`failed_ == true`). It will never exist in both resolved states at once, nor
|
||||
will it return to a pending state after reaching a resolved state.
|
||||
|
||||
Like promises, some workers can accept continuations to be called when they reach a resolved state, or immediately if they are already resolved. `SkipListAcquire` and `LedgerDeltaAcquire` both accept continuations of a type specific to their payload, both via a method named `addDataCallback()`. Continuations cannot be removed explicitly, but they are held by `std::weak_ptr` so they can be removed implicitly.
|
||||
Like promises, some workers can accept continuations to be called when they
|
||||
reach a resolved state, or immediately if they are already resolved.
|
||||
`SkipListAcquire` and `LedgerDeltaAcquire` both accept continuations of a type
|
||||
specific to their payload, both via a method named `addDataCallback()`. Continuations
|
||||
cannot be removed explicitly, but they are held by `std::weak_ptr` so they can
|
||||
be removed implicitly.
|
||||
|
||||
`LedgerReplayTask` is simultaneously:
|
||||
|
||||
@@ -30,7 +73,13 @@ Each of these roles corresponds to different entry points:
|
||||
1. the callback added to `LedgerDeltaAcquire`, which calls `deltaReady(...)` or `cancel()`
|
||||
1. `onTimer()`
|
||||
|
||||
Each of these entry points does something unique to that entry point. They either (a) transition `LedgerReplayTask` to a terminal failed resolved state (`cancel()` and `onTimer()`) or (b) try to make progress toward the successful resolved state. `init()` and `updateSkipList(...)` call `trigger()` while `deltaReady(...)` calls `tryAdvance()`. There's a similarity between this pattern and the way coroutines are implemented, where every yield saves the spot in the code where it left off and every resume jumps back to that spot.
|
||||
Each of these entry points does something unique to that entry point. They
|
||||
either (a) transition `LedgerReplayTask` to a terminal failed resolved state
|
||||
(`cancel()` and `onTimer()`) or (b) try to make progress toward the successful
|
||||
resolved state. `init()` and `updateSkipList(...)` call `trigger()` while
|
||||
`deltaReady(...)` calls `tryAdvance()`. There's a similarity between this
|
||||
pattern and the way coroutines are implemented, where every yield saves the spot
|
||||
in the code where it left off and every resume jumps back to that spot.
|
||||
|
||||
### Sequence Diagram
|
||||
|
||||
|
||||
+50
-17
@@ -1,10 +1,13 @@
|
||||
# Coding Standards
|
||||
|
||||
Coding standards used here gradually evolve and propagate through code reviews. Some aspects are enforced more strictly than others.
|
||||
Coding standards used here gradually evolve and propagate through
|
||||
code reviews. Some aspects are enforced more strictly than others.
|
||||
|
||||
## Rules
|
||||
|
||||
These rules only apply to our own code. We can't enforce any sort of style on the external repositories and libraries we include. The best guideline is to maintain the standards that are used in those libraries.
|
||||
These rules only apply to our own code. We can't enforce any sort of
|
||||
style on the external repositories and libraries we include. The best
|
||||
guideline is to maintain the standards that are used in those libraries.
|
||||
|
||||
- Tab inserts 4 spaces. No tab characters.
|
||||
- Braces are indented in the [Allman style][1].
|
||||
@@ -13,37 +16,67 @@ These rules only apply to our own code. We can't enforce any sort of style on th
|
||||
|
||||
## Guidelines
|
||||
|
||||
If you want to do something contrary to these guidelines, understand why you're doing it. Think, use common sense, and consider that these changes will probably need to be maintained long after you've moved on to other projects.
|
||||
If you want to do something contrary to these guidelines, understand
|
||||
why you're doing it. Think, use common sense, and consider that these
|
||||
changes will probably need to be maintained long after you've
|
||||
moved on to other projects.
|
||||
|
||||
- Use white space and blank lines to guide the eye and keep your intent clear.
|
||||
- Put private data members at the top of a class, and the 6 public special members immediately after, in the following order:
|
||||
- Put private data members at the top of a class, and the 6 public special
|
||||
members immediately after, in the following order:
|
||||
- Destructor
|
||||
- Default constructor
|
||||
- Copy constructor
|
||||
- Copy assignment
|
||||
- Move constructor
|
||||
- Move assignment
|
||||
- Don't over-inline by defining large functions within the class declaration, not even for template classes.
|
||||
- Don't over-inline by defining large functions within the class
|
||||
declaration, not even for template classes.
|
||||
|
||||
## Formatting
|
||||
|
||||
The goal of source code formatting should always be to make things as easy to read as possible. White space is used to guide the eye so that details are not overlooked. Blank lines are used to separate code into "paragraphs."
|
||||
The goal of source code formatting should always be to make things as easy to
|
||||
read as possible. White space is used to guide the eye so that details are not
|
||||
overlooked. Blank lines are used to separate code into "paragraphs."
|
||||
|
||||
- Always place a space before and after all binary operators, especially assignments (`operator=`).
|
||||
- Always place a space before and after all binary operators,
|
||||
especially assignments (`operator=`).
|
||||
- The `!` operator should be preceded by a space, but not followed by one.
|
||||
- The `~` operator should be preceded by a space, but not followed by one.
|
||||
- The `++` and `--` operators should have no spaces between the operator and the operand.
|
||||
- The `++` and `--` operators should have no spaces between the operator and
|
||||
the operand.
|
||||
- A space never appears before a comma, and always appears after a comma.
|
||||
- Don't put spaces after a parenthesis. A typical member function call might look like this: `foobar (1, 2, 3);`
|
||||
- Don't put spaces after a parenthesis. A typical member function call might
|
||||
look like this: `foobar (1, 2, 3);`
|
||||
- In general, leave a blank line before an `if` statement.
|
||||
- In general, leave a blank line after a closing brace `}`.
|
||||
- Do not place code on the same line as any opening or closing brace.
|
||||
- Do not write `if` statements all-on-one-line. The exception to this is when you've got a sequence of similar `if` statements, and are aligning them all vertically to highlight their similarities.
|
||||
- In an `if-else` statement, if you surround one half of the statement with braces, you also need to put braces around the other half, to match.
|
||||
- When writing a pointer type, use this spacing: `SomeObject* myObject`. Technically, a more correct spacing would be `SomeObject *myObject`, but it makes more sense for the asterisk to be grouped with the type name, since being a pointer is part of the type, not the variable name. The only time that this can lead to any problems is when you're declaring multiple pointers of the same type in the same statement - which leads on to the next rule:
|
||||
- When declaring multiple pointers, never do so in a single statement, e.g. `SomeObject* p1, *p2;` - instead, always split them out onto separate lines and write the type name again, to make it quite clear what's going on, and avoid the danger of missing out any vital asterisks.
|
||||
- The previous point also applies to references, so always put the `&` next to the type rather than the variable, e.g. `void foo (Thing const& thing)`. And don't put a space on both sides of the `*` or `&` - always put a space after it, but never before it.
|
||||
- The word `const` should be placed to the right of the thing that it modifies, for consistency. For example `int const` refers to an int which is const. `int const*` is a pointer to an int which is const. `int *const` is a const pointer to an int.
|
||||
- Always place a space in between the template angle brackets and the type name. Template code is already hard enough to read!
|
||||
- Do not place code on the same line as any opening or
|
||||
closing brace.
|
||||
- Do not write `if` statements all-on-one-line. The exception to this is when
|
||||
you've got a sequence of similar `if` statements, and are aligning them all
|
||||
vertically to highlight their similarities.
|
||||
- In an `if-else` statement, if you surround one half of the statement with
|
||||
braces, you also need to put braces around the other half, to match.
|
||||
- When writing a pointer type, use this spacing: `SomeObject* myObject`.
|
||||
Technically, a more correct spacing would be `SomeObject *myObject`, but
|
||||
it makes more sense for the asterisk to be grouped with the type name,
|
||||
since being a pointer is part of the type, not the variable name. The only
|
||||
time that this can lead to any problems is when you're declaring multiple
|
||||
pointers of the same type in the same statement - which leads on to the next
|
||||
rule:
|
||||
- When declaring multiple pointers, never do so in a single statement, e.g.
|
||||
`SomeObject* p1, *p2;` - instead, always split them out onto separate lines
|
||||
and write the type name again, to make it quite clear what's going on, and
|
||||
avoid the danger of missing out any vital asterisks.
|
||||
- The previous point also applies to references, so always put the `&` next to
|
||||
the type rather than the variable, e.g. `void foo (Thing const& thing)`. And
|
||||
don't put a space on both sides of the `*` or `&` - always put a space after
|
||||
it, but never before it.
|
||||
- The word `const` should be placed to the right of the thing that it modifies,
|
||||
for consistency. For example `int const` refers to an int which is const.
|
||||
`int const*` is a pointer to an int which is const. `int *const` is a const
|
||||
pointer to an int.
|
||||
- Always place a space in between the template angle brackets and the type
|
||||
name. Template code is already hard enough to read!
|
||||
|
||||
[1]: http://en.wikipedia.org/wiki/Indent_style#Allman_style
|
||||
+17
-3
@@ -1,5 +1,19 @@
|
||||
# `xrpld` Docker Image
|
||||
|
||||
- Some info relating to Docker containers can be found here: [../Builds/containers](../Builds/containers)
|
||||
- Images for building and testing xrpld can be found here: [thejohnfreeman/rippled-docker](https://github.com/thejohnfreeman/rippled-docker/)
|
||||
- These images do not have xrpld. They have all the tools necessary to build xrpld.
|
||||
`xrpld` is published to Docker Hub as [`xrplf/xrpld`](https://hub.docker.com/r/xrplf/xrpld):
|
||||
the `xrpld` DEB package installed on Ubuntu 26.04, running as the `xrpld` user.
|
||||
Each release is tagged with its version, `xrplf/xrpld:<version>`, and
|
||||
`xrplf/xrpld:develop` follows the `develop` branch.
|
||||
See [`package/README.md`](../package/README.md#docker-images) for how it is built
|
||||
and tagged.
|
||||
|
||||
```bash
|
||||
docker run --detach --name xrpld \
|
||||
--volume xrpld-db:/var/lib/xrpld \
|
||||
--publish 2459:2459 \
|
||||
xrplf/xrpld:develop
|
||||
```
|
||||
|
||||
The admin ports (5005, 6006 and 50051) listen on `127.0.0.1` in the shipped
|
||||
configuration; to reach them from outside the container, mount your own over
|
||||
`/etc/xrpld/xrpld.cfg`.
|
||||
+31
-8
@@ -1,18 +1,33 @@
|
||||
## Heap profiling of xrpld with jemalloc
|
||||
|
||||
The jemalloc library provides a good API for doing heap analysis, including a mechanism to dump a description of the heap from within the running application via a function call. Details on how to perform this activity in general, as well as how to acquire the software, are available on the jemalloc site: [https://github.com/jemalloc/jemalloc/wiki/Use-Case:-Heap-Profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case:-Heap-Profiling)
|
||||
The jemalloc library provides a good API for doing heap analysis,
|
||||
including a mechanism to dump a description of the heap from within the
|
||||
running application via a function call. Details on how to perform this
|
||||
activity in general, as well as how to acquire the software, are available on
|
||||
the jemalloc site:
|
||||
[https://github.com/jemalloc/jemalloc/wiki/Use-Case:-Heap-Profiling](https://github.com/jemalloc/jemalloc/wiki/Use-Case:-Heap-Profiling)
|
||||
|
||||
jemalloc is acquired separately from xrpld, and is not affiliated with Ripple Labs. If you compile and install jemalloc from the source release with default options, it will install the library and header under `/usr/local/lib` and `/usr/local/include`, respectively. Heap profiling has been tested with xrpld on a Linux platform. It should work on platforms on which both xrpld and jemalloc are available.
|
||||
jemalloc is acquired separately from xrpld, and is not affiliated
|
||||
with Ripple Labs. If you compile and install jemalloc from the
|
||||
source release with default options, it will install the library and header
|
||||
under `/usr/local/lib` and `/usr/local/include`, respectively. Heap
|
||||
profiling has been tested with xrpld on a Linux platform. It should
|
||||
work on platforms on which both xrpld and jemalloc are available.
|
||||
|
||||
To link xrpld with jemalloc, the argument `profile-jemalloc=<jemalloc_dir>` is provided after the optional target. The `<jemalloc_dir>` argument should be the same as that of the `--prefix` parameter passed to the jemalloc configure script when building.
|
||||
To link xrpld with jemalloc, the argument
|
||||
`profile-jemalloc=<jemalloc_dir>` is provided after the optional target.
|
||||
The `<jemalloc_dir>` argument should be the same as that of the
|
||||
`--prefix` parameter passed to the jemalloc configure script when building.
|
||||
|
||||
## Examples:
|
||||
|
||||
Build xrpld with jemalloc library under /usr/local/lib and header under /usr/local/include:
|
||||
Build xrpld with jemalloc library under /usr/local/lib and
|
||||
header under /usr/local/include:
|
||||
|
||||
$ scons profile-jemalloc=/usr/local
|
||||
|
||||
Build xrpld using clang with the jemalloc library under /opt/local/lib and header under /opt/local/include:
|
||||
Build xrpld using clang with the jemalloc library under /opt/local/lib
|
||||
and header under /opt/local/include:
|
||||
|
||||
$ scons clang profile-jemalloc=/opt/local
|
||||
|
||||
@@ -20,7 +35,10 @@ Build xrpld using clang with the jemalloc library under /opt/local/lib and heade
|
||||
|
||||
## Using the jemalloc library from within the code
|
||||
|
||||
The `profile-jemalloc` parameter enables a macro definition called `PROFILE_JEMALLOC`. Include the jemalloc header file as well as the api call(s) that you wish to make within preprocessor conditional groups, such as:
|
||||
The `profile-jemalloc` parameter enables a macro definition called
|
||||
`PROFILE_JEMALLOC`. Include the jemalloc header file as
|
||||
well as the api call(s) that you wish to make within preprocessor
|
||||
conditional groups, such as:
|
||||
|
||||
In global scope:
|
||||
|
||||
@@ -34,6 +52,11 @@ And later, within a function scope:
|
||||
mallctl("prof.dump", NULL, NULL, NULL, 0);
|
||||
#endif
|
||||
|
||||
Fuller descriptions of how to acquire and use jemalloc's api to do memory analysis are available at the [jemalloc site.](http://www.canonware.com/jemalloc/)
|
||||
Fuller descriptions of how to acquire and use jemalloc's api to do memory
|
||||
analysis are available at the [jemalloc
|
||||
site.](http://www.canonware.com/jemalloc/)
|
||||
|
||||
Linking against the jemalloc library will override the system's default `malloc()` and related functions with jemalloc's implementation. This is the case even if the code is not instrumented to use jemalloc's specific API.
|
||||
Linking against the jemalloc library will override
|
||||
the system's default `malloc()` and related functions with jemalloc's
|
||||
implementation. This is the case even if the code is not instrumented
|
||||
to use jemalloc's specific API.
|
||||
Vendored
+54
-20
@@ -4,25 +4,34 @@ This document provides advanced instructions for setting up and configuring Cona
|
||||
|
||||
## Custom profile
|
||||
|
||||
If the default profile does not work for you and you do not yet have a Conan profile, you can create one by running:
|
||||
If the default profile does not work for you and you do not yet have a Conan
|
||||
profile, you can create one by running:
|
||||
|
||||
```bash
|
||||
conan profile detect
|
||||
```
|
||||
|
||||
You may need to make changes to the profile to suit your environment. You can refer to the provided `conan/profiles/default` profile for inspiration, and you may also need to apply the required [tweaks](#conan-profile-tweaks) to this default profile.
|
||||
You may need to make changes to the profile to suit your environment. You can
|
||||
refer to the provided `conan/profiles/default` profile for inspiration, and you
|
||||
may also need to apply the required [tweaks](#conan-profile-tweaks) to this
|
||||
default profile.
|
||||
|
||||
## Conan lockfile
|
||||
|
||||
To achieve reproducible dependencies, we use a [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html), which has to be updated every time dependencies change.
|
||||
To achieve reproducible dependencies, we use a [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html),
|
||||
which has to be updated every time dependencies change.
|
||||
|
||||
Please see the [instructions on how to regenerate the lockfile](../../conan/lockfile/README.md).
|
||||
|
||||
## Patched recipes
|
||||
|
||||
Occasionally, we need patched recipes or recipes not present in Conan Center. We maintain a fork of the Conan Center Index [here](https://github.com/XRPLF/conan-center-index/) containing the modified and newly added recipes.
|
||||
Occasionally, we need patched recipes or recipes not present in Conan Center.
|
||||
We maintain a fork of the Conan Center Index
|
||||
[here](https://github.com/XRPLF/conan-center-index/) containing the modified and newly added recipes.
|
||||
|
||||
To ensure our patched recipes are used, you must add our Conan remote at a higher index than the default Conan Center remote, so it is consulted first. You can do this by running:
|
||||
To ensure our patched recipes are used, you must add our Conan remote at a
|
||||
higher index than the default Conan Center remote, so it is consulted first. You
|
||||
can do this by running:
|
||||
|
||||
```bash
|
||||
conan remote add --index 0 --force xrplf https://conan.xrplf.org/repository/conan/
|
||||
@@ -52,11 +61,16 @@ git checkout master
|
||||
cd ../../
|
||||
```
|
||||
|
||||
In the case we switch to a newer version of a dependency that still requires a patch or add a new dependency, it will be necessary for you to pull in the changes and re-export the updated dependencies with the newer version. However, if we switch to a newer version that no longer requires a patch, no action is required on your part, as the new recipe will be automatically pulled from the official Conan Center.
|
||||
In the case we switch to a newer version of a dependency that still requires a
|
||||
patch or add a new dependency, it will be necessary for you to pull in the changes and re-export the
|
||||
updated dependencies with the newer version. However, if we switch to a newer
|
||||
version that no longer requires a patch, no action is required on your part, as
|
||||
the new recipe will be automatically pulled from the official Conan Center.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!NOTE]
|
||||
> You might need to add `--lockfile=""` to your `conan install` command to avoid automatic use of the existing `conan.lock` file when you run `conan export` manually on your machine
|
||||
> You might need to add `--lockfile=""` to your `conan install` command
|
||||
> to avoid automatic use of the existing `conan.lock` file when you run
|
||||
> `conan export` manually on your machine
|
||||
>
|
||||
> This is not recommended though, as you might end up using different revisions of recipes.
|
||||
|
||||
@@ -74,7 +88,8 @@ Possible values are ['5.0', '5.1', '6.0', '6.1', '7.0', '7.3', '8.0', '8.1',
|
||||
Read "http://docs.conan.io/2/knowledge/faq.html#error-invalid-setting"
|
||||
```
|
||||
|
||||
you need to create `$(conan config home)/settings_user.yml` file if it doesn't exist and add the required version number(s) to the `version` array specific for your compiler. For example:
|
||||
you need to create `$(conan config home)/settings_user.yml` file if it doesn't exist and add the required version number(s)
|
||||
to the `version` array specific for your compiler. For example:
|
||||
|
||||
```yaml
|
||||
compiler:
|
||||
@@ -84,9 +99,13 @@ compiler:
|
||||
|
||||
### Multiple compilers
|
||||
|
||||
If you have multiple compilers installed, make sure to select the one to use in your default Conan configuration **before** running `conan profile detect`, by setting the `CC` and `CXX` environment variables.
|
||||
If you have multiple compilers installed, make sure to select the one to use in
|
||||
your default Conan configuration **before** running `conan profile detect`, by
|
||||
setting the `CC` and `CXX` environment variables.
|
||||
|
||||
For example, if you are running MacOS and have [homebrew LLVM@18](https://formulae.brew.sh/formula/llvm@18), and want to use it as a compiler in the new Conan profile:
|
||||
For example, if you are running MacOS and have [homebrew
|
||||
LLVM@18](https://formulae.brew.sh/formula/llvm@18), and want to use it as a
|
||||
compiler in the new Conan profile:
|
||||
|
||||
```bash
|
||||
export CC=$(brew --prefix llvm@18)/bin/clang
|
||||
@@ -94,7 +113,9 @@ export CXX=$(brew --prefix llvm@18)/bin/clang++
|
||||
conan profile detect
|
||||
```
|
||||
|
||||
You should also explicitly set the path to the compiler in the profile file, which helps to avoid errors when `CC` and/or `CXX` are set and disagree with the selected Conan profile. For example:
|
||||
You should also explicitly set the path to the compiler in the profile file,
|
||||
which helps to avoid errors when `CC` and/or `CXX` are set and disagree with the
|
||||
selected Conan profile. For example:
|
||||
|
||||
```text
|
||||
[conf]
|
||||
@@ -103,11 +124,15 @@ tools.build:compiler_executables={'c':'/usr/bin/gcc','cpp':'/usr/bin/g++'}
|
||||
|
||||
### Multiple profiles
|
||||
|
||||
You can manage multiple Conan profiles in the directory `$(conan config home)/profiles`, for example renaming `default` to a different name and then creating a new `default` profile for a different compiler.
|
||||
You can manage multiple Conan profiles in the directory
|
||||
`$(conan config home)/profiles`, for example renaming `default` to a different
|
||||
name and then creating a new `default` profile for a different compiler.
|
||||
|
||||
### Select language
|
||||
|
||||
The default profile created by Conan will typically select different C++ dialect than C++23 used by this project. You should set `23` in the profile line starting with `compiler.cppstd=`. For example:
|
||||
The default profile created by Conan will typically select different C++ dialect
|
||||
than C++23 used by this project. You should set `23` in the profile line
|
||||
starting with `compiler.cppstd=`. For example:
|
||||
|
||||
```bash
|
||||
sed -i.bak -e 's|^compiler\.cppstd=.*$|compiler.cppstd=23|' $(conan config home)/profiles/default
|
||||
@@ -115,7 +140,10 @@ sed -i.bak -e 's|^compiler\.cppstd=.*$|compiler.cppstd=23|' $(conan config home)
|
||||
|
||||
### Select standard library in Linux
|
||||
|
||||
**Linux** developers will commonly have a default Conan [profile][] that compiles with GCC and links with libstdc++. If you are linking with libstdc++ (see profile setting `compiler.libcxx`), then you will need to choose the `libstdc++11` ABI:
|
||||
**Linux** developers will commonly have a default Conan [profile][] that
|
||||
compiles with GCC and links with libstdc++. If you are linking with libstdc++
|
||||
(see profile setting `compiler.libcxx`), then you will need to choose the
|
||||
`libstdc++11` ABI:
|
||||
|
||||
```bash
|
||||
sed -i.bak -e 's|^compiler\.libcxx=.*$|compiler.libcxx=libstdc++11|' $(conan config home)/profiles/default
|
||||
@@ -123,9 +151,12 @@ sed -i.bak -e 's|^compiler\.libcxx=.*$|compiler.libcxx=libstdc++11|' $(conan con
|
||||
|
||||
### Select architecture and runtime in Windows
|
||||
|
||||
**Windows** developers may need to use the x64 native build tools. An easy way to do that is to run the shortcut "x64 Native Tools Command Prompt" for the version of Visual Studio that you have installed.
|
||||
**Windows** developers may need to use the x64 native build tools. An easy way
|
||||
to do that is to run the shortcut "x64 Native Tools Command Prompt" for the
|
||||
version of Visual Studio that you have installed.
|
||||
|
||||
Windows developers must also build `xrpld` and its dependencies for the x64 architecture:
|
||||
Windows developers must also build `xrpld` and its dependencies for the x64
|
||||
architecture:
|
||||
|
||||
```bash
|
||||
sed -i.bak -e 's|^arch=.*$|arch=x86_64|' $(conan config home)/profiles/default
|
||||
@@ -144,8 +175,10 @@ If you want to experiment with a new package, follow these steps:
|
||||
1. Search for the package on [Conan Center](https://conan.io/center/).
|
||||
2. Modify [`conanfile.py`](../../conanfile.py):
|
||||
- Add a version of the package to the `requires` property.
|
||||
- Change any default options for the package by adding them to the `default_options` property (with syntax `'$package:$option': $value`).
|
||||
3. Regenerate the [Conan lockfile](../../conan/lockfile/README.md) so the new dependency is captured:
|
||||
- Change any default options for the package by adding them to the
|
||||
`default_options` property (with syntax `'$package:$option': $value`).
|
||||
3. Regenerate the [Conan lockfile](../../conan/lockfile/README.md) so the new
|
||||
dependency is captured:
|
||||
|
||||
```bash
|
||||
./conan/lockfile/regenerate.sh
|
||||
@@ -153,7 +186,8 @@ If you want to experiment with a new package, follow these steps:
|
||||
|
||||
4. Modify [`CMakeLists.txt`](../../CMakeLists.txt):
|
||||
- Add a call to `find_package($package REQUIRED)`.
|
||||
- Link a library from the package to the target `xrpl_libs` (search for the existing call to `target_link_libraries(xrpl_libs INTERFACE ...)`).
|
||||
- Link a library from the package to the target `xrpl_libs`
|
||||
(search for the existing call to `target_link_libraries(xrpl_libs INTERFACE ...)`).
|
||||
5. Start coding! Don't forget to include whatever headers you need from the package.
|
||||
|
||||
[profile]: https://docs.conan.io/2/reference/config_files/profiles.html
|
||||
Vendored
+87
-16
@@ -1,44 +1,115 @@
|
||||
## A crash course in CMake and Conan
|
||||
|
||||
To better understand how to use Conan, we should first understand _why_ we use Conan, and to understand that, we need to understand how we use CMake.
|
||||
To better understand how to use Conan,
|
||||
we should first understand _why_ we use Conan,
|
||||
and to understand that,
|
||||
we need to understand how we use CMake.
|
||||
|
||||
### CMake
|
||||
|
||||
Technically, you don't need CMake to build this project. You could manually compile every translation unit into an object file, using the right compiler options, and then manually link all those objects together, using the right linker options. However, that is very tedious and error-prone, which is why we lean on tools like CMake.
|
||||
Technically, you don't need CMake to build this project.
|
||||
You could manually compile every translation unit into an object file,
|
||||
using the right compiler options,
|
||||
and then manually link all those objects together,
|
||||
using the right linker options.
|
||||
However, that is very tedious and error-prone,
|
||||
which is why we lean on tools like CMake.
|
||||
|
||||
We have written CMake configuration files ([`CMakeLists.txt`](./CMakeLists.txt) and friends) for this project so that CMake can be used to correctly compile and link all of the translation units in it. Or rather, CMake will generate files for a separate build system (e.g. Make, Ninja, Visual Studio, Xcode, etc.) that compile and link all of the translation units. Even then, CMake has parameters, some of which are platform-specific. In CMake's parlance, parameters are specially-named **variables** like [`CMAKE_BUILD_TYPE`][build_type] or [`CMAKE_MSVC_RUNTIME_LIBRARY`][runtime]. Parameters include:
|
||||
We have written CMake configuration files
|
||||
([`CMakeLists.txt`](./CMakeLists.txt) and friends)
|
||||
for this project so that CMake can be used to correctly compile and link
|
||||
all of the translation units in it.
|
||||
Or rather, CMake will generate files for a separate build system
|
||||
(e.g. Make, Ninja, Visual Studio, Xcode, etc.)
|
||||
that compile and link all of the translation units.
|
||||
Even then, CMake has parameters, some of which are platform-specific.
|
||||
In CMake's parlance, parameters are specially-named **variables** like
|
||||
[`CMAKE_BUILD_TYPE`][build_type] or
|
||||
[`CMAKE_MSVC_RUNTIME_LIBRARY`][runtime].
|
||||
Parameters include:
|
||||
|
||||
- what build system to generate files for
|
||||
- where to find the compiler and linker
|
||||
- where to find dependencies, e.g. libraries and headers
|
||||
- how to link dependencies, e.g. any special compiler or linker flags that need to be used with them, including preprocessor definitions
|
||||
- how to compile translation units, e.g. with optimizations, debug symbols, position-independent code, etc.
|
||||
- how to link dependencies, e.g. any special compiler or linker flags that
|
||||
need to be used with them, including preprocessor definitions
|
||||
- how to compile translation units, e.g. with optimizations, debug symbols,
|
||||
position-independent code, etc.
|
||||
- on Windows, which runtime library to link with
|
||||
|
||||
For some of these parameters, like the build system and compiler, CMake goes through a complicated search process to choose default values. For others, like the dependencies, _we_ had written in the CMake configuration files of this project our own complicated process to choose defaults. For most developers, things "just worked"... until they didn't, and then you were left trying to debug one of these complicated processes, instead of choosing and manually passing the parameter values yourself.
|
||||
For some of these parameters, like the build system and compiler,
|
||||
CMake goes through a complicated search process to choose default values.
|
||||
For others, like the dependencies,
|
||||
_we_ had written in the CMake configuration files of this project
|
||||
our own complicated process to choose defaults.
|
||||
For most developers, things "just worked"... until they didn't, and then
|
||||
you were left trying to debug one of these complicated processes, instead of
|
||||
choosing and manually passing the parameter values yourself.
|
||||
|
||||
You can pass every parameter to CMake on the command line, but writing out these parameters every time we want to configure CMake is a pain. Most humans prefer to put them into a configuration file, once, that CMake can read every time it is configured. For CMake, that file is a [toolchain file][toolchain].
|
||||
You can pass every parameter to CMake on the command line,
|
||||
but writing out these parameters every time we want to configure CMake is
|
||||
a pain.
|
||||
Most humans prefer to put them into a configuration file, once, that
|
||||
CMake can read every time it is configured.
|
||||
For CMake, that file is a [toolchain file][toolchain].
|
||||
|
||||
### Conan
|
||||
|
||||
These next few paragraphs on Conan are going to read much like the ones above for CMake.
|
||||
These next few paragraphs on Conan are going to read much like the ones above
|
||||
for CMake.
|
||||
|
||||
Technically, you don't need Conan to build this project. You could manually download, configure, build, and install all of the dependencies yourself, and then pass all of the parameters necessary for CMake to link to those dependencies. To guarantee ABI compatibility, you must be sure to use the same set of compiler and linker options for all dependencies _and_ this project. However, that is very tedious and error-prone, which is why we lean on tools like Conan.
|
||||
Technically, you don't need Conan to build this project.
|
||||
You could manually download, configure, build, and install all of the
|
||||
dependencies yourself, and then pass all of the parameters necessary for
|
||||
CMake to link to those dependencies.
|
||||
To guarantee ABI compatibility, you must be sure to use the same set of
|
||||
compiler and linker options for all dependencies _and_ this project.
|
||||
However, that is very tedious and error-prone, which is why we lean on tools
|
||||
like Conan.
|
||||
|
||||
We have written a Conan configuration file ([`conanfile.py`](../../conanfile.py)) so that Conan can be used to correctly download, configure, build, and install all of the dependencies for this project, using a single set of compiler and linker options for all of them. It generates files that contain almost all of the parameters that CMake expects. Those files include:
|
||||
We have written a Conan configuration file ([`conanfile.py`](../../conanfile.py))
|
||||
so that Conan can be used to correctly download, configure, build, and install
|
||||
all of the dependencies for this project,
|
||||
using a single set of compiler and linker options for all of them.
|
||||
It generates files that contain almost all of the parameters that CMake
|
||||
expects.
|
||||
Those files include:
|
||||
|
||||
- A single toolchain file.
|
||||
- For every dependency, a CMake [package configuration file][pcf], [package version file][pvf], and for every build type, a package targets file. Together, these files implement version checking and define `IMPORTED` targets for the dependencies.
|
||||
- For every dependency, a CMake [package configuration file][pcf],
|
||||
[package version file][pvf], and for every build type, a package
|
||||
targets file.
|
||||
Together, these files implement version checking and define `IMPORTED`
|
||||
targets for the dependencies.
|
||||
|
||||
The toolchain file itself amends the search path ([`CMAKE_PREFIX_PATH`][prefix_path]) so that [`find_package()`][find_package] will [discover][search] the generated package configuration files.
|
||||
The toolchain file itself amends the search path
|
||||
([`CMAKE_PREFIX_PATH`][prefix_path]) so that [`find_package()`][find_package]
|
||||
will [discover][search] the generated package configuration files.
|
||||
|
||||
**Nearly all we must do to properly configure CMake is pass the toolchain file.** What CMake parameters are left out? You'll still need to pick a build system generator, and if you choose a single-configuration generator, you'll need to pass the `CMAKE_BUILD_TYPE`, which should match the `build_type` setting you gave to Conan.
|
||||
**Nearly all we must do to properly configure CMake is pass the toolchain
|
||||
file.**
|
||||
What CMake parameters are left out?
|
||||
You'll still need to pick a build system generator,
|
||||
and if you choose a single-configuration generator,
|
||||
you'll need to pass the `CMAKE_BUILD_TYPE`,
|
||||
which should match the `build_type` setting you gave to Conan.
|
||||
|
||||
Even then, Conan has parameters, some of which are platform-specific. In Conan's parlance, parameters are either settings or options. **Settings** are shared by all packages, e.g. the build type. **Options** are specific to a given package, e.g. whether to build and link OpenSSL as a shared library.
|
||||
Even then, Conan has parameters, some of which are platform-specific.
|
||||
In Conan's parlance, parameters are either settings or options.
|
||||
**Settings** are shared by all packages, e.g. the build type.
|
||||
**Options** are specific to a given package, e.g. whether to build and link
|
||||
OpenSSL as a shared library.
|
||||
|
||||
For settings, Conan goes through a complicated search process to choose defaults. For options, each package recipe defines its own defaults.
|
||||
For settings, Conan goes through a complicated search process to choose
|
||||
defaults.
|
||||
For options, each package recipe defines its own defaults.
|
||||
|
||||
You can pass every parameter to Conan on the command line, but it is more convenient to put them in a configuration file, once, that Conan can read every time it is configured. For Conan, that file is a [profile][]. **All we must do to properly configure Conan is edit and pass the profile.** By default, Conan will use the profile named "default".
|
||||
You can pass every parameter to Conan on the command line,
|
||||
but it is more convenient to put them in a configuration file, once, that
|
||||
Conan can read every time it is configured.
|
||||
For Conan, that file is a [profile][].
|
||||
**All we must do to properly configure Conan is edit and pass the profile.**
|
||||
By default, Conan will use the profile named "default".
|
||||
|
||||
[build_type]: https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
|
||||
[find_package]: https://cmake.org/cmake/help/latest/command/find_package.html
|
||||
|
||||
Vendored
+16
-3
@@ -1,8 +1,12 @@
|
||||
We recommend two different methods to depend on libxrpl in your own [CMake][] project. Both methods add a CMake library target named `xrpl::libxrpl`.
|
||||
We recommend two different methods to depend on libxrpl in your own [CMake][]
|
||||
project.
|
||||
Both methods add a CMake library target named `xrpl::libxrpl`.
|
||||
|
||||
## Conan requirement
|
||||
|
||||
The first method adds libxrpl as a [Conan][] requirement. With this method, there is no need for a Git [submodule][]. It is good for when you just need a dependency on libxrpl as-is.
|
||||
The first method adds libxrpl as a [Conan][] requirement.
|
||||
With this method, there is no need for a Git [submodule][].
|
||||
It is good for when you just need a dependency on libxrpl as-is.
|
||||
|
||||
```
|
||||
# This conanfile.txt is just an example.
|
||||
@@ -45,7 +49,16 @@ cmake --build . --parallel
|
||||
|
||||
## CMake subdirectory
|
||||
|
||||
The second method adds the [xrpld][] project as a CMake [subdirectory][add_subdirectory]. This method works well when you keep the xrpld project as a Git [submodule][]. It's good for when you want to make changes to libxrpl as part of your own project. Be careful, though. Your project will inherit all of the same CMake options, so watch out for name collisions. We still recommend using [Conan][] to download, build, and connect dependencies.
|
||||
The second method adds the [xrpld][] project as a CMake
|
||||
[subdirectory][add_subdirectory].
|
||||
This method works well when you keep the xrpld project as a Git
|
||||
[submodule][].
|
||||
It's good for when you want to make changes to libxrpl as part of your own
|
||||
project.
|
||||
Be careful, though.
|
||||
Your project will inherit all of the same CMake options,
|
||||
so watch out for name collisions.
|
||||
We still recommend using [Conan][] to download, build, and connect dependencies.
|
||||
|
||||
```
|
||||
# Add the project as a Git submodule.
|
||||
|
||||
Vendored
+66
-20
@@ -1,10 +1,14 @@
|
||||
Our [build instructions][BUILD.md] assume you have a C++ development environment complete with Git, Python, Conan, CMake, and a C++ compiler. This document explains how to set one up.
|
||||
Our [build instructions][BUILD.md] assume you have a C++ development
|
||||
environment complete with Git, Python, Conan, CMake, and a C++ compiler.
|
||||
This document explains how to set one up.
|
||||
|
||||
[BUILD.md]: ../../BUILD.md
|
||||
|
||||
## Tested compiler versions
|
||||
|
||||
`xrpld` is built in the **C++23** dialect by default, so your toolchain has to support it — see [compiler support for C++23][cpp23-support]. The versions currently tested in CI are:
|
||||
`xrpld` is built in the **C++23** dialect by default, so your toolchain has to
|
||||
support it — see [compiler support for C++23][cpp23-support].
|
||||
The versions currently tested in CI are:
|
||||
|
||||
| Compiler | Version |
|
||||
| ----------- | ------------------ |
|
||||
@@ -17,9 +21,16 @@ LLVM tools (`clang-tidy` and `clang-format`) are also pinned to version 22.
|
||||
|
||||
### Older compilers
|
||||
|
||||
Older compilers may fail to build the latest `develop` code: the codebase now relies on C++23 features and has been adjusted for `clang-tidy`. If the latest code doesn't build for you, update your build toolchain first.
|
||||
Older compilers may fail to build the latest `develop` code: the codebase now
|
||||
relies on C++23 features and has been adjusted for `clang-tidy`.
|
||||
If the latest code doesn't build for you, update your build toolchain first.
|
||||
|
||||
If updating isn't an option for you, we do accept pull requests that fix builds on older compilers, as long as the change is small and doesn't make the code harder to read. What we can't promise is that older compilers will keep working: only the versions in the table above are tested in CI, and we won't hold back the use of C++23 features or add invasive workarounds to keep an untested compiler building. Treat support for anything outside the table as best-effort.
|
||||
If updating isn't an option for you, we do accept pull requests that fix builds
|
||||
on older compilers, as long as the change is small and doesn't make the code
|
||||
harder to read. What we can't promise is that older compilers will keep working:
|
||||
only the versions in the table above are tested in CI, and we won't hold back
|
||||
the use of C++23 features or add invasive workarounds to keep an untested
|
||||
compiler building. Treat support for anything outside the table as best-effort.
|
||||
|
||||
## Required tools
|
||||
|
||||
@@ -32,9 +43,11 @@ Besides a compiler, building `xrpld` requires:
|
||||
| [Conan](https://conan.io/downloads.html) | 2.17 |
|
||||
| [CMake](https://cmake.org/download/) | 3.16 |
|
||||
|
||||
On Linux and macOS, the [Nix development shell](./nix.md) provides all of them (see below). On Windows they have to be installed manually.
|
||||
On Linux and macOS, the [Nix development shell](./nix.md) provides all of them
|
||||
(see below). On Windows they have to be installed manually.
|
||||
|
||||
Building with `-Drust=ON` additionally requires a Rust toolchain, see [Rust](#rust). A default build does not, so it is not in the table above.
|
||||
Building with `-Drust=ON` additionally requires a Rust toolchain, see
|
||||
[Rust](#rust). A default build does not, so it is not in the table above.
|
||||
|
||||
Once they are in place, verify that everything is installed and runnable with:
|
||||
|
||||
@@ -44,25 +57,38 @@ Once they are in place, verify that everything is installed and runnable with:
|
||||
|
||||
## Linux and macOS
|
||||
|
||||
The **recommended way** to get a development environment on Linux and macOS is the Nix development shell. It provides the exact tooling used in CI — `git`, `python`, `conan`, `cmake`, `clang-tidy`, `clang-format`, and everything else — with a single command and without installing anything system-wide:
|
||||
The **recommended way** to get a development environment on Linux and macOS is
|
||||
the Nix development shell. It provides the exact tooling used in CI — `git`,
|
||||
`python`, `conan`, `cmake`, `clang-tidy`, `clang-format`, and everything else —
|
||||
with a single command and without installing anything system-wide:
|
||||
|
||||
```bash
|
||||
nix --experimental-features 'nix-command flakes' develop
|
||||
```
|
||||
|
||||
On **Linux**, Nix also provides the compiler (GCC); on **macOS**, it provides Clang. If you instead opt to use your system-wide Apple Clang (via `nix develop .#apple-clang`), you need to manage its version yourself (see below).
|
||||
On **Linux**, Nix also provides the compiler (GCC); on **macOS**, it provides
|
||||
Clang. If you instead opt to use your system-wide Apple Clang (via
|
||||
`nix develop .#apple-clang`), you need to manage its version yourself (see
|
||||
below).
|
||||
|
||||
See [Using the Nix development shell](./nix.md) for installation and usage details, including how to select a different compiler and why we recommend Nix over a hand-maintained environment.
|
||||
See [Using the Nix development shell](./nix.md) for installation and usage
|
||||
details, including how to select a different compiler and why we recommend Nix
|
||||
over a hand-maintained environment.
|
||||
|
||||
### macOS: managing the Apple Clang version
|
||||
|
||||
If you use your system-wide Apple Clang on macOS (via `nix develop .#apple-clang`), the compiler version is whatever your installed Xcode (or Command Line Tools) provides. The following command should return a version greater than or equal to the [tested one](#tested-compiler-versions):
|
||||
If you use your system-wide Apple Clang on macOS (via `nix develop .#apple-clang`),
|
||||
the compiler version is whatever your installed Xcode (or Command Line Tools)
|
||||
provides. The following command should return a version greater than or equal to
|
||||
the [tested one](#tested-compiler-versions):
|
||||
|
||||
```bash
|
||||
clang --version
|
||||
```
|
||||
|
||||
If you develop other applications using Xcode, you might be consistently updating to the newest version of Apple Clang, which will likely cause issues building xrpld. You may want to install and pin a specific version of Xcode:
|
||||
If you develop other applications using Xcode, you might be consistently
|
||||
updating to the newest version of Apple Clang, which will likely cause issues
|
||||
building xrpld. You may want to install and pin a specific version of Xcode:
|
||||
|
||||
1. **Download Xcode**
|
||||
- Visit [Apple Developer Downloads](https://developer.apple.com/download/more/)
|
||||
@@ -88,25 +114,45 @@ If you develop other applications using Xcode, you might be consistently updatin
|
||||
|
||||
## Windows
|
||||
|
||||
Nix is not available on Windows, so the required tools have to be installed manually:
|
||||
Nix is not available on Windows, so the required tools have to be installed
|
||||
manually:
|
||||
|
||||
- [Visual Studio 2026](https://visualstudio.microsoft.com/) with the **"Desktop development with C++"** workload — this provides MSVC and the "x64 Native Tools Command Prompt". CI configures CMake with the `Visual Studio 18 2026` generator.
|
||||
- [Visual Studio 2026](https://visualstudio.microsoft.com/) with the
|
||||
**"Desktop development with C++"** workload — this provides MSVC and the
|
||||
"x64 Native Tools Command Prompt". CI configures CMake with the
|
||||
`Visual Studio 18 2026` generator.
|
||||
- [Git for Windows](https://git-scm.com/download/win)
|
||||
- Python, Conan, and CMake, at the versions listed in [Required tools](#required-tools).
|
||||
- a [Rust toolchain](https://rustup.rs) — only needed to build with `-Drust=ON`, see [Rust](#rust)
|
||||
- Python, Conan, and CMake, at the versions listed in
|
||||
[Required tools](#required-tools).
|
||||
- a [Rust toolchain](https://rustup.rs) — only needed to build with
|
||||
`-Drust=ON`, see [Rust](#rust)
|
||||
|
||||
## Rust
|
||||
|
||||
The repository contains a Rust workspace in [`crates/`](../../crates), whose crates are exposed to C++ through [cxx](https://cxx.rs) bindings. It is **not** part of a default build: the CMake `rust` option is OFF by default, and with it off no Rust toolchain is needed. It is only required when configuring with `-Drust=ON` (which is what CI does), see [Options](../../BUILD.md#options).
|
||||
The repository contains a Rust workspace in [`crates/`](../../crates), whose
|
||||
crates are exposed to C++ through [cxx](https://cxx.rs) bindings. It is **not**
|
||||
part of a default build: the CMake `rust` option is OFF by default, and with it
|
||||
off no Rust toolchain is needed. It is only required when configuring with
|
||||
`-Drust=ON` (which is what CI does), see [Options](../../BUILD.md#options).
|
||||
|
||||
The toolchain (`cargo`, `rustc`) is pinned to the channel in [`rust-toolchain.toml`](../../rust-toolchain.toml) at the repository root. If you install Rust with [rustup](https://rustup.rs), that file is picked up automatically, and `cargo`/`rustc` in the repository will use the pinned version.
|
||||
The toolchain (`cargo`, `rustc`) is pinned to the channel in
|
||||
[`rust-toolchain.toml`](../../rust-toolchain.toml) at the repository root. If
|
||||
you install Rust with [rustup](https://rustup.rs), that file is picked up
|
||||
automatically, and `cargo`/`rustc` in the repository will use the pinned
|
||||
version.
|
||||
|
||||
Everything else the Rust build needs on the CMake side comes from Conan along with the rest of the dependencies, so there is nothing further to install.
|
||||
Everything else the Rust build needs on the CMake side comes from Conan along
|
||||
with the rest of the dependencies, so there is nothing further to install.
|
||||
|
||||
## Clang-tidy
|
||||
|
||||
`clang-tidy` is required to run static analysis checks locally (see [CONTRIBUTING.md](../../CONTRIBUTING.md)). It is not required to build the project. The version this project uses is listed in [Tested compiler versions](#tested-compiler-versions).
|
||||
`clang-tidy` is required to run static analysis checks locally (see
|
||||
[CONTRIBUTING.md](../../CONTRIBUTING.md)). It is not required to build the
|
||||
project. The version this project uses is listed in
|
||||
[Tested compiler versions](#tested-compiler-versions).
|
||||
|
||||
On Linux and macOS, the [Nix development shell](./nix.md) provides that exact version out of the box — run it via `run-clang-tidy`. No separate installation is needed.
|
||||
On Linux and macOS, the [Nix development shell](./nix.md) provides that exact
|
||||
version out of the box — run it via `run-clang-tidy`. No separate installation
|
||||
is needed.
|
||||
|
||||
[cpp23-support]: https://en.cppreference.com/w/cpp/compiler_support/23
|
||||
Vendored
+91
-24
@@ -38,10 +38,17 @@ The first time you run this command, it will take a few minutes to download and
|
||||
|
||||
### Platform notes
|
||||
|
||||
- **Linux**: `nix develop` gives you a shell with all the tooling necessary to develop xrpld and with the same GCC/glibc toolchain that Nix builds for CI. See [Choosing a different compiler](#choosing-a-different-compiler) for the custom-vs-plain toolchain trade-off.
|
||||
- **macOS**: `nix develop` gives you a full environment too, with Clang (and every other tool, including Conan) provided by Nix. To use your system-wide Apple Clang instead, enter `nix develop .#apple-clang`. Conan has no binary in the Nix cache for macOS, so it is built from source the first time you enter the shell, which makes the initial setup slower (this is handled automatically; see [`nix/devshell.nix`](../../nix/devshell.nix)).
|
||||
- **Linux**: `nix develop` gives you a shell with all the tooling necessary to develop xrpld
|
||||
and with the same GCC/glibc toolchain that Nix builds for CI.
|
||||
See [Choosing a different compiler](#choosing-a-different-compiler)
|
||||
for the custom-vs-plain toolchain trade-off.
|
||||
- **macOS**: `nix develop` gives you a full environment too, with Clang (and
|
||||
every other tool, including Conan) provided by Nix. To use your system-wide
|
||||
Apple Clang instead, enter `nix develop .#apple-clang`. Conan has no binary in
|
||||
the Nix cache for macOS, so it is built from source the first time you enter
|
||||
the shell, which makes the initial setup slower (this is handled
|
||||
automatically; see [`nix/devshell.nix`](../../nix/devshell.nix)).
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!TIP]
|
||||
> To avoid typing `--experimental-features 'nix-command flakes'` every time, you can permanently enable flakes by creating `~/.config/nix/nix.conf`:
|
||||
>
|
||||
@@ -52,7 +59,6 @@ The first time you run this command, it will take a few minutes to download and
|
||||
>
|
||||
> After this, you can simply use `nix develop` instead.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!NOTE]
|
||||
> The examples below assume you've enabled flakes in your config. If you haven't, add `--experimental-features 'nix-command flakes'` after each `nix` command.
|
||||
|
||||
@@ -60,7 +66,14 @@ The first time you run this command, it will take a few minutes to download and
|
||||
|
||||
A compiler can be chosen by providing its name with the `.#` prefix, e.g. `nix develop .#clang`.
|
||||
|
||||
On Linux, `.#gcc` and `.#clang` provide the exact toolchain CI uses: the compiler (pinned in [`nix/packages.nix`](../../nix/packages.nix)) rebuilt against the pinned custom glibc (see [`nix/linux.nix`](../../nix/linux.nix)). Building that toolchain the first time is slow unless it is fetched from a Nix binary cache. If you don't need the custom glibc, the Linux-only `.#gcc-plain` and `.#clang-plain` give you the stock nixpkgs compilers of the same versions. On macOS there is no custom glibc, so `.#gcc` and `.#clang` are already the plain nixpkgs toolchain, and the `-plain` variants do not exist.
|
||||
On Linux, `.#gcc` and `.#clang` provide the exact toolchain CI uses:
|
||||
the compiler (pinned in [`nix/packages.nix`](../../nix/packages.nix))
|
||||
rebuilt against the pinned custom glibc (see [`nix/linux.nix`](../../nix/linux.nix)).
|
||||
Building that toolchain the first time is slow unless it is fetched from a Nix binary cache.
|
||||
If you don't need the custom glibc, the Linux-only `.#gcc-plain` and `.#clang-plain`
|
||||
give you the stock nixpkgs compilers of the same versions.
|
||||
On macOS there is no custom glibc, so `.#gcc` and `.#clang` are already the plain nixpkgs toolchain,
|
||||
and the `-plain` variants do not exist.
|
||||
|
||||
Use `nix flake show` to see all the available development shells.
|
||||
|
||||
@@ -98,7 +111,6 @@ nix develop -c fish
|
||||
nix develop -c "$SHELL"
|
||||
```
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!WARNING]
|
||||
> Your shell's interactive startup files (e.g. `config.fish`, `.zshrc`) may prepend other directories — most commonly Homebrew — to `$PATH`, which can shadow the tools provided by the Nix shell. After entering, verify that tools resolve into the Nix store:
|
||||
>
|
||||
@@ -112,40 +124,80 @@ nix develop -c "$SHELL"
|
||||
|
||||
Once inside the Nix development shell, follow the standard [build instructions](../../BUILD.md#steps). The Nix shell provides all necessary tools (CMake, Ninja, Conan, etc.).
|
||||
|
||||
Coverage builds (`-Dcoverage=ON`) work in the `gcc` shell (and `gcc-plain` on Linux): each ships a `gcov` matching its compiler, since Nix's cc-wrapper does not expose one. The `clang` shells do not include `llvm-cov`, so use a `gcc` shell for coverage.
|
||||
Coverage builds (`-Dcoverage=ON`) work in the `gcc` shell (and `gcc-plain` on Linux):
|
||||
each ships a `gcov` matching its compiler, since Nix's cc-wrapper does not expose one.
|
||||
The `clang` shells do not include `llvm-cov`, so use a `gcc` shell for coverage.
|
||||
|
||||
Builds of the Rust crates (`-Drust=ON`) also work out of the box: every shell provides the Rust toolchain pinned in [`rust-toolchain.toml`](../../rust-toolchain.toml) (see [Rust](./environment.md#rust)), plus the `cargo-audit`, `cargo-llvm-cov` and `cargo-nextest` plugins.
|
||||
Builds of the Rust crates (`-Drust=ON`) also work out of the box: every shell
|
||||
provides the Rust toolchain pinned in
|
||||
[`rust-toolchain.toml`](../../rust-toolchain.toml) (see
|
||||
[Rust](./environment.md#rust)), plus the `cargo-audit`, `cargo-llvm-cov` and
|
||||
`cargo-nextest` plugins.
|
||||
|
||||
## Conan configuration
|
||||
|
||||
The shell runs [`conan/init.sh`](../../conan/init.sh) on entry, so [Set Up Conan](../../BUILD.md#set-up-conan) is already done for you. It installs into the shell's own Conan home: `CONAN_HOME=~/.conan2-nix`.
|
||||
The shell runs [`conan/init.sh`](../../conan/init.sh) on entry, so
|
||||
[Set Up Conan](../../BUILD.md#set-up-conan) is already done for you. It installs
|
||||
into the shell's own Conan home: `CONAN_HOME=~/.conan2-nix`.
|
||||
|
||||
### Prebuilt packages
|
||||
|
||||
On **Linux**, the binaries on the `xrplf` remote are built in this same Nix environment — CI runs in Docker images that bundle the dev shell's toolchain (see [`nix/docker`](../../nix/docker)) — so `.#gcc` and `.#clang` can reuse them. The `-plain` shells do not match that toolchain's glibc, so binaries from the remote are not a reliable match there.
|
||||
On **Linux**, the binaries on the `xrplf` remote are built in this same Nix
|
||||
environment — CI runs in Docker images that bundle the dev shell's toolchain (see
|
||||
[`nix/docker`](../../nix/docker)) — so `.#gcc` and `.#clang` can reuse them. The
|
||||
`-plain` shells do not match that toolchain's glibc, so binaries from the remote
|
||||
are not a reliable match there.
|
||||
|
||||
On **macOS**, CI also builds in this Nix environment, in Debug and Release (the `macos-arm64-*-nix` configurations — Debug because the profile defaults to it). The Nix build resolves to `compiler=clang`, so it gets its own package IDs, separate from the Apple Clang ones. The [dependency upload](../../.github/workflows/upload-conan-deps.yml) publishes them on pushes to `develop` and on manual runs — its nightly run rebuilds everything from source but uploads nothing — so once a set has been published `nix develop` can reuse it instead of compiling every dependency locally. These configurations run outside the reduced pull-request matrix, so label a PR `Full CI build` when it touches `flake.lock` or `nix/`.
|
||||
On **macOS**, CI also builds in this Nix environment, in Debug and Release (the
|
||||
`macos-arm64-*-nix` configurations — Debug because the profile defaults to it).
|
||||
The Nix build resolves to `compiler=clang`, so it gets its own package IDs,
|
||||
separate from the Apple Clang ones. The
|
||||
[dependency upload](../../.github/workflows/upload-conan-deps.yml) publishes them
|
||||
on pushes to `develop` and on manual runs — its nightly run rebuilds everything
|
||||
from source but uploads nothing — so once a set has been published `nix develop`
|
||||
can reuse it instead of compiling every dependency locally. These configurations
|
||||
run outside the reduced pull-request matrix, so label a PR `Full CI build` when it
|
||||
touches `flake.lock` or `nix/`.
|
||||
|
||||
To compile everything from source, add `--build '*'` to the `conan install` command.
|
||||
To compile everything from source, add `--build '*'` to the `conan install`
|
||||
command.
|
||||
|
||||
### Why the nixpkgs revision is not part of the package ID
|
||||
|
||||
A Conan package ID records the compiler and its major version, but nothing about the nixpkgs revision the toolchain came from — and `flake.lock` moves far more often than the toolchain meaningfully changes, so folding it in would rebuild every dependency on every bump for nothing.
|
||||
A Conan package ID records the compiler and its major version, but nothing about
|
||||
the nixpkgs revision the toolchain came from — and `flake.lock` moves far more
|
||||
often than the toolchain meaningfully changes, so folding it in would rebuild
|
||||
every dependency on every bump for nothing.
|
||||
|
||||
That is safe as long as no cached artifact resolves a `/nix/store` path at run time, because store paths change on every update and the old ones disappear with `nix-collect-garbage`. With the `clang` toolchain macOS CI and the dev shell use, they do not: it links against `/usr/lib/libc++` and `/usr/lib/libSystem`, and store paths reach the `.a` files only through debug info, which nothing resolves at link or run time.
|
||||
That is safe as long as no cached artifact resolves a `/nix/store` path at run
|
||||
time, because store paths change on every update and the old ones disappear with
|
||||
`nix-collect-garbage`. With the `clang` toolchain macOS CI and the dev shell use,
|
||||
they do not: it links against `/usr/lib/libc++` and `/usr/lib/libSystem`, and
|
||||
store paths reach the `.a` files only through debug info, which nothing resolves
|
||||
at link or run time.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!WARNING]
|
||||
> This does not hold for `nix develop .#gcc` on macOS. There is no system libstdc++, so GCC links its own from the store and every binary keeps a `/nix/store` reference. That shell is fine for tooling, but it is not a build configuration CI covers, and no dependency binaries are published for it.
|
||||
> This does not hold for `nix develop .#gcc` on macOS. There is no system
|
||||
> libstdc++, so GCC links its own from the store and every binary keeps a
|
||||
> `/nix/store` reference. That shell is fine for tooling, but it is not a build
|
||||
> configuration CI covers, and no dependency binaries are published for it.
|
||||
|
||||
This is checked rather than assumed. [`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) takes one file or directory and fails if a binary under it resolves a store path at run time. CI runs it over the build output and the Conan cache, and again in the upload job before anything is published. You can run it yourself:
|
||||
This is checked rather than assumed.
|
||||
[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) takes one file
|
||||
or directory and fails if a binary under it resolves a store path at run time.
|
||||
CI runs it over the build output and the Conan cache, and again in the upload job
|
||||
before anything is published. You can run it yourself:
|
||||
|
||||
```bash
|
||||
bin/check-nix-store-refs.sh build
|
||||
bin/check-nix-store-refs.sh ~/.conan2-nix
|
||||
```
|
||||
|
||||
It works on Linux too, but asserts something narrower there: the toolchain always writes the store into `PT_INTERP` and `RUNPATH`, and CI builds inside an image whose store is fixed for its lifetime, so that is fine. Only the binaries [`PatchNixBinary.cmake`](../../cmake/PatchNixBinary.cmake) retargets to the system loader have to be clean, and those are what CI checks:
|
||||
It works on Linux too, but asserts something narrower there: the toolchain always
|
||||
writes the store into `PT_INTERP` and `RUNPATH`, and CI builds inside an image
|
||||
whose store is fixed for its lifetime, so that is fine. Only the binaries
|
||||
[`PatchNixBinary.cmake`](../../cmake/PatchNixBinary.cmake) retargets to the
|
||||
system loader have to be clean, and those are what CI checks:
|
||||
|
||||
```bash
|
||||
bin/check-nix-store-refs.sh build/xrpld
|
||||
@@ -153,11 +205,22 @@ bin/check-nix-store-refs.sh build/xrpld
|
||||
|
||||
### The libresolv stub
|
||||
|
||||
This is not hypothetical: `xrpld` used to be caught by it. The c-ares package tells the linker to pass `-lresolv`, and nixpkgs keeps `libresolv` out of the macOS SDK and ships it as an ordinary store dylib — so every Nix-built `xrpld` recorded a `/nix/store/…-libresolv-93/lib/libresolv.9.dylib` load command and stopped running once that path was collected. Nothing in the link uses a single symbol from it.
|
||||
This is not hypothetical: `xrpld` used to be caught by it. The c-ares package
|
||||
tells the linker to pass `-lresolv`, and nixpkgs keeps `libresolv` out of the
|
||||
macOS SDK and ships it as an ordinary store dylib — so every Nix-built `xrpld`
|
||||
recorded a `/nix/store/…-libresolv-93/lib/libresolv.9.dylib` load command and
|
||||
stopped running once that path was collected. Nothing in the link uses a single
|
||||
symbol from it.
|
||||
|
||||
Both environments now put a stub on the linker search path (`libresolvSystemStub` in [`nix/darwin.nix`](../../nix/darwin.nix)): the same library with its install name set to `/usr/lib/libresolv.9.dylib`, which is exactly the load command the Apple Clang build records.
|
||||
Both environments now put a stub on the linker search path
|
||||
(`libresolvSystemStub` in [`nix/darwin.nix`](../../nix/darwin.nix)): the
|
||||
same library with its install name set to `/usr/lib/libresolv.9.dylib`, which is
|
||||
exactly the load command the Apple Clang build records.
|
||||
|
||||
Package IDs did not change, so Conan keeps serving anything built before the stub landed. If a binary fails to start with `Library not loaded: /nix/store/…`, see [that entry](./nix_troubleshooting.md#library-not-loaded-nixstore-from-a-binary-that-used-to-work) in the troubleshooting guide.
|
||||
Package IDs did not change, so Conan keeps serving anything built before the
|
||||
stub landed. If a binary fails to start with `Library not loaded: /nix/store/…`,
|
||||
see [that entry](./nix_troubleshooting.md#library-not-loaded-nixstore-from-a-binary-that-used-to-work)
|
||||
in the troubleshooting guide.
|
||||
|
||||
## Automatic Activation with direnv
|
||||
|
||||
@@ -170,7 +233,6 @@ The repository already ships an `.envrc` at its root that activates the Nix flak
|
||||
1. [Install direnv](https://direnv.net/docs/installation.html) and [hook it into your shell](https://direnv.net/docs/hook.html) (bash, zsh, fish, …). Installing [nix-direnv](https://github.com/nix-community/nix-direnv) as well is recommended: it caches the shell so that activation is near-instant after the first run.
|
||||
2. Run `direnv allow` once in the repository root. direnv will then load (and reload) the Nix development shell automatically whenever you enter the directory.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!NOTE]
|
||||
> direnv only caches the `.direnv` directory (already listed in `.gitignore`); no other repository files are affected.
|
||||
|
||||
@@ -180,8 +242,13 @@ To update `flake.lock` to the latest revision use `nix flake update` command.
|
||||
|
||||
## Tooling snapshots
|
||||
|
||||
The tool versions in each Nix environment are recorded in [`nix/check-tools/`](../../nix/check-tools) and verified by CI. If you change the environment (bump the CI image tag, update `flake.lock`, or edit the tool list in `bin/check-tools.sh`), CI fails until you regenerate and commit the affected snapshot — see [`nix/check-tools/README.md`](../../nix/check-tools/README.md).
|
||||
The tool versions in each Nix environment are recorded in
|
||||
[`nix/check-tools/`](../../nix/check-tools) and verified by CI. If you change the
|
||||
environment (bump the CI image tag, update `flake.lock`, or edit the tool list in
|
||||
`bin/check-tools.sh`), CI fails until you regenerate and commit the affected
|
||||
snapshot — see [`nix/check-tools/README.md`](../../nix/check-tools/README.md).
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
See [Troubleshooting Nix problems](./nix_troubleshooting.md) for common issues, such as `nix develop` failing inside Git worktrees.
|
||||
See [Troubleshooting Nix problems](./nix_troubleshooting.md) for common issues,
|
||||
such as `nix develop` failing inside Git worktrees.
|
||||
Vendored
+76
-28
@@ -1,6 +1,7 @@
|
||||
# Troubleshooting Nix problems
|
||||
|
||||
Common issues encountered when using the [Nix development shell](./nix.md), and how to resolve them.
|
||||
Common issues encountered when using the [Nix development shell](./nix.md), and
|
||||
how to resolve them.
|
||||
|
||||
## `command not found: nix` after a macOS update
|
||||
|
||||
@@ -11,7 +12,8 @@ $ nix develop
|
||||
zsh: command not found: nix
|
||||
```
|
||||
|
||||
then Nix is almost certainly still installed — only the shell hook that puts it on your `PATH` is gone. Confirm that first:
|
||||
then Nix is almost certainly still installed — only the shell hook that puts it
|
||||
on your `PATH` is gone. Confirm that first:
|
||||
|
||||
```bash
|
||||
ls -l /nix/var/nix/profiles/default/bin/nix
|
||||
@@ -21,7 +23,8 @@ If that exists, the installation is fine and this is purely a `PATH` problem.
|
||||
|
||||
### Why it happens
|
||||
|
||||
The installer does not touch your dotfiles. Instead it sources a setup script from the Nix store by editing **system-wide** rc files:
|
||||
The installer does not touch your dotfiles. Instead it sources a setup script
|
||||
from the Nix store by editing **system-wide** rc files:
|
||||
|
||||
| Shell | File the installer edits |
|
||||
| ----- | ------------------------------------- |
|
||||
@@ -29,13 +32,17 @@ The installer does not touch your dotfiles. Instead it sources a setup script fr
|
||||
| zsh | `/etc/zshrc` |
|
||||
| fish | `$__fish_sysconf_dir/conf.d/nix.fish` |
|
||||
|
||||
macOS manages `/etc/zshrc`, so an OS update can replace it with the vendor copy and silently drop the Nix block. `/etc/bashrc` and the fish file usually survive, which is why the breakage often shows up in zsh only. You can verify this by diffing against the backup the installer left behind:
|
||||
macOS manages `/etc/zshrc`, so an OS update can replace it with the vendor copy
|
||||
and silently drop the Nix block. `/etc/bashrc` and the fish file usually survive,
|
||||
which is why the breakage often shows up in zsh only. You can verify this by
|
||||
diffing against the backup the installer left behind:
|
||||
|
||||
```bash
|
||||
diff /etc/zshrc /etc/zshrc.backup-before-nix
|
||||
```
|
||||
|
||||
If they are identical, the Nix snippet was wiped. This is upstream issue [NixOS/nix#3616](https://github.com/NixOS/nix/issues/3616).
|
||||
If they are identical, the Nix snippet was wiped. This is upstream issue
|
||||
[NixOS/nix#3616](https://github.com/NixOS/nix/issues/3616).
|
||||
|
||||
### Fix
|
||||
|
||||
@@ -45,7 +52,8 @@ To unblock the current shell:
|
||||
. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh
|
||||
```
|
||||
|
||||
For a permanent fix, add the snippet to your **user** rc file rather than restoring `/etc/zshrc` — user dotfiles are not clobbered by OS updates:
|
||||
For a permanent fix, add the snippet to your **user** rc file rather than
|
||||
restoring `/etc/zshrc` — user dotfiles are not clobbered by OS updates:
|
||||
|
||||
```bash
|
||||
cat >>~/.zshrc <<'EOF'
|
||||
@@ -58,11 +66,14 @@ fi
|
||||
EOF
|
||||
```
|
||||
|
||||
The scripts guard against double-sourcing via `__ETC_PROFILE_NIX_SOURCED`, so this is safe even if a system-wide hook is later restored.
|
||||
The scripts guard against double-sourcing via `__ETC_PROFILE_NIX_SOURCED`, so
|
||||
this is safe even if a system-wide hook is later restored.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!NOTE]
|
||||
> `/etc/zshrc` and `~/.zshrc` are only read by **interactive** zsh. If the snippet is present but `zsh -c '…'`, a script, or an IDE terminal still can't find `nix`, that shell is non-interactive — put the snippet in `~/.zshenv` instead.
|
||||
> `/etc/zshrc` and `~/.zshrc` are only read by **interactive** zsh. If the
|
||||
> snippet is present but `zsh -c '…'`, a script, or an IDE terminal still can't
|
||||
> find `nix`, that shell is non-interactive — put the snippet in `~/.zshenv`
|
||||
> instead.
|
||||
|
||||
## Git worktrees
|
||||
|
||||
@@ -75,39 +86,56 @@ error:
|
||||
error: opening Git repository "/path/to/rippled": unsupported extension name extensions.relativeworktrees (libgit2 error code = 6)
|
||||
```
|
||||
|
||||
then your Nix is linked against a libgit2 older than **1.9.4**. Git 2.48+ writes the `extensions.relativeWorktrees` config entry when a worktree is created with relative paths (`git worktree add --relative-paths`, or with `worktree.useRelativePaths=true`), and older libgit2 versions refuse to open a repository that uses it. Nix uses libgit2 to read the flake, so evaluation fails.
|
||||
then your Nix is linked against a libgit2 older than **1.9.4**. Git 2.48+ writes
|
||||
the `extensions.relativeWorktrees` config entry when a worktree is created with
|
||||
relative paths (`git worktree add --relative-paths`, or with
|
||||
`worktree.useRelativePaths=true`), and older libgit2 versions refuse to open a
|
||||
repository that uses it. Nix uses libgit2 to read the flake, so evaluation
|
||||
fails.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!IMPORTANT]
|
||||
> This entry is written to the **shared** repository config, so once any relative worktree exists, `nix develop` fails in the main checkout too — not just inside the worktree.
|
||||
> This entry is written to the **shared** repository config, so once any
|
||||
> relative worktree exists, `nix develop` fails in the main checkout too — not
|
||||
> just inside the worktree.
|
||||
|
||||
### Workarounds
|
||||
|
||||
These work today, with any Nix version:
|
||||
|
||||
- bypass libgit2 with a `path:` flakeref: `nix develop "path:$PWD"` (note: this copies the working tree to the store and ignores `.gitignore`); or
|
||||
- bypass libgit2 with a `path:` flakeref: `nix develop "path:$PWD"`
|
||||
(note: this copies the working tree to the store and ignores `.gitignore`); or
|
||||
- create worktrees with absolute paths (omit `--relative-paths`); or
|
||||
- clear the extension if you don't need relative worktrees: `git config --unset extensions.relativeWorktrees`.
|
||||
- clear the extension if you don't need relative worktrees:
|
||||
`git config --unset extensions.relativeWorktrees`.
|
||||
|
||||
### Permanent fix
|
||||
|
||||
The fix is in [libgit2 1.9.4](https://github.com/libgit2/libgit2/releases/tag/v1.9.4), so the real solution is a Nix that links against libgit2 `1.9.4` or newer. Check which version yours links against:
|
||||
The fix is in [libgit2 1.9.4](https://github.com/libgit2/libgit2/releases/tag/v1.9.4),
|
||||
so the real solution is a Nix that links against libgit2 `1.9.4` or newer. Check
|
||||
which version yours links against:
|
||||
|
||||
```bash
|
||||
nix-store -qR "$(readlink -f "$(command -v nix)")" | grep libgit2
|
||||
```
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!WARNING]
|
||||
> `nix upgrade-nix` does **not** help yet. It installs the build from the official [`nix-fallback-paths`](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/installer/tools/nix-fallback-paths.nix), which is still linked against libgit2 `1.9.2` — there is no new upstream Nix release with the fix. (On some systems that build is even the exact store path you already have, making the upgrade a no-op.)
|
||||
> `nix upgrade-nix` does **not** help yet. It installs the build from the
|
||||
> official [`nix-fallback-paths`](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/installer/tools/nix-fallback-paths.nix),
|
||||
> which is still linked against libgit2 `1.9.2` — there is no new upstream Nix
|
||||
> release with the fix. (On some systems that build is even the exact store path
|
||||
> you already have, making the upgrade a no-op.)
|
||||
|
||||
nixpkgs has already rebuilt Nix against the fixed libgit2 (e.g. `nix-2.34.7+1`), so the cleanest path is to reinstall Nix using your usual installation method once it picks up that rebuild, then re-run the `grep libgit2` check above to confirm it reports `1.9.4` or newer.
|
||||
nixpkgs has already rebuilt Nix against the fixed libgit2 (e.g. `nix-2.34.7+1`),
|
||||
so the cleanest path is to reinstall Nix using your usual installation method
|
||||
once it picks up that rebuild, then re-run the `grep libgit2` check above to
|
||||
confirm it reports `1.9.4` or newer.
|
||||
|
||||
Until then, prefer the workarounds above.
|
||||
|
||||
## `wint_t` / `uint32_t` errors from the Nix libc++ headers
|
||||
|
||||
A build that mixes the Nix toolchain with the system SDK fails in libc++ itself, with errors that look nothing like your code:
|
||||
A build that mixes the Nix toolchain with the system SDK fails in libc++ itself,
|
||||
with errors that look nothing like your code:
|
||||
|
||||
```
|
||||
/nix/store/...-libcxx-.../include/c++/v1/cwchar:136:9: error: target of using declaration conflicts with declaration already in scope
|
||||
@@ -117,31 +145,41 @@ A build that mixes the Nix toolchain with the system SDK fails in libc++ itself,
|
||||
error: use of undeclared identifier 'UINT32_C'
|
||||
```
|
||||
|
||||
The give-away is the second path: Nix's libc++ headers are being combined with the **Xcode Command Line Tools** SDK instead of the Nix one.
|
||||
The give-away is the second path: Nix's libc++ headers are being combined with
|
||||
the **Xcode Command Line Tools** SDK instead of the Nix one.
|
||||
|
||||
### Why it happens
|
||||
|
||||
`SDKROOT` and `DEVELOPER_DIR` are what point the toolchain at the Nix SDK, and they are not baked into the compiler — a dev shell gets them from the `apple-sdk` setup hook. CMake, finding neither, asks `xcrun`, which answers with the system SDK. Nix's `libc++` and Apple's headers then declare the same types twice.
|
||||
`SDKROOT` and `DEVELOPER_DIR` are what point the toolchain at the Nix SDK, and
|
||||
they are not baked into the compiler — a dev shell gets them from the
|
||||
`apple-sdk` setup hook. CMake, finding neither, asks `xcrun`, which answers with
|
||||
the system SDK. Nix's `libc++` and Apple's headers then declare the same types
|
||||
twice.
|
||||
|
||||
### Fix
|
||||
|
||||
Run the build from inside the dev shell (`nix develop`), or from an environment that exports both variables. To confirm which SDK a configured build is using:
|
||||
Run the build from inside the dev shell (`nix develop`), or from an environment
|
||||
that exports both variables. To confirm which SDK a configured build is using:
|
||||
|
||||
```bash
|
||||
grep -o '\-isysroot [^ ]*' build/compile_commands.json | sort -u
|
||||
```
|
||||
|
||||
It should print a `/nix/store/...-apple-sdk-*` path. If it prints `/Library/Developer/CommandLineTools/...`, re-configure from within the shell — CMake caches the sysroot, so an existing `build/` directory keeps the wrong one.
|
||||
It should print a `/nix/store/...-apple-sdk-*` path. If it prints
|
||||
`/Library/Developer/CommandLineTools/...`, re-configure from within the shell —
|
||||
CMake caches the sysroot, so an existing `build/` directory keeps the wrong one.
|
||||
|
||||
## `Library not loaded: /nix/store/…` from a binary that used to work
|
||||
|
||||
A binary stops starting after a `nix flake update`, or after `nix-collect-garbage` removes the paths the previous toolchain used:
|
||||
A binary stops starting after a `nix flake update`, or after
|
||||
`nix-collect-garbage` removes the paths the previous toolchain used:
|
||||
|
||||
```
|
||||
dyld[57271]: Library not loaded: /nix/store/…-libresolv-93/lib/libresolv.9.dylib
|
||||
```
|
||||
|
||||
[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) finds the same thing without having to run anything, and names the file:
|
||||
[`bin/check-nix-store-refs.sh`](../../bin/check-nix-store-refs.sh) finds the same
|
||||
thing without having to run anything, and names the file:
|
||||
|
||||
```
|
||||
$ bin/check-nix-store-refs.sh ~/.conan2-nix
|
||||
@@ -151,7 +189,9 @@ $ bin/check-nix-store-refs.sh ~/.conan2-nix
|
||||
/Users/you/.conan2-nix: checked 135, skipped 2495, 1 with Nix store references.
|
||||
```
|
||||
|
||||
Conan's cache folders are named after a truncated package name plus a hash, so ask Conan which package the offending one belongs to — pass the folder holding the hash, not the file itself:
|
||||
Conan's cache folders are named after a truncated package name plus a hash, so
|
||||
ask Conan which package the offending one belongs to — pass the folder holding
|
||||
the hash, not the file itself:
|
||||
|
||||
```
|
||||
$ conan cache ref ~/.conan2-nix/p/b/c-area24ded30c388c
|
||||
@@ -160,9 +200,17 @@ c-ares/1.34.6#545240bb1c40e2cacd4362d6b8967650:dab5992496abe6d219defb7986ecbf367
|
||||
|
||||
### Why it happens
|
||||
|
||||
The binary records a store path that no longer exists. Nothing we build should: see [Prebuilt packages](./nix.md#prebuilt-packages) for why, and `libresolvSystemStub` in [`nix/darwin.nix`](../../nix/darwin.nix) for the one dependency that needed help to comply.
|
||||
The binary records a store path that no longer exists. Nothing we build should:
|
||||
see [Prebuilt packages](./nix.md#prebuilt-packages) for why, and
|
||||
`libresolvSystemStub` in [`nix/darwin.nix`](../../nix/darwin.nix) for the one
|
||||
dependency that needed help to comply.
|
||||
|
||||
A Conan package ID does not encode the nixpkgs revision, so a package built before that stub existed stays in your local cache and keeps being reused. The dev shell is also what tends to produce one: it is a slightly _less_ isolated build environment than CI's, because `mkShell` puts every tool's headers and libraries on the compiler's search path — which is how c-ares found the Nix `libresolv` in the first place.
|
||||
A Conan package ID does not encode the nixpkgs revision, so a package built
|
||||
before that stub existed stays in your local cache and keeps being reused. The
|
||||
dev shell is also what tends to produce one: it is a slightly _less_ isolated
|
||||
build environment than CI's, because `mkShell` puts every tool's headers and
|
||||
libraries on the compiler's search path — which is how c-ares found the Nix
|
||||
`libresolv` in the first place.
|
||||
|
||||
### Fix
|
||||
|
||||
|
||||
Vendored
+6
-5
@@ -1,10 +1,11 @@
|
||||
# Sanitizer Configuration for Xrpld
|
||||
|
||||
This document explains how to properly configure and run sanitizers (`AddressSanitizer`, `UndefinedBehaviorSanitizer`, `ThreadSanitizer`) with the xrpld project. Corresponding suppression files are located in the `sanitizers/suppressions` directory.
|
||||
This document explains how to properly configure and run sanitizers (`AddressSanitizer`, `UndefinedBehaviorSanitizer`, `ThreadSanitizer`) with the xrpld project.
|
||||
Corresponding suppression files are located in the `sanitizers/suppressions` directory.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!CAUTION]
|
||||
> Do not mix Address and Thread sanitizers - they are incompatible. Also, we don't yet support MSVC sanitizers, so this is only for Clang/GCC builds.
|
||||
> Do not mix Address and Thread sanitizers - they are incompatible.
|
||||
> Also, we don't yet support MSVC sanitizers, so this is only for Clang/GCC builds.
|
||||
|
||||
- [Sanitizer Configuration for Xrpld](#sanitizer-configuration-for-xrpld)
|
||||
- [Building with Sanitizers](#building-with-sanitizers)
|
||||
@@ -34,10 +35,10 @@ This document explains how to properly configure and run sanitizers (`AddressSan
|
||||
Follow the same instructions as mentioned in [BUILD.md](../../BUILD.md) but with the following changes:
|
||||
|
||||
1. Make sure you have a clean build directory.
|
||||
2. Set the `SANITIZERS` environment variable before calling `conan install`. Only set it once. Example: `export SANITIZERS=address,undefinedbehavior`
|
||||
2. Set the `SANITIZERS` environment variable before calling `conan install`. Only set it once.
|
||||
Example: `export SANITIZERS=address,undefinedbehavior`
|
||||
3. Use `--profile:all sanitizers` with Conan to build dependencies with sanitizer instrumentation.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!NOTE]
|
||||
> Building with sanitizer-instrumented dependencies is slower but produces fewer false positives.
|
||||
|
||||
|
||||
+327
-73
@@ -2,116 +2,277 @@
|
||||
|
||||
**This section is a work in progress!!**
|
||||
|
||||
Consensus is the task of reaching agreement within a distributed system in the presence of faulty or even malicious participants. This document outlines the [XRP Ledger Consensus Algorithm](https://arxiv.org/abs/1802.07242) as implemented in [xrpld](https://github.com/XRPLF/rippled), but focuses on its utility as a generic consensus algorithm independent of the detailed mechanics of the XRPL consensus Ledger. Most notably, the algorithm does not require fully synchronous communication between all nodes in the network, or even a fixed network topology, but instead achieves consensus via collectively trusted subnetworks.
|
||||
Consensus is the task of reaching agreement within a distributed system in the
|
||||
presence of faulty or even malicious participants. This document outlines the
|
||||
[XRP Ledger Consensus Algorithm](https://arxiv.org/abs/1802.07242)
|
||||
as implemented in [xrpld](https://github.com/XRPLF/rippled), but
|
||||
focuses on its utility as a generic consensus algorithm independent of the
|
||||
detailed mechanics of the XRPL consensus Ledger. Most notably, the algorithm
|
||||
does not require fully synchronous communication between all nodes in the
|
||||
network, or even a fixed network topology, but instead achieves consensus via
|
||||
collectively trusted subnetworks.
|
||||
|
||||
## Distributed Agreement
|
||||
|
||||
A challenge for distributed systems is reaching agreement on changes in shared state. For the XRPL network, the shared state is the current ledger--account information, account balances, order books and other financial data. We will refer to shared distributed state as a /ledger/ throughout the remainder of this document.
|
||||
A challenge for distributed systems is reaching agreement on changes in shared
|
||||
state. For the XRPL network, the shared state is the current ledger--account
|
||||
information, account balances, order books and other financial data. We will
|
||||
refer to shared distributed state as a /ledger/ throughout the remainder of this
|
||||
document.
|
||||
|
||||

|
||||
|
||||
As shown above, new ledgers are made by applying a set of transactions to the prior ledger. For the XRPL network, transactions include payments, modification of account settings, updates to offers and more.
|
||||
As shown above, new ledgers are made by applying a set of transactions to the
|
||||
prior ledger. For the XRPL network, transactions include payments,
|
||||
modification of account settings, updates to offers and more.
|
||||
|
||||
In a centralized system, generating the next ledger is trivial since there is a single unique arbiter of which transactions to include and how to apply them to a ledger. For decentralized systems, participants must resolve disagreements on the set of transactions to include, the order to apply those transactions, and even the resulting ledger after applying the transactions. This is even more difficult when some participants are faulty or malicious.
|
||||
In a centralized system, generating the next ledger is trivial since there is a
|
||||
single unique arbiter of which transactions to include and how to apply them to
|
||||
a ledger. For decentralized systems, participants must resolve disagreements on
|
||||
the set of transactions to include, the order to apply those transactions, and
|
||||
even the resulting ledger after applying the transactions. This is even more
|
||||
difficult when some participants are faulty or malicious.
|
||||
|
||||
The XRPL network is a decentralized and **trust-full** network. Anyone is free to join and participants are free to choose a subset of peers that are collectively trusted to not collude in an attempt to defraud the participant. Leveraging this network of trust, the XRPL algorithm has two main components.
|
||||
The XRPL network is a decentralized and **trust-full** network. Anyone is free
|
||||
to join and participants are free to choose a subset of peers that are
|
||||
collectively trusted to not collude in an attempt to defraud the participant.
|
||||
Leveraging this network of trust, the XRPL algorithm has two main components.
|
||||
|
||||
- _Consensus_ in which network participants agree on the transactions to apply to a prior ledger, based on the positions of their chosen peers.
|
||||
- _Validation_ in which network participants agree on what ledger was generated, based on the ledgers generated by chosen peers.
|
||||
- _Consensus_ in which network participants agree on the transactions to apply
|
||||
to a prior ledger, based on the positions of their chosen peers.
|
||||
- _Validation_ in which network participants agree on what ledger was
|
||||
generated, based on the ledgers generated by chosen peers.
|
||||
|
||||
These phases are continually repeated to process transactions submitted to the network, generating successive ledgers and giving rise to the blockchain ledger history depicted below. In this diagram, time is flowing to the right, but links between ledgers point backward to the parent. Also note the alternate Ledger 2 that was generated by some participants, but which failed validation and was abandoned.
|
||||
These phases are continually repeated to process transactions submitted to the
|
||||
network, generating successive ledgers and giving rise to the blockchain ledger
|
||||
history depicted below. In this diagram, time is flowing to the right, but
|
||||
links between ledgers point backward to the parent. Also note the alternate
|
||||
Ledger 2 that was generated by some participants, but which failed validation
|
||||
and was abandoned.
|
||||
|
||||

|
||||
|
||||
The remainder of this section describes the Consensus and Validation algorithms in more detail and is meant as a companion guide to understanding the generic implementation in `xrpld`. The document **does not** discuss correctness, fault-tolerance or liveness properties of the algorithms or the full details of how they integrate within `xrpld` to support the XRPL consensus Ledger.
|
||||
The remainder of this section describes the Consensus and Validation algorithms
|
||||
in more detail and is meant as a companion guide to understanding the generic
|
||||
implementation in `xrpld`. The document **does not** discuss correctness,
|
||||
fault-tolerance or liveness properties of the algorithms or the full details of
|
||||
how they integrate within `xrpld` to support the XRPL consensus Ledger.
|
||||
|
||||
## Consensus Overview
|
||||
|
||||
### Definitions
|
||||
|
||||
- The _ledger_ is the shared distributed state. Each ledger has a unique ID to distinguish it from all other ledgers. During consensus, the _previous_, _prior_ or _last-closed_ ledger is the most recent ledger seen by consensus and is the basis upon which it will build the next ledger.
|
||||
- A _transaction_ is an instruction for an atomic change in the ledger state. A unique ID distinguishes a transaction from other transactions.
|
||||
- A _transaction set_ is a set of transactions under consideration by consensus. The goal of consensus is to reach agreement on this set. The generic consensus algorithm does not rely on an ordering of transactions within the set, nor does it specify how to apply a transaction set to a ledger to generate a new ledger. A unique ID distinguishes a set of transactions from all other sets of transactions.
|
||||
- A _node_ is one of the distributed actors running the consensus algorithm. It has a unique ID to distinguish it from all other nodes.
|
||||
- A _peer_ of a node is another node that it has chosen to follow and which it believes will not collude with other chosen peers. The choice of peers is not symmetric, since participants can decide on their chosen sets independently.
|
||||
- A /position/ is the current belief of the next ledger's transaction set and close time. Position can refer to the node's own position or the position of a peer.
|
||||
- A _proposal_ is one of a sequence of positions a node shares during consensus. An initial proposal contains the starting position taken by a node before it considers any peer positions. If a node subsequently updates its position in response to its peers, it will issue an updated proposal. A proposal is uniquely identified by the ID of the proposing node, the ID of the position taken, the ID of the prior ledger the proposal is for, and the sequence number of the proposal.
|
||||
- A _dispute_ is a transaction that is either not part of a node's position or not in a peer's position. During consensus, the node will add or remove disputed transactions from its position based on that transaction's support amongst its peers.
|
||||
- The _ledger_ is the shared distributed state. Each ledger has a unique ID to
|
||||
distinguish it from all other ledgers. During consensus, the _previous_,
|
||||
_prior_ or _last-closed_ ledger is the most recent ledger seen by consensus
|
||||
and is the basis upon which it will build the next ledger.
|
||||
- A _transaction_ is an instruction for an atomic change in the ledger state. A
|
||||
unique ID distinguishes a transaction from other transactions.
|
||||
- A _transaction set_ is a set of transactions under consideration by consensus.
|
||||
The goal of consensus is to reach agreement on this set. The generic
|
||||
consensus algorithm does not rely on an ordering of transactions within the
|
||||
set, nor does it specify how to apply a transaction set to a ledger to
|
||||
generate a new ledger. A unique ID distinguishes a set of transactions from
|
||||
all other sets of transactions.
|
||||
- A _node_ is one of the distributed actors running the consensus algorithm. It
|
||||
has a unique ID to distinguish it from all other nodes.
|
||||
- A _peer_ of a node is another node that it has chosen to follow and which it
|
||||
believes will not collude with other chosen peers. The choice of peers is not
|
||||
symmetric, since participants can decide on their chosen sets independently.
|
||||
- A /position/ is the current belief of the next ledger's transaction set and
|
||||
close time. Position can refer to the node's own position or the position of a
|
||||
peer.
|
||||
- A _proposal_ is one of a sequence of positions a node shares during consensus.
|
||||
An initial proposal contains the starting position taken by a node before it
|
||||
considers any peer positions. If a node subsequently updates its position in
|
||||
response to its peers, it will issue an updated proposal. A proposal is
|
||||
uniquely identified by the ID of the proposing node, the ID of the position
|
||||
taken, the ID of the prior ledger the proposal is for, and the sequence number
|
||||
of the proposal.
|
||||
- A _dispute_ is a transaction that is either not part of a node's position or
|
||||
not in a peer's position. During consensus, the node will add or remove
|
||||
disputed transactions from its position based on that transaction's support
|
||||
amongst its peers.
|
||||
|
||||
Note that most types have an ID as a lightweight identifier of instances of that type. Consensus often operates on the IDs directly since the underlying type is potentially expensive to share over the network. For example, proposal's only contain the ID of the position of a peer. Since many peers likely have the same position, this reduces the need to send the full transaction set multiple times. Instead, a node can request the transaction set from the network if necessary.
|
||||
Note that most types have an ID as a lightweight identifier of instances of that
|
||||
type. Consensus often operates on the IDs directly since the underlying type is
|
||||
potentially expensive to share over the network. For example, proposal's only
|
||||
contain the ID of the position of a peer. Since many peers likely have the same
|
||||
position, this reduces the need to send the full transaction set multiple times.
|
||||
Instead, a node can request the transaction set from the network if necessary.
|
||||
|
||||
### Overview
|
||||
|
||||

|
||||
|
||||
The diagram above is an overview of the consensus process from the perspective of a single participant. Recall that during a single consensus round, a node is trying to agree with its peers on which transactions to apply to its prior ledger when generating the next ledger. It also attempts to agree on the [network time when the ledger closed](#effective_close_time). There are 3 main phases to a consensus round:
|
||||
The diagram above is an overview of the consensus process from the perspective
|
||||
of a single participant. Recall that during a single consensus round, a node is
|
||||
trying to agree with its peers on which transactions to apply to its prior
|
||||
ledger when generating the next ledger. It also attempts to agree on the
|
||||
[network time when the ledger closed](#effective_close_time). There are
|
||||
3 main phases to a consensus round:
|
||||
|
||||
- A call to `startRound` places the node in the `Open` phase. In this phase, the node is waiting for transactions to include in its open ledger.
|
||||
- At some point, the node will `Close` the open ledger and transition to the `Establish` phase. In this phase, the node shares/receives peer proposals on which transactions should be accepted in the closed ledger.
|
||||
- At some point, the node determines it has reached consensus with its peers on which transactions to include. It transitions to the `Accept` phase. In this phase, the node works on applying the transactions to the prior ledger to generate a new closed ledger. Once the new ledger is completed, the node shares the validated ledger hash with the network and makes a call to `startRound` to start the cycle again for the next ledger.
|
||||
- A call to `startRound` places the node in the `Open` phase. In this phase,
|
||||
the node is waiting for transactions to include in its open ledger.
|
||||
- At some point, the node will `Close` the open ledger and transition to the
|
||||
`Establish` phase. In this phase, the node shares/receives peer proposals on
|
||||
which transactions should be accepted in the closed ledger.
|
||||
- At some point, the node determines it has reached consensus with its peers on
|
||||
which transactions to include. It transitions to the `Accept` phase. In this
|
||||
phase, the node works on applying the transactions to the prior ledger to
|
||||
generate a new closed ledger. Once the new ledger is completed, the node shares
|
||||
the validated ledger hash with the network and makes a call to `startRound` to
|
||||
start the cycle again for the next ledger.
|
||||
|
||||
Throughout, a heartbeat timer calls `timerEntry` at a regular frequency to drive the process forward. Although the `startRound` call occurs at arbitrary times based on when the initial round began and the time it takes to apply transactions, the transitions from `Open` to `Establish` and `Establish` to `Accept` only occur during calls to `timerEntry`. Similarly, transactions can arrive at arbitrary times, independent of the heartbeat timer. Transactions received after the `Open` to `Close` transition and not part of peer proposals won't be considered until the next consensus round. They are represented above by the light green triangles.
|
||||
Throughout, a heartbeat timer calls `timerEntry` at a regular frequency to drive
|
||||
the process forward. Although the `startRound` call occurs at arbitrary times
|
||||
based on when the initial round began and the time it takes to apply
|
||||
transactions, the transitions from `Open` to `Establish` and `Establish` to
|
||||
`Accept` only occur during calls to `timerEntry`. Similarly, transactions can
|
||||
arrive at arbitrary times, independent of the heartbeat timer. Transactions
|
||||
received after the `Open` to `Close` transition and not part of peer proposals
|
||||
won't be considered until the next consensus round. They are represented above
|
||||
by the light green triangles.
|
||||
|
||||
Peer proposals are issued by a node during a `timerEntry` call, but since peers do not synchronize `timerEntry` calls, they are received by other peers at arbitrary times. Peer proposals are only considered if received prior to the `Establish` to `Accept` transition, and only if the peer is working on the same prior ledger. Peer proposals received after consensus is reached will not be meaningful and are represented above by the circle with the X in it. Only proposals from chosen peers are considered.
|
||||
Peer proposals are issued by a node during a `timerEntry` call, but since peers
|
||||
do not synchronize `timerEntry` calls, they are received by other peers at
|
||||
arbitrary times. Peer proposals are only considered if received prior to the
|
||||
`Establish` to `Accept` transition, and only if the peer is working on the same
|
||||
prior ledger. Peer proposals received after consensus is reached will not be
|
||||
meaningful and are represented above by the circle with the X in it. Only
|
||||
proposals from chosen peers are considered.
|
||||
|
||||
### Effective Close Time ### {#effective_close_time}
|
||||
|
||||
In addition to agreeing on a transaction set, each consensus round tries to agree on the time the ledger closed. Each node calculates its own close time when it closes the open ledger. This exact close time is rounded to the nearest multiple of the current _effective close time resolution_. It is this _effective close time_ that nodes seek to agree on. This allows servers to derive a common time for a ledger without the need for perfectly synchronized clocks. As depicted below, the 3 pink arrows represent exact close times from 3 consensus nodes that round to the same effective close time given the current resolution. The purple arrow represents a peer whose estimate rounds to a different effective close time given the current resolution.
|
||||
In addition to agreeing on a transaction set, each consensus round tries to
|
||||
agree on the time the ledger closed. Each node calculates its own close time
|
||||
when it closes the open ledger. This exact close time is rounded to the nearest
|
||||
multiple of the current _effective close time resolution_. It is this
|
||||
_effective close time_ that nodes seek to agree on. This allows servers to
|
||||
derive a common time for a ledger without the need for perfectly synchronized
|
||||
clocks. As depicted below, the 3 pink arrows represent exact close times from 3
|
||||
consensus nodes that round to the same effective close time given the current
|
||||
resolution. The purple arrow represents a peer whose estimate rounds to a
|
||||
different effective close time given the current resolution.
|
||||
|
||||

|
||||
|
||||
The effective close time is part of the node's position and is shared with peers in its proposals. Just like the position on the consensus transaction set, a node will update its close time position in response to its peers' effective close time positions. Peers can agree to disagree on the close time, in which case the effective close time is taken as 1 second past the prior close.
|
||||
The effective close time is part of the node's position and is shared with peers
|
||||
in its proposals. Just like the position on the consensus transaction set, a
|
||||
node will update its close time position in response to its peers' effective
|
||||
close time positions. Peers can agree to disagree on the close time, in which
|
||||
case the effective close time is taken as 1 second past the prior close.
|
||||
|
||||
The close time resolution is itself dynamic, decreasing (coarser) resolution in subsequent consensus rounds if nodes are unable to reach consensus on an effective close time and increasing (finer) resolution if nodes consistently reach close time consensus.
|
||||
The close time resolution is itself dynamic, decreasing (coarser) resolution in
|
||||
subsequent consensus rounds if nodes are unable to reach consensus on an
|
||||
effective close time and increasing (finer) resolution if nodes consistently
|
||||
reach close time consensus.
|
||||
|
||||
### Modes
|
||||
|
||||
Internally, a node operates under one of the following consensus modes. Either of the first two modes may be chosen when a consensus round starts.
|
||||
Internally, a node operates under one of the following consensus modes. Either
|
||||
of the first two modes may be chosen when a consensus round starts.
|
||||
|
||||
- _Proposing_ indicates the node is a full-fledged consensus participant. It takes on positions and sends proposals to its peers.
|
||||
- _Observing_ indicates the node is a passive consensus participant. It maintains a position internally, but does not propose that position to its peers. Instead, it receives peer proposals and updates its position to track the majority of its peers. This may be preferred if the node is only being used to track the state of the network or during a start-up phase while it is still synchronizing with the network.
|
||||
- _Proposing_ indicates the node is a full-fledged consensus participant. It
|
||||
takes on positions and sends proposals to its peers.
|
||||
- _Observing_ indicates the node is a passive consensus participant. It
|
||||
maintains a position internally, but does not propose that position to its
|
||||
peers. Instead, it receives peer proposals and updates its position
|
||||
to track the majority of its peers. This may be preferred if the node is only
|
||||
being used to track the state of the network or during a start-up phase while
|
||||
it is still synchronizing with the network.
|
||||
|
||||
The other two modes are set internally during the consensus round when the node believes it is no longer working on the dominant ledger chain based on peer validations. It checks this on every call to `timerEntry`.
|
||||
The other two modes are set internally during the consensus round when the node
|
||||
believes it is no longer working on the dominant ledger chain based on peer
|
||||
validations. It checks this on every call to `timerEntry`.
|
||||
|
||||
- _Wrong Ledger_ indicates the node is not working on the correct prior ledger and does not have it available. It requests that ledger from the network, but continues to work towards consensus this round while waiting. If it had been _proposing_, it will send a special "bow-out" proposal to its peers to indicate its change in mode for the rest of this round. For the duration of the round, it defers to peer positions for determining the consensus outcome as if it were just _observing_.
|
||||
- _Switch Ledger_ indicates that the node has acquired the correct prior ledger from the network. Although it now has the correct prior ledger, the fact that it had the wrong one at some point during this round means it is likely behind and should defer to peer positions for determining the consensus outcome.
|
||||
- _Wrong Ledger_ indicates the node is not working on the correct prior ledger
|
||||
and does not have it available. It requests that ledger from the network, but
|
||||
continues to work towards consensus this round while waiting. If it had been
|
||||
_proposing_, it will send a special "bow-out" proposal to its peers to indicate
|
||||
its change in mode for the rest of this round. For the duration of the round,
|
||||
it defers to peer positions for determining the consensus outcome as if it
|
||||
were just _observing_.
|
||||
- _Switch Ledger_ indicates that the node has acquired the correct prior ledger
|
||||
from the network. Although it now has the correct prior ledger, the fact that
|
||||
it had the wrong one at some point during this round means it is likely behind
|
||||
and should defer to peer positions for determining the consensus outcome.
|
||||
|
||||

|
||||
|
||||
Once either wrong ledger or switch ledger are reached, the node cannot return to proposing or observing until the next consensus round. However, the node could change its view of the correct prior ledger, so going from switch ledger to wrong ledger and back again is possible.
|
||||
Once either wrong ledger or switch ledger are reached, the node cannot
|
||||
return to proposing or observing until the next consensus round. However,
|
||||
the node could change its view of the correct prior ledger, so going from
|
||||
switch ledger to wrong ledger and back again is possible.
|
||||
|
||||
The distinction between the wrong and switched ledger modes arises because a ledger's unique identifier may be known by a node before the ledger itself. This reflects that fact that the data corresponding to a ledger may be large and take time to share over the network, whereas the smaller ID could be shared in a peer validation much more quickly. Distinguishing the two states allows the node to decide how best to generate the next ledger once it declares consensus.
|
||||
The distinction between the wrong and switched ledger modes arises because a
|
||||
ledger's unique identifier may be known by a node before the ledger itself. This
|
||||
reflects that fact that the data corresponding to a ledger may be large and take
|
||||
time to share over the network, whereas the smaller ID could be shared in a peer
|
||||
validation much more quickly. Distinguishing the two states allows the node to
|
||||
decide how best to generate the next ledger once it declares consensus.
|
||||
|
||||
### Phases
|
||||
|
||||
As depicted in the overview diagram, consensus is best viewed as a progression through 3 phases. There are 4 public methods of the generic consensus algorithm that determine this progression
|
||||
As depicted in the overview diagram, consensus is best viewed as a progression
|
||||
through 3 phases. There are 4 public methods of the generic consensus algorithm
|
||||
that determine this progression
|
||||
|
||||
- `startRound` begins a consensus round.
|
||||
- `timerEntry` is called at a regular frequency (`LEDGER_MIN_CLOSE`) and is the only call to consensus that can change the phase from `Open` to `Establish` or `Accept`.
|
||||
- `peerProposal` is called whenever a peer proposal is received and is what allows a node to update its position in a subsequent `timerEntry` call.
|
||||
- `gotTxSet` is called when a transaction set is received from the network. This is typically in response to a prior request from the node to acquire the transaction set corresponding to a disagreeing peer's position.
|
||||
- `timerEntry` is called at a regular frequency (`LEDGER_MIN_CLOSE`) and is the
|
||||
only call to consensus that can change the phase from `Open` to `Establish`
|
||||
or `Accept`.
|
||||
- `peerProposal` is called whenever a peer proposal is received and is what
|
||||
allows a node to update its position in a subsequent `timerEntry` call.
|
||||
- `gotTxSet` is called when a transaction set is received from the network. This
|
||||
is typically in response to a prior request from the node to acquire the
|
||||
transaction set corresponding to a disagreeing peer's position.
|
||||
|
||||
The following subsections describe each consensus phase in more detail and what actions are taken in response to these calls.
|
||||
The following subsections describe each consensus phase in more detail and what
|
||||
actions are taken in response to these calls.
|
||||
|
||||
#### Open
|
||||
|
||||
The `Open` phase is a quiescent period to allow transactions to build up in the node's open ledger. The duration is a trade-off between latency and throughput. A shorter window reduces the latency to generating the next ledger, but also reduces transaction throughput due to fewer transactions accepted into the ledger.
|
||||
The `Open` phase is a quiescent period to allow transactions to build up in the
|
||||
node's open ledger. The duration is a trade-off between latency and throughput.
|
||||
A shorter window reduces the latency to generating the next ledger, but also
|
||||
reduces transaction throughput due to fewer transactions accepted into the
|
||||
ledger.
|
||||
|
||||
A call to `startRound` would forcibly begin the next consensus round, skipping completion of the current round. This is not expected during normal operation. Calls to `peerProposal` or `gotTxSet` simply store the proposal or transaction set for use in the coming `Establish` phase.
|
||||
A call to `startRound` would forcibly begin the next consensus round, skipping
|
||||
completion of the current round. This is not expected during normal operation.
|
||||
Calls to `peerProposal` or `gotTxSet` simply store the proposal or transaction
|
||||
set for use in the coming `Establish` phase.
|
||||
|
||||
A call to `timerEntry` first checks that the node is working on the correct prior ledger. If not, it will update the mode and request the correct ledger. Otherwise, the node checks whether to switch to the `Establish` phase and close the ledger.
|
||||
A call to `timerEntry` first checks that the node is working on the correct
|
||||
prior ledger. If not, it will update the mode and request the correct ledger.
|
||||
Otherwise, the node checks whether to switch to the `Establish` phase and close
|
||||
the ledger.
|
||||
|
||||
##### Ledger Close
|
||||
|
||||
Under normal circumstances, the open ledger period ends when one of the following is true
|
||||
Under normal circumstances, the open ledger period ends when one of the following
|
||||
is true
|
||||
|
||||
- if there are transactions in the open ledger and more than `LEDGER_MIN_CLOSE` have elapsed. This is the typical behavior.
|
||||
- if there are no open transactions and a suitably longer idle interval has elapsed. This increases the opportunity to get some transaction into the next ledger and avoids doing useless work closing an empty ledger.
|
||||
- if more than half the number of prior round peers have already closed or finished this round. This indicates the node is falling behind and needs to catch up.
|
||||
- if there are transactions in the open ledger and more than `LEDGER_MIN_CLOSE`
|
||||
have elapsed. This is the typical behavior.
|
||||
- if there are no open transactions and a suitably longer idle interval has
|
||||
elapsed. This increases the opportunity to get some transaction into
|
||||
the next ledger and avoids doing useless work closing an empty ledger.
|
||||
- if more than half the number of prior round peers have already closed or finished
|
||||
this round. This indicates the node is falling behind and needs to catch up.
|
||||
|
||||
When closing the ledger, the node takes its initial position based on the transactions in the open ledger and uses the current time as its initial close time estimate. If in the proposing mode, the node shares its initial position with peers. Now that the node has taken a position, it will consider any peer positions for this round that arrived earlier. The node generates disputed transactions for each transaction not in common with a peer's position. The node also records the vote of each peer for each disputed transaction.
|
||||
When closing the ledger, the node takes its initial position based on the
|
||||
transactions in the open ledger and uses the current time as
|
||||
its initial close time estimate. If in the proposing mode, the node shares its
|
||||
initial position with peers. Now that the node has taken a position, it will
|
||||
consider any peer positions for this round that arrived earlier. The node
|
||||
generates disputed transactions for each transaction not in common with a peer's
|
||||
position. The node also records the vote of each peer for each disputed
|
||||
transaction.
|
||||
|
||||
In the example below, we suppose our node has closed with transactions 1,2 and 3. It creates disputes for transactions 2,3 and 4, since at least one peer position differs on each.
|
||||
In the example below, we suppose our node has closed with transactions 1,2 and 3. It creates disputes
|
||||
for transactions 2,3 and 4, since at least one peer position differs on each.
|
||||
|
||||
##### disputes ##### {#disputes_image}
|
||||
|
||||
@@ -119,55 +280,119 @@ In the example below, we suppose our node has closed with transactions 1,2 and 3
|
||||
|
||||
#### Establish
|
||||
|
||||
The establish phase is the active period of consensus in which the node exchanges proposals with peers in an attempt to reach agreement on the consensus transactions and effective close time.
|
||||
The establish phase is the active period of consensus in which the node
|
||||
exchanges proposals with peers in an attempt to reach agreement on the consensus
|
||||
transactions and effective close time.
|
||||
|
||||
A call to `startRound` would forcibly begin the next consensus round, skipping completion of the current round. This is not expected during normal operation. Calls to `peerProposal` or `gotTxSet` that reflect new positions will generate disputed transactions for any new disagreements and will update the peer's vote for all disputed transactions.
|
||||
A call to `startRound` would forcibly begin the next consensus round, skipping
|
||||
completion of the current round. This is not expected during normal operation.
|
||||
Calls to `peerProposal` or `gotTxSet` that reflect new positions will generate
|
||||
disputed transactions for any new disagreements and will update the peer's vote
|
||||
for all disputed transactions.
|
||||
|
||||
A call to `timerEntry` first checks that the node is working from the correct prior ledger. If not, the node will update the mode and request the correct ledger. Otherwise, the node updates the node's position and considers whether to switch to the `Accepted` phase and declare consensus reached. However, at least `LEDGER_MIN_CONSENSUS` time must have elapsed before doing either. This allows peers an opportunity to take an initial position and share it.
|
||||
A call to `timerEntry` first checks that the node is working from the correct
|
||||
prior ledger. If not, the node will update the mode and request the correct
|
||||
ledger. Otherwise, the node updates the node's position and considers whether
|
||||
to switch to the `Accepted` phase and declare consensus reached. However, at
|
||||
least `LEDGER_MIN_CONSENSUS` time must have elapsed before doing either. This
|
||||
allows peers an opportunity to take an initial position and share it.
|
||||
|
||||
##### Update Position
|
||||
|
||||
In order to achieve consensus, the node is looking for a transaction set that is supported by a super-majority of peers. The node works towards this set by adding or removing disputed transactions from its position based on an increasing threshold for inclusion.
|
||||
In order to achieve consensus, the node is looking for a transaction set that is
|
||||
supported by a super-majority of peers. The node works towards this set by
|
||||
adding or removing disputed transactions from its position based on an
|
||||
increasing threshold for inclusion.
|
||||
|
||||

|
||||
|
||||
By starting with a lower threshold, a node initially allows a wide set of transactions into its position. If the establish round continues and the node is "stuck", a higher threshold can focus on accepting transactions with the most support. The constants that define the thresholds and durations at which the thresholds change are given by `AV_XXX_CONSENSUS_PCT` and `AV_XXX_CONSENSUS_TIME` respectively, where `XXX` is `INIT`,`MID`,`LATE` and `STUCK`. The effective close time position is updated using the same thresholds.
|
||||
By starting with a lower threshold, a node initially allows a wide set of
|
||||
transactions into its position. If the establish round continues and the node is
|
||||
"stuck", a higher threshold can focus on accepting transactions with the most
|
||||
support. The constants that define the thresholds and durations at which the
|
||||
thresholds change are given by `AV_XXX_CONSENSUS_PCT` and
|
||||
`AV_XXX_CONSENSUS_TIME` respectively, where `XXX` is `INIT`,`MID`,`LATE` and
|
||||
`STUCK`. The effective close time position is updated using the same
|
||||
thresholds.
|
||||
|
||||
Given the [example disputes above](#disputes_image) and an initial threshold of 50%, our node would retain its position since transaction 1 was not in dispute and transactions 2 and 3 have 75% support. Since its position did not change, it would not need to send a new proposal to peers. Peer C would not change either. Peer A would add transaction 3 to its position and Peer B would remove transaction 4 from its position; both would then send an updated position.
|
||||
Given the [example disputes above](#disputes_image) and an initial threshold
|
||||
of 50%, our node would retain its position since transaction 1 was not in
|
||||
dispute and transactions 2 and 3 have 75% support. Since its position did not
|
||||
change, it would not need to send a new proposal to peers. Peer C would not
|
||||
change either. Peer A would add transaction 3 to its position and Peer B would
|
||||
remove transaction 4 from its position; both would then send an updated
|
||||
position.
|
||||
|
||||
Conversely, if the diagram reflected a later call to =timerEntry= that occurs in the stuck region with a threshold of say 95%, our node would remove transactions 2 and 3 from its candidate set and send an updated position. Likewise, all the other peers would end up with only transaction 1 in their position.
|
||||
Conversely, if the diagram reflected a later call to =timerEntry= that occurs in
|
||||
the stuck region with a threshold of say 95%, our node would remove transactions
|
||||
2 and 3 from its candidate set and send an updated position. Likewise, all the
|
||||
other peers would end up with only transaction 1 in their position.
|
||||
|
||||
Lastly, if our node were not in the proposing mode, it would not include its own vote and just take the majority (>50%) position of its peers. In this example, our node would maintain its position of transactions 1, 2 and 3.
|
||||
Lastly, if our node were not in the proposing mode, it would not include its own
|
||||
vote and just take the majority (>50%) position of its peers. In this example,
|
||||
our node would maintain its position of transactions 1, 2 and 3.
|
||||
|
||||
##### Checking Consensus
|
||||
|
||||
After updating its position, the node checks for supermajority agreement with its peers on its current position. This agreement is of the exact transaction set, not just the support of individual transactions. That is, if our position is a subset of a peer's position, that counts as a disagreement. Also recall that effective close time agreement allows a supermajority of participants agreeing to disagree.
|
||||
After updating its position, the node checks for supermajority agreement with
|
||||
its peers on its current position. This agreement is of the exact transaction
|
||||
set, not just the support of individual transactions. That is, if our position
|
||||
is a subset of a peer's position, that counts as a disagreement. Also recall
|
||||
that effective close time agreement allows a supermajority of participants
|
||||
agreeing to disagree.
|
||||
|
||||
Consensus is declared when the following 3 clauses are true:
|
||||
|
||||
- `LEDGER_MIN_CONSENSUS` time has elapsed in the establish phase
|
||||
- At least 75% of the prior round proposers have proposed OR this establish phase is `LEDGER_MIN_CONSENSUS` longer than the last round's establish phase
|
||||
- At least 75% of the prior round proposers have proposed OR this establish
|
||||
phase is `LEDGER_MIN_CONSENSUS` longer than the last round's establish phase
|
||||
- `minimumConsensusPercentage` of ourself and our peers share the same position
|
||||
|
||||
The middle condition ensures slower peers have a chance to share positions, but prevents waiting too long on peers that have disconnected. Additionally, a node can declare that consensus has moved on if `minimumConsensusPercentage` peers have sent validations and moved on to the next ledger. This outcome indicates the node has fallen behind its peers and needs to catch up.
|
||||
The middle condition ensures slower peers have a chance to share positions, but
|
||||
prevents waiting too long on peers that have disconnected. Additionally, a node
|
||||
can declare that consensus has moved on if `minimumConsensusPercentage` peers
|
||||
have sent validations and moved on to the next ledger. This outcome indicates
|
||||
the node has fallen behind its peers and needs to catch up.
|
||||
|
||||
If a node is not proposing, it does not include its own position when calculating the percent of agreeing participants but otherwise follows the above logic.
|
||||
If a node is not proposing, it does not include its own position when
|
||||
calculating the percent of agreeing participants but otherwise follows the above
|
||||
logic.
|
||||
|
||||
##### Accepting Consensus
|
||||
|
||||
Once consensus is reached (or moved on), the node switches to the `Accept` phase and signals to the implementing code that the round is complete. That code is responsible for using the consensus transaction set to generate the next ledger and calling `startRound` to begin the next round. The implementation has total freedom on ordering transactions, deciding what to do if consensus moved on, determining whether to retry or abandon local transactions that did not make the consensus set and updating any internal state based on the consensus progress.
|
||||
Once consensus is reached (or moved on), the node switches to the `Accept` phase
|
||||
and signals to the implementing code that the round is complete. That code is
|
||||
responsible for using the consensus transaction set to generate the next ledger
|
||||
and calling `startRound` to begin the next round. The implementation has total
|
||||
freedom on ordering transactions, deciding what to do if consensus moved on,
|
||||
determining whether to retry or abandon local transactions that did not make the
|
||||
consensus set and updating any internal state based on the consensus progress.
|
||||
|
||||
#### Accept
|
||||
|
||||
The `Accept` phase is the terminal phase of the consensus algorithm. Calls to `timerEntry`, `peerProposal` and `gotTxSet` will not change the internal consensus state while in the accept phase. The expectation is that the application specific code is working to generate the new ledger based on the consensus outcome. Once complete, that code should make a call to `startRound` to kick off the next consensus round. The `startRound` call includes the new prior ledger, prior ledger ID and whether the round should begin in the proposing or observing mode. After setting some initial state, the phase transitions to `Open`. The node will also check if the provided prior ledger and ID are correct, updating the mode and requesting the proper ledger from the network if necessary.
|
||||
The `Accept` phase is the terminal phase of the consensus algorithm. Calls to
|
||||
`timerEntry`, `peerProposal` and `gotTxSet` will not change the internal
|
||||
consensus state while in the accept phase. The expectation is that the
|
||||
application specific code is working to generate the new ledger based on the
|
||||
consensus outcome. Once complete, that code should make a call to `startRound`
|
||||
to kick off the next consensus round. The `startRound` call includes the new
|
||||
prior ledger, prior ledger ID and whether the round should begin in the
|
||||
proposing or observing mode. After setting some initial state, the phase
|
||||
transitions to `Open`. The node will also check if the provided prior ledger
|
||||
and ID are correct, updating the mode and requesting the proper ledger from the
|
||||
network if necessary.
|
||||
|
||||
## Consensus Type Requirements
|
||||
|
||||
The consensus type requirements are given below as minimal implementation stubs. Actual implementations would augment these stubs with members appropriate for managing the details of transactions and ledgers within the larger application framework.
|
||||
The consensus type requirements are given below as minimal implementation stubs.
|
||||
Actual implementations would augment these stubs with members appropriate for
|
||||
managing the details of transactions and ledgers within the larger application
|
||||
framework.
|
||||
|
||||
### Transaction
|
||||
|
||||
The transaction type `Tx` encapsulates a single transaction under consideration by consensus.
|
||||
The transaction type `Tx` encapsulates a single transaction under consideration
|
||||
by consensus.
|
||||
|
||||
```{.cpp}
|
||||
struct Tx
|
||||
@@ -181,7 +406,10 @@ struct Tx
|
||||
|
||||
### Transaction Set
|
||||
|
||||
The transaction set type `TxSet` represents a set of `Tx`s that are collectively under consideration by consensus. A `TxSet` can be compared against other `TxSet`s (typically from peers) and can be modified to add or remove transactions via the mutable subtype.
|
||||
The transaction set type `TxSet` represents a set of `Tx`s that are collectively
|
||||
under consideration by consensus. A `TxSet` can be compared against other `TxSet`s
|
||||
(typically from peers) and can be modified to add or remove transactions via
|
||||
the mutable subtype.
|
||||
|
||||
```{.cpp}
|
||||
struct TxSet
|
||||
@@ -218,7 +446,12 @@ struct TxSet
|
||||
|
||||
### Ledger
|
||||
|
||||
The `Ledger` type represents the state shared amongst the distributed participants. Notice that the details of how the next ledger is generated from the prior ledger and the consensus accepted transaction set is not part of the interface. Within the generic code, this type is primarily used to know that peers are working on the same tip of the ledger chain and to provide some basic timing data for consensus.
|
||||
The `Ledger` type represents the state shared amongst the
|
||||
distributed participants. Notice that the details of how the next ledger is
|
||||
generated from the prior ledger and the consensus accepted transaction set is
|
||||
not part of the interface. Within the generic code, this type is primarily used
|
||||
to know that peers are working on the same tip of the ledger chain and to
|
||||
provide some basic timing data for consensus.
|
||||
|
||||
```{.cpp}
|
||||
struct Ledger
|
||||
@@ -252,7 +485,9 @@ struct Ledger
|
||||
|
||||
### PeerProposal
|
||||
|
||||
The `PeerProposal` type represents the signed position taken by a peer during consensus. The only type requirement is owning an instance of a generic `ConsensusProposal`.
|
||||
The `PeerProposal` type represents the signed position taken
|
||||
by a peer during consensus. The only type requirement is owning an instance of a
|
||||
generic `ConsensusProposal`.
|
||||
|
||||
```{.cpp}
|
||||
// Represents our proposed position or a peer's proposed position
|
||||
@@ -273,7 +508,11 @@ struct PeerPosition
|
||||
|
||||
### Generic Consensus Interface
|
||||
|
||||
The generic `Consensus` relies on `Adaptor` template class to implement a set of helper functions that plug the consensus algorithm into a specific application. The `Adaptor` class also defines the types above needed by the algorithm. Below are excerpts of the generic consensus implementation and of helper types that will interact with the concrete implementing class.
|
||||
The generic `Consensus` relies on `Adaptor` template class to implement a set
|
||||
of helper functions that plug the consensus algorithm into a specific application.
|
||||
The `Adaptor` class also defines the types above needed by the algorithm. Below
|
||||
are excerpts of the generic consensus implementation and of helper types that will
|
||||
interact with the concrete implementing class.
|
||||
|
||||
```{.cpp}
|
||||
// Represents a transaction under dispute this round
|
||||
@@ -414,11 +653,26 @@ struct Adaptor
|
||||
};
|
||||
```
|
||||
|
||||
The implementing class hides many details of the peer communication model from the generic code.
|
||||
The implementing class hides many details of the peer communication
|
||||
model from the generic code.
|
||||
|
||||
- The `share` member functions are responsible for sharing the given type with a node's peers, but are agnostic to the mechanism. Ideally, messages are delivered faster than `LEDGER_GRANULARITY`.
|
||||
- The generic code does not specify how transactions are submitted by clients, propagated through the network or stored in the open ledger. Indeed, the open ledger is only conceptual from the perspective of the generic code---the initial position and transaction set are opaquely generated in a `Consensus::Result` instance returned from the `onClose` callback.
|
||||
- The calls to `acquireLedger` and `acquireTxSet` only have non-trivial return if the ledger or transaction set of interest is available. The implementing class is free to block while acquiring, or return the empty option while servicing the request asynchronously. Due to legacy reasons, the two calls are not symmetric. `acquireTxSet` requires the host application to call `gotTxSet` when an asynchronous `acquire` completes. Conversely, `acquireLedger` will be called again later by the consensus code if it still desires the ledger with the hope that the asynchronous acquisition is complete.
|
||||
- The `share` member functions are responsible for sharing the given type with a
|
||||
node's peers, but are agnostic to the mechanism. Ideally, messages are delivered
|
||||
faster than `LEDGER_GRANULARITY`.
|
||||
- The generic code does not specify how transactions are submitted by clients,
|
||||
propagated through the network or stored in the open ledger. Indeed, the open
|
||||
ledger is only conceptual from the perspective of the generic code---the
|
||||
initial position and transaction set are opaquely generated in a
|
||||
`Consensus::Result` instance returned from the `onClose` callback.
|
||||
- The calls to `acquireLedger` and `acquireTxSet` only have non-trivial return
|
||||
if the ledger or transaction set of interest is available. The implementing
|
||||
class is free to block while acquiring, or return the empty option while
|
||||
servicing the request asynchronously. Due to legacy reasons, the two calls
|
||||
are not symmetric. `acquireTxSet` requires the host application to call
|
||||
`gotTxSet` when an asynchronous `acquire` completes. Conversely,
|
||||
`acquireLedger` will be called again later by the consensus code if it still
|
||||
desires the ledger with the hope that the asynchronous acquisition is
|
||||
complete.
|
||||
|
||||
## Validation
|
||||
|
||||
|
||||
+14
-5
@@ -1,20 +1,29 @@
|
||||
# Installing xrpld 3.3.0 and earlier
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!IMPORTANT]
|
||||
> These instructions apply to xrpld 3.3.0 and earlier, published to repos.ripple.com. For later releases see [install.md](./install.md).
|
||||
> These instructions apply to xrpld 3.3.0 and earlier, published to
|
||||
> repos.ripple.com.
|
||||
> For later releases see [install.md](./install.md).
|
||||
|
||||
This document contains instructions for installing xrpld. The APT package manager is common on Debian-based Linux distributions like Ubuntu, while the YUM package manager is common on Red Hat-based Linux distributions like CentOS. Installing from source is an option for all platforms, and the only supported option for installing custom builds.
|
||||
This document contains instructions for installing xrpld.
|
||||
The APT package manager is common on Debian-based Linux distributions like
|
||||
Ubuntu,
|
||||
while the YUM package manager is common on Red Hat-based Linux distributions
|
||||
like CentOS.
|
||||
Installing from source is an option for all platforms,
|
||||
and the only supported option for installing custom builds.
|
||||
|
||||
## From source
|
||||
|
||||
From a source build, you can install xrpld and libxrpl using CMake's `--install` mode:
|
||||
From a source build, you can install xrpld and libxrpl using CMake's
|
||||
`--install` mode:
|
||||
|
||||
```
|
||||
cmake --install . --prefix /opt/local
|
||||
```
|
||||
|
||||
The default [prefix][1] is typically `/usr/local` on Linux and macOS and `C:/Program Files/xrpld` on Windows.
|
||||
The default [prefix][1] is typically `/usr/local` on Linux and macOS and
|
||||
`C:/Program Files/xrpld` on Windows.
|
||||
|
||||
[1]: https://cmake.org/cmake/help/latest/variable/CMAKE_INSTALL_PREFIX.html
|
||||
|
||||
|
||||
+35
-12
@@ -1,10 +1,14 @@
|
||||
# Installing xrpld
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!NOTE]
|
||||
> These instructions apply to packages published from 2026-08-19 onwards. For xrpld 3.3.0 and earlier see [install-legacy.md](./install-legacy.md).
|
||||
> These instructions apply to packages published from 2026-08-19 onwards.
|
||||
> For xrpld 3.3.0 and earlier see [install-legacy.md](./install-legacy.md).
|
||||
|
||||
`xrpld` is published as DEB and RPM packages for 64-bit x86 Linux. Use APT on Debian-based distributions such as Debian and Ubuntu, and YUM on Red Hat-based distributions such as RHEL, AlmaLinux, and Rocky Linux. To build from source instead, see [BUILD.md](../BUILD.md).
|
||||
`xrpld` is published as DEB and RPM packages for 64-bit x86 Linux.
|
||||
Use APT on Debian-based distributions such as Debian and Ubuntu,
|
||||
and DNF on Red Hat-based distributions such as RHEL, AlmaLinux, and Rocky Linux,
|
||||
where `yum` is a symlink to `dnf`.
|
||||
To build from source instead, see [BUILD.md](../BUILD.md).
|
||||
|
||||
## Release channels
|
||||
|
||||
@@ -17,11 +21,13 @@ Packages are published to four channels:
|
||||
|
||||
See [Publishing packages](../package/README.md#publishing-packages) for how channels are produced.
|
||||
|
||||
The instructions below use `stable`. To follow another channel, replace `stable` with its name wherever it appears in the repository configuration.
|
||||
The instructions below use `stable`.
|
||||
To follow another channel, replace `stable` with its name
|
||||
wherever it appears in the repository configuration.
|
||||
|
||||
<!-- prettier-ignore -->
|
||||
> [!WARNING]
|
||||
> Channels other than `stable` may be broken at any time. Do not use them for production servers.
|
||||
> Channels other than `stable` may be broken at any time.
|
||||
> Do not use them for production servers.
|
||||
|
||||
## Install the xrpld package
|
||||
|
||||
@@ -76,7 +82,7 @@ The instructions below use `stable`. To follow another channel, replace `stable`
|
||||
sudo apt -y install xrpld
|
||||
```
|
||||
|
||||
### With the YUM package manager
|
||||
### With the DNF package manager
|
||||
|
||||
1. Add the XRPL Foundation package-signing key:
|
||||
|
||||
@@ -98,23 +104,39 @@ The instructions below use `stable`. To follow another channel, replace `stable`
|
||||
REPOFILE
|
||||
```
|
||||
|
||||
`gpgcheck=1` verifies each package against the key above. `repo_gpgcheck=1` verifies the repository metadata, which the server signs with the same key.
|
||||
`gpgcheck=1` verifies each package against the key above.
|
||||
`repo_gpgcheck=1` verifies the repository metadata, which the server signs with the same key.
|
||||
|
||||
3. Install the `xrpld` package:
|
||||
|
||||
```bash
|
||||
sudo yum install -y xrpld
|
||||
sudo dnf install -y xrpld
|
||||
```
|
||||
|
||||
### Optional: the assert-enabled build
|
||||
|
||||
Every channel also carries `xrpld-assert` as a DEB, the same build with assertions
|
||||
enabled, for diagnosing a problem on a non-production server.
|
||||
It installs the same files as `xrpld` and replaces it, so install one or the other:
|
||||
|
||||
```bash
|
||||
sudo apt -y install xrpld-assert # APT removes xrpld itself
|
||||
```
|
||||
|
||||
Switching stops the service, since it is a removal and an installation rather than an upgrade,
|
||||
and APT starts it again.
|
||||
Install `xrpld` the same way to switch back.
|
||||
|
||||
## The xrpld service
|
||||
|
||||
Both package managers install a systemd unit and enable it, so `xrpld` starts on boot. Check whether it is already running:
|
||||
Both package managers install a systemd unit and enable it, so `xrpld` starts on boot.
|
||||
Check whether it is already running:
|
||||
|
||||
```bash
|
||||
systemctl status xrpld.service
|
||||
```
|
||||
|
||||
The APT packages start it immediately as well; the YUM packages do not, so start it yourself:
|
||||
The DEB packages start it immediately as well; the RPM packages do not, so start it yourself:
|
||||
|
||||
```bash
|
||||
sudo systemctl start xrpld.service
|
||||
@@ -122,7 +144,8 @@ sudo systemctl start xrpld.service
|
||||
|
||||
### Optional: binding to privileged ports
|
||||
|
||||
To serve incoming API requests on port 80 or 443, grant the service the capability to bind them. You must also update the config file's port settings.
|
||||
To serve incoming API requests on port 80 or 443, grant the service the capability to bind them.
|
||||
You must also update the config file's port settings.
|
||||
|
||||
```bash
|
||||
sudo install -d -m 0755 /etc/systemd/system/xrpld.service.d
|
||||
|
||||
Generated
+6
-6
@@ -2,11 +2,11 @@
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1781173989,
|
||||
"narHash": "sha256-fnzKKPvS+oieI/pTzotA5tkoM47EB1NpaBcgk4R97hE=",
|
||||
"lastModified": 1791130267,
|
||||
"narHash": "sha256-1sjwcQMcgAbmiip/VivBSiK8p2bTkQGOvCuO7vnwfx4=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "8c91a71d13451abc40eb9dae8910f972f979852f",
|
||||
"rev": "9013764fcc0ea99fa16cf7aa7decf8a5c3889dd2",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -47,11 +47,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1784611586,
|
||||
"narHash": "sha256-OfqgY+0hp/zseZB7uyH0U8kIDPS4scZZCyAurEplvG0=",
|
||||
"lastModified": 1791189933,
|
||||
"narHash": "sha256-Y8eN7ki0Urx4Ojf4w1xSpWg8uk/3lJqK+E0Oth77rH0=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "14f58845249f3552a89b07772626b8d3c632fa86",
|
||||
"rev": "e60029353d0c48d216bc4b065168ccd4079c166f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# basics' container adapters (TaggedCache, partitioned_unordered_map,
|
||||
# base_uint) model std concepts, which mandate
|
||||
# snake_case member type names.
|
||||
#
|
||||
# clang-tidy replaces rather than merges this option, so the list is the
|
||||
# top-level one restated plus the 6 name(s) needed only here:
|
||||
# allocator_type const_pointer const_reverse_iterator hasher key_equal reverse_iterator
|
||||
InheritParentConfig: true
|
||||
CheckOptions:
|
||||
readability-identifier-naming.TypeAliasIgnoredRegexp: "^(allocator_type|const_iterator|const_pointer|const_reference|const_reverse_iterator|difference_type|duration|hasher|iterator|iterator_category|key_equal|key_type|mapped_type|period|pointer|reference|rep|result_type|reverse_iterator|size_type|time_point|value_type)$"
|
||||
readability-identifier-naming.TypedefIgnoredRegexp: "^(allocator_type|const_iterator|const_pointer|const_reference|const_reverse_iterator|difference_type|duration|hasher|iterator|iterator_category|key_equal|key_type|mapped_type|period|pointer|reference|rep|result_type|reverse_iterator|size_type|time_point|value_type)$"
|
||||
@@ -1,29 +1,31 @@
|
||||
#pragma once
|
||||
|
||||
#include <xrpl/beast/type_name.h>
|
||||
#include <xrpl/beast/utility/instrumentation.h>
|
||||
|
||||
#include <boost/core/type_name.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <iterator>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
/**
|
||||
* Manages all counted object types.
|
||||
*
|
||||
* Counters register themselves on a lock-free intrusive list maintained
|
||||
* by this object when constructed. Because counters are never destroyed
|
||||
* or removed, the ABA problem does not apply.
|
||||
*
|
||||
* The registry is iterable as a forward range.
|
||||
*/
|
||||
class CountedObjects
|
||||
{
|
||||
public:
|
||||
static CountedObjects&
|
||||
getInstance() noexcept;
|
||||
|
||||
using Entry = std::pair<std::string, int>;
|
||||
using List = std::vector<Entry>;
|
||||
|
||||
[[nodiscard]] List
|
||||
getCounts(int minimumThreshold) const;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Implementation for @ref CountedObject.
|
||||
@@ -33,68 +35,147 @@ public:
|
||||
class Counter
|
||||
{
|
||||
public:
|
||||
Counter(std::string name) noexcept : name_(std::move(name)), count_(0)
|
||||
{
|
||||
// Insert ourselves at the front of the lock-free linked list
|
||||
CountedObjects& instance = CountedObjects::getInstance();
|
||||
Counter* head = nullptr;
|
||||
Counter(std::string name) noexcept;
|
||||
|
||||
do
|
||||
{
|
||||
head = instance.head_.load();
|
||||
next_ = head;
|
||||
} while (instance.head_.exchange(this) != head);
|
||||
// Counters are intrusive list nodes whose addresses are published
|
||||
// in the registry; they must never be copied or moved. The atomic
|
||||
// members already force this, but we make it explicit.
|
||||
Counter(Counter const&) = delete;
|
||||
Counter&
|
||||
operator=(Counter const&) = delete;
|
||||
Counter(Counter&&) = delete;
|
||||
Counter&
|
||||
operator=(Counter&&) = delete;
|
||||
|
||||
++instance.count_;
|
||||
}
|
||||
|
||||
~Counter() noexcept = default;
|
||||
|
||||
int
|
||||
std::uint32_t
|
||||
increment() noexcept
|
||||
{
|
||||
return ++count_;
|
||||
auto const newCount = count_.fetch_add(1, std::memory_order::relaxed) + 1;
|
||||
XRPL_ASSERT(newCount != 0, "xrpl::CountedObjects::Counter::increment : no overflow");
|
||||
|
||||
auto maxCount = maxCount_.load(std::memory_order::relaxed);
|
||||
|
||||
while (newCount > maxCount &&
|
||||
!maxCount_.compare_exchange_weak(maxCount, newCount, std::memory_order::relaxed))
|
||||
{
|
||||
}
|
||||
|
||||
return newCount;
|
||||
}
|
||||
|
||||
int
|
||||
std::uint32_t
|
||||
decrement() noexcept
|
||||
{
|
||||
return --count_;
|
||||
auto const prev = count_.fetch_sub(1, std::memory_order::relaxed);
|
||||
XRPL_ASSERT(prev != 0, "xrpl::CountedObjects::Counter::decrement : no underflow");
|
||||
return prev - 1;
|
||||
}
|
||||
|
||||
[[nodiscard]] int
|
||||
getCount() const noexcept
|
||||
[[nodiscard]] std::uint32_t
|
||||
count() const noexcept
|
||||
{
|
||||
return count_.load();
|
||||
return count_.load(std::memory_order::relaxed);
|
||||
}
|
||||
|
||||
[[nodiscard]] Counter*
|
||||
getNext() const noexcept
|
||||
[[nodiscard]] std::uint32_t
|
||||
max() const noexcept
|
||||
{
|
||||
return next_;
|
||||
return std::max(
|
||||
count_.load(std::memory_order::relaxed),
|
||||
maxCount_.load(std::memory_order::relaxed));
|
||||
}
|
||||
|
||||
[[nodiscard]] std::string const&
|
||||
getName() const noexcept
|
||||
name() const noexcept
|
||||
{
|
||||
return name_;
|
||||
}
|
||||
|
||||
private:
|
||||
friend class CountedObjects;
|
||||
|
||||
Counter* next_ = nullptr;
|
||||
std::atomic<std::uint32_t> count_ = 0;
|
||||
std::atomic<std::uint32_t> maxCount_ = 0;
|
||||
std::string const name_;
|
||||
std::atomic<int> count_;
|
||||
Counter* next_;
|
||||
};
|
||||
|
||||
private:
|
||||
CountedObjects() noexcept;
|
||||
~CountedObjects() noexcept = default;
|
||||
class Iterator
|
||||
{
|
||||
public:
|
||||
using value_type = Counter const;
|
||||
using reference = value_type&;
|
||||
using pointer = value_type*;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
|
||||
explicit Iterator(Counter* c = nullptr) noexcept : current_(c)
|
||||
{
|
||||
}
|
||||
|
||||
reference
|
||||
operator*() const noexcept
|
||||
{
|
||||
return *current_;
|
||||
}
|
||||
|
||||
pointer
|
||||
operator->() const noexcept
|
||||
{
|
||||
return current_;
|
||||
}
|
||||
|
||||
Iterator&
|
||||
operator++() noexcept
|
||||
{
|
||||
current_ = current_->next_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
Iterator
|
||||
operator++(int) noexcept
|
||||
{
|
||||
auto tmp = *this;
|
||||
++*this;
|
||||
return tmp;
|
||||
}
|
||||
|
||||
bool
|
||||
operator==(Iterator const&) const noexcept = default;
|
||||
|
||||
private:
|
||||
Counter* current_;
|
||||
};
|
||||
|
||||
constexpr CountedObjects() noexcept = default;
|
||||
|
||||
[[nodiscard]] auto
|
||||
begin() const noexcept
|
||||
{
|
||||
return Iterator{head_.load(std::memory_order::acquire)};
|
||||
}
|
||||
|
||||
[[nodiscard]] auto
|
||||
end() const noexcept
|
||||
{
|
||||
return Iterator{};
|
||||
}
|
||||
|
||||
private:
|
||||
std::atomic<int> count_;
|
||||
std::atomic<Counter*> head_;
|
||||
std::atomic<Counter*> head_ = nullptr;
|
||||
};
|
||||
|
||||
/** The global counted object registry. */
|
||||
inline constinit CountedObjects gCountedObjects;
|
||||
|
||||
inline CountedObjects::Counter::Counter(std::string name) noexcept
|
||||
: next_(gCountedObjects.head_.load(std::memory_order::relaxed)), name_(std::move(name))
|
||||
{
|
||||
while (!gCountedObjects.head_.compare_exchange_weak(
|
||||
next_, this, std::memory_order::release, std::memory_order::relaxed))
|
||||
;
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
@@ -103,27 +184,37 @@ private:
|
||||
* Derived classes have their instances counted automatically. This is used
|
||||
* for reporting purposes.
|
||||
*
|
||||
* The constructors are private and `Object` is befriended so that the
|
||||
* CRTP parameter must be the deriving class itself: a copy-paste error
|
||||
* like `class B : public CountedObject<A>` fails to compile instead of
|
||||
* silently polluting A's count.
|
||||
*
|
||||
* @note This class has no move operations by design: a derived class's
|
||||
* move constructor falls back to the copy constructor for this
|
||||
* base, so the newly created instance is counted. This keeps the
|
||||
* invariant that count is the number of outstanding subobjects.
|
||||
*
|
||||
* @warning Counted objects constructed during dynamic initialization of
|
||||
* other translation units may have their increments discarded when
|
||||
* counter itself is dynamically initialized. Do not create counted
|
||||
* objects before main() begins.
|
||||
*
|
||||
* @ingroup basics
|
||||
*/
|
||||
template <class Object>
|
||||
requires std::is_class_v<Object>
|
||||
class CountedObject
|
||||
{
|
||||
private:
|
||||
static auto&
|
||||
getCounter() noexcept
|
||||
{
|
||||
static CountedObjects::Counter kC{beast::typeName<Object>()};
|
||||
return kC;
|
||||
}
|
||||
static inline CountedObjects::Counter counter{boost::core::type_name<Object>()};
|
||||
|
||||
CountedObject() noexcept
|
||||
{
|
||||
getCounter().increment();
|
||||
counter.increment();
|
||||
}
|
||||
|
||||
CountedObject(CountedObject const&) noexcept
|
||||
{
|
||||
getCounter().increment();
|
||||
counter.increment();
|
||||
}
|
||||
|
||||
CountedObject&
|
||||
@@ -132,7 +223,7 @@ private:
|
||||
public:
|
||||
~CountedObject() noexcept
|
||||
{
|
||||
getCounter().decrement();
|
||||
counter.decrement();
|
||||
}
|
||||
|
||||
friend Object;
|
||||
|
||||
@@ -618,13 +618,16 @@ SharedWeakUnion<T>::convertToWeak()
|
||||
unsafeSetRawPtr(nullptr);
|
||||
return true; // Should never happen
|
||||
// LCOV_EXCL_STOP
|
||||
case PartialDestroy:
|
||||
// This is a weird case. We just converted the last strong
|
||||
// pointer to a weak pointer.
|
||||
case PartialDestroy: {
|
||||
// We just converted the last strong pointer to a weak pointer.
|
||||
// The weak ref we now hold keeps the object from being fully
|
||||
// destroyed, so `p` stays valid; only the copy passed to
|
||||
// `partialDestructorFinished` is nulled.
|
||||
p->partialDestructor();
|
||||
partialDestructorFinished(&p);
|
||||
// p is null and may no longer be used
|
||||
auto finished = p;
|
||||
partialDestructorFinished(&finished);
|
||||
break;
|
||||
}
|
||||
}
|
||||
unsafeSetRawPtr(p, RefStrength::Weak);
|
||||
return true;
|
||||
|
||||
@@ -77,21 +77,13 @@ struct IntrusiveRefCounts
|
||||
std::size_t
|
||||
useCount() const noexcept;
|
||||
|
||||
// This function MUST be called after a partial destructor finishes running.
|
||||
// Calling this function may cause other threads to delete the object
|
||||
// pointed to by `o`, so `o` should never be used after calling this
|
||||
// function. The parameter will be set to a `nullptr` after calling this
|
||||
// function to emphasize that it should not be used.
|
||||
// Note: This is intentionally NOT called at the end of `partialDestructor`.
|
||||
// The reason for this is if new classes are written to support this smart
|
||||
// pointer class, they need to write their own `partialDestructor` function
|
||||
// and ensure `partialDestructorFinished` is called at the end. Putting this
|
||||
// call inside the smart pointer class itself is expected to be less error
|
||||
// prone.
|
||||
// Note: The "two-star" programming is intentional. It emphasizes that `o`
|
||||
// may be deleted and the unergonomic API is meant to signal the special
|
||||
// nature of this function call to callers.
|
||||
// Note: This is a template to support incompletely defined classes.
|
||||
// MUST be called after `partialDestructor` returns. Another thread may
|
||||
// then delete the object, so `*o` is nulled and must not be used after
|
||||
// (unless the caller holds its own weak ref, e.g.
|
||||
// SharedWeakUnion::convertToWeak). Called by the smart pointers, not
|
||||
// `partialDestructor`, so custom partial destructors can't forget it.
|
||||
// The two-star API signals that `*o` may be deleted. Templated to
|
||||
// support incomplete types.
|
||||
template <class T>
|
||||
friend void
|
||||
partialDestructorFinished(T** o);
|
||||
@@ -334,15 +326,11 @@ IntrusiveRefCounts::addWeakReleaseStrongRef() const
|
||||
ReleaseStrongRefAction action = NoOp;
|
||||
if (prevVal.strong == 1)
|
||||
{
|
||||
if (prevVal.weak == 0)
|
||||
{
|
||||
action = NoOp;
|
||||
}
|
||||
else
|
||||
{
|
||||
nextIntVal |= kPartialDestroyStartedMask;
|
||||
action = PartialDestroy;
|
||||
}
|
||||
// The weak ref added here keeps the weak count non-zero, so
|
||||
// releasing the last strong ref always starts a partial destroy,
|
||||
// regardless of the previous weak count.
|
||||
nextIntVal |= kPartialDestroyStartedMask;
|
||||
action = PartialDestroy;
|
||||
}
|
||||
if (refCounts_.compare_exchange_weak(prevIntVal, nextIntVal, std::memory_order_acq_rel))
|
||||
{
|
||||
@@ -358,24 +346,26 @@ IntrusiveRefCounts::addWeakReleaseStrongRef() const
|
||||
inline ReleaseWeakRefAction
|
||||
IntrusiveRefCounts::releaseWeakRef() const
|
||||
{
|
||||
auto prevIntVal = refCounts_.fetch_sub(kWeakDelta, std::memory_order_acq_rel);
|
||||
RefCountPair prev = prevIntVal;
|
||||
auto const prevIntVal = refCounts_.fetch_sub(kWeakDelta, std::memory_order_acq_rel);
|
||||
RefCountPair const prev = prevIntVal;
|
||||
if (prev.weak == 1 && prev.strong == 0)
|
||||
{
|
||||
// `wait` blocks while the value equals its argument, so it must be
|
||||
// given the value as it is after the decrement above.
|
||||
auto curIntVal = prevIntVal - kWeakDelta;
|
||||
if (prev.partialDestroyStartedBit == 0u)
|
||||
{
|
||||
// This case should only be hit if the partialDestroyStartedBit is
|
||||
// set non-atomically (and even then very rarely). The code is kept
|
||||
// in case we need to set the flag non-atomically for perf reasons.
|
||||
refCounts_.wait(prevIntVal, std::memory_order_acquire);
|
||||
prevIntVal = refCounts_.load(std::memory_order_acquire);
|
||||
prev = RefCountPair{prevIntVal};
|
||||
refCounts_.wait(curIntVal, std::memory_order_acquire);
|
||||
curIntVal = refCounts_.load(std::memory_order_acquire);
|
||||
}
|
||||
if (prev.partialDestroyFinishedBit == 0u)
|
||||
if (RefCountPair{curIntVal}.partialDestroyFinishedBit == 0u)
|
||||
{
|
||||
// partial destroy MUST finish before running a full destroy (when
|
||||
// using weak pointers)
|
||||
refCounts_.wait(prevIntVal - kWeakDelta, std::memory_order_acquire);
|
||||
refCounts_.wait(curIntVal, std::memory_order_acquire);
|
||||
}
|
||||
return ReleaseWeakRefAction::Destroy;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,6 @@
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
using KeyCache = TaggedCache<uint256, int, true>;
|
||||
using KeyCache = TaggedCache<UInt256, int, true>;
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -110,10 +110,10 @@ static_assert(
|
||||
*
|
||||
* However, it does not have sufficient precision to represent the full integer
|
||||
* range of int64_t values (-2^63 to 2^63-1), which are needed for XRP and MPT
|
||||
* values. The implementation of SingleAssetVault, and LendingProtocol need to
|
||||
* represent those integer values accurately and precisely, both for the
|
||||
* STNumber field type, and for internal calculations. That necessitated the
|
||||
* "large" scale.
|
||||
* values. The implementation of SingleAssetVault, LendingProtocol, and
|
||||
* MPTokensV2 need to represent those integer values accurately and precisely,
|
||||
* both for the STNumber field type, and for internal calculations. That
|
||||
* necessitated the "large" scale.
|
||||
*
|
||||
* The "Large" scales are intended to represent all values that can be represented
|
||||
* by an STAmount - IOUs, XRP, and MPTs. It has a min value of 10^18, and a max
|
||||
@@ -134,8 +134,8 @@ struct MantissaRange final
|
||||
// NOLINTBEGIN(readability-enum-initial-value)
|
||||
// The values don't matter, except for Large
|
||||
enum class MantissaScale {
|
||||
// Small can be removed when either featureSingleAssetVault or featureLendingProtocol are
|
||||
// retired
|
||||
// Small can be removed when any of featureSingleAssetVault, featureLendingProtocol, or
|
||||
// featureMPTokensV2 are retired
|
||||
Small,
|
||||
// LargeLegacy can be removed when fixCleanup3_2_0 is retired
|
||||
LargeLegacy,
|
||||
@@ -311,10 +311,10 @@ concept Integral64 = std::is_same_v<T, std::int64_t> || std::is_same_v<T, std::u
|
||||
*
|
||||
* The mantissa range may be changed at runtime via setMantissaScale(). The
|
||||
* default mantissa range is "large". The range is updated whenever transaction
|
||||
* processing begins, based on whether SingleAssetVault or LendingProtocol are
|
||||
* enabled. If either is enabled, the mantissa range is set to "large". If not,
|
||||
* it is set to "small", preserving backward compatibility and correct
|
||||
* "amendment-gating".
|
||||
* processing begins, based on whether SingleAssetVault, LendingProtocol, or
|
||||
* MPTokensV2 are enabled. If any is enabled, the mantissa range is set to
|
||||
* "large". If not, it is set to "small", preserving backward compatibility and
|
||||
* correct "amendment-gating".
|
||||
*
|
||||
* It is extremely unlikely that any more calls to setMantissaScale() will be
|
||||
* needed outside of unit tests.
|
||||
@@ -344,16 +344,16 @@ concept Integral64 = std::is_same_v<T, std::int64_t> || std::is_same_v<T, std::u
|
||||
* set/getMantissaScale() functions may be most appropriate. However, if the
|
||||
* test has anything to do with transaction processing, it should enable or
|
||||
* disable the amendments that control the mantissa range choice
|
||||
* (SingleAssetVault and LendingProtocol), and/or check if either of those
|
||||
* amendments are enabled to determine which result to expect.
|
||||
* (SingleAssetVault, LendingProtocol, and MPTokensV2), and/or check if any of
|
||||
* those amendments are enabled to determine which result to expect.
|
||||
*/
|
||||
class Number final
|
||||
{
|
||||
using rep = std::int64_t;
|
||||
using internalrep = MantissaRange::rep;
|
||||
using InternalRep = MantissaRange::rep;
|
||||
|
||||
bool negative_{false};
|
||||
internalrep mantissa_{0};
|
||||
InternalRep mantissa_{0};
|
||||
int exponent_{std::numeric_limits<int>::lowest()};
|
||||
|
||||
public:
|
||||
@@ -361,10 +361,10 @@ public:
|
||||
static constexpr int kMinExponent = -32768;
|
||||
static constexpr int kMaxExponent = 32768;
|
||||
|
||||
static constexpr internalrep kMaxRep = std::numeric_limits<rep>::max();
|
||||
static constexpr InternalRep kMaxRep = std::numeric_limits<rep>::max();
|
||||
static_assert(kMaxRep == 9'223'372'036'854'775'807);
|
||||
static_assert(-kMaxRep == std::numeric_limits<rep>::min() + 1);
|
||||
static constexpr internalrep kMaxRepUp = ((kMaxRep / 10) + 1) * 10;
|
||||
static constexpr InternalRep kMaxRepUp = ((kMaxRep / 10) + 1) * 10;
|
||||
static_assert(kMaxRepUp == 9'223'372'036'854'775'810ULL);
|
||||
|
||||
// May need to make unchecked private
|
||||
@@ -388,15 +388,15 @@ public:
|
||||
explicit Number(rep mantissa, int exponent);
|
||||
explicit constexpr Number(
|
||||
bool negative,
|
||||
internalrep mantissa,
|
||||
InternalRep mantissa,
|
||||
int exponent,
|
||||
Unchecked) noexcept;
|
||||
// Assume unsigned values are... unsigned. i.e. positive
|
||||
explicit constexpr Number(internalrep mantissa, int exponent, Unchecked) noexcept;
|
||||
explicit constexpr Number(InternalRep mantissa, int exponent, Unchecked) noexcept;
|
||||
// Only unit tests are expected to use this ctor
|
||||
explicit Number(bool negative, internalrep mantissa, int exponent, Normalized);
|
||||
explicit Number(bool negative, InternalRep mantissa, int exponent, Normalized);
|
||||
// Assume unsigned values are... unsigned. i.e. positive
|
||||
explicit Number(internalrep mantissa, int exponent, Normalized);
|
||||
explicit Number(InternalRep mantissa, int exponent, Normalized);
|
||||
|
||||
[[nodiscard]] constexpr rep
|
||||
mantissa() const noexcept;
|
||||
@@ -558,13 +558,13 @@ public:
|
||||
static void
|
||||
setMantissaScale(MantissaRange::MantissaScale scale);
|
||||
|
||||
static internalrep
|
||||
static InternalRep
|
||||
minMantissa()
|
||||
{
|
||||
return kRange.get().min;
|
||||
}
|
||||
|
||||
static internalrep
|
||||
static InternalRep
|
||||
maxMantissa()
|
||||
{
|
||||
return kRange.get().max;
|
||||
@@ -591,7 +591,7 @@ public:
|
||||
// is negative, returns the positive value. This takes a little extra work
|
||||
// because converting std::numeric_limits<std::int64_t>::min() flirts with
|
||||
// UB, and can vary across compilers.
|
||||
static internalrep
|
||||
static InternalRep
|
||||
externalToInternal(rep mantissa);
|
||||
|
||||
private:
|
||||
@@ -625,8 +625,8 @@ private:
|
||||
bool& negative,
|
||||
T& mantissa,
|
||||
int& exponent,
|
||||
internalrep const& minMantissa,
|
||||
internalrep const& maxMantissa,
|
||||
InternalRep const& minMantissa,
|
||||
InternalRep const& maxMantissa,
|
||||
MantissaRange::CuspRoundingFix cuspRoundingFix);
|
||||
|
||||
template <class T>
|
||||
@@ -650,25 +650,25 @@ private:
|
||||
shiftExponent(int exponentDelta) const;
|
||||
};
|
||||
|
||||
constexpr Number::Number(bool negative, internalrep mantissa, int exponent, Unchecked) noexcept
|
||||
constexpr Number::Number(bool negative, InternalRep mantissa, int exponent, Unchecked) noexcept
|
||||
: negative_(negative), mantissa_{mantissa}, exponent_{exponent}
|
||||
{
|
||||
}
|
||||
|
||||
constexpr Number::Number(internalrep mantissa, int exponent, Unchecked) noexcept
|
||||
constexpr Number::Number(InternalRep mantissa, int exponent, Unchecked) noexcept
|
||||
: Number(false, mantissa, exponent, Unchecked{})
|
||||
{
|
||||
}
|
||||
|
||||
static constexpr Number kNumZero{};
|
||||
|
||||
inline Number::Number(bool negative, internalrep mantissa, int exponent, Normalized)
|
||||
inline Number::Number(bool negative, InternalRep mantissa, int exponent, Normalized)
|
||||
: Number(negative, mantissa, exponent, Unchecked{})
|
||||
{
|
||||
normalize(kRange);
|
||||
}
|
||||
|
||||
inline Number::Number(internalrep mantissa, int exponent, Normalized)
|
||||
inline Number::Number(InternalRep mantissa, int exponent, Normalized)
|
||||
: Number(false, mantissa, exponent, Normalized{})
|
||||
{
|
||||
}
|
||||
@@ -853,7 +853,7 @@ Number::normalizeToRange() const
|
||||
static_assert((kMAX + 1) / 10 == kMIN);
|
||||
|
||||
bool negative = negative_;
|
||||
internalrep mantissa = mantissa_;
|
||||
InternalRep mantissa = mantissa_;
|
||||
int exponent = exponent_;
|
||||
|
||||
if constexpr (std::is_unsigned_v<T>)
|
||||
|
||||
@@ -15,20 +15,23 @@ The module xrpl/basics should contain no dependencies on other modules.
|
||||
- `std::list`
|
||||
- For ordered containers with inserts and erases to the middle.
|
||||
- For containers with iterators stable over insert and erase.
|
||||
- Generally slower and bigger than `std::vector` or `std::deque` except for those cases.
|
||||
- Generally slower and bigger than `std::vector` or `std::deque` except for
|
||||
those cases.
|
||||
|
||||
- `std::set`
|
||||
- For sorted containers.
|
||||
|
||||
- `xrpl::hash_set`
|
||||
- `xrpl::HashSet`
|
||||
- Where inserts and contains need to be O(1).
|
||||
- For "small" sets, `std::set` might be faster and smaller.
|
||||
|
||||
- `xrpl::hardened_hash_set`
|
||||
- For data sets where the key could be manipulated by an attacker in an attempt to mount an algorithmic complexity attack: see http://en.wikipedia.org/wiki/Algorithmic_complexity_attack
|
||||
- `xrpl::HardenedHashSet`
|
||||
- For data sets where the key could be manipulated by an attacker
|
||||
in an attempt to mount an algorithmic complexity attack: see
|
||||
http://en.wikipedia.org/wiki/Algorithmic_complexity_attack
|
||||
|
||||
The following container is deprecated
|
||||
|
||||
- `std::unordered_set`
|
||||
- Use `xrpl::hash_set` instead, which uses a better hashing algorithm.
|
||||
- Or use `xrpl::hardened_hash_set` to prevent algorithmic complexity attacks.
|
||||
- Use `xrpl::HashSet` instead, which uses a better hashing algorithm.
|
||||
- Or use `xrpl::HardenedHashSet` to prevent algorithmic complexity attacks.
|
||||
@@ -14,20 +14,20 @@ namespace xrpl {
|
||||
|
||||
class SHAMapHash
|
||||
{
|
||||
uint256 hash_;
|
||||
UInt256 hash_;
|
||||
|
||||
public:
|
||||
SHAMapHash() = default;
|
||||
explicit SHAMapHash(uint256 const& hash) : hash_(hash)
|
||||
explicit SHAMapHash(UInt256 const& hash) : hash_(hash)
|
||||
{
|
||||
}
|
||||
|
||||
[[nodiscard]] uint256 const&
|
||||
[[nodiscard]] UInt256 const&
|
||||
asUInt256() const
|
||||
{
|
||||
return hash_;
|
||||
}
|
||||
uint256&
|
||||
UInt256&
|
||||
asUInt256()
|
||||
{
|
||||
return hash_;
|
||||
|
||||
+957
-296
File diff suppressed because it is too large.
Load diff
@@ -162,4 +162,89 @@ toUInt64(std::string const& s);
|
||||
bool
|
||||
isProperlyFormedTomlDomain(std::string_view domain);
|
||||
|
||||
/**
|
||||
* Whether a view can be passed on as a C string.
|
||||
*
|
||||
* A reader given only data() stops at the first null, so the view must reach the
|
||||
* terminating null. The test rebuilds the view from data() and compares: a view
|
||||
* that stops earlier rebuilds longer, and so compares unequal.
|
||||
*
|
||||
* consteval because reading the byte after the view is only defined when @p str
|
||||
* points into storage holding a null at or after its end, such as a string
|
||||
* literal. An unterminated view is then a compile error, not an out-of-bounds
|
||||
* read.
|
||||
*
|
||||
* @param str The view to test.
|
||||
* @return Whether @p str is null-terminated. A view with no data is not.
|
||||
*/
|
||||
consteval bool
|
||||
isNullTerminated(std::string_view str)
|
||||
{
|
||||
if (str.data() == nullptr)
|
||||
return false;
|
||||
|
||||
// Reading past the view is the point, so the usual data() warning does not
|
||||
// apply.
|
||||
// NOLINTNEXTLINE(bugprone-suspicious-stringview-data-usage)
|
||||
return std::string_view{str.data()} == str;
|
||||
}
|
||||
|
||||
/**
|
||||
* A string that is known to reach its terminating null.
|
||||
*
|
||||
* Converts to std::string_view, so it compares and hashes as one. Unlike a
|
||||
* view, asCString() may be handed to a reader that expects a C string, such
|
||||
* as json::StaticString.
|
||||
*
|
||||
* The only constructor is consteval and rejects a view that stops before the
|
||||
* null, so the property holds by construction and no caller asserts it.
|
||||
*/
|
||||
class NullTerminatedView
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Build a view from one that reaches its terminating null.
|
||||
*
|
||||
* Explicit, so that a plain view cannot become a proof of termination by
|
||||
* accident. The conversion the other way stays implicit.
|
||||
*
|
||||
* @param view The string to hold. Rejected at compile time if it stops
|
||||
* before its terminating null, or has no data.
|
||||
*/
|
||||
explicit consteval NullTerminatedView(std::string_view view)
|
||||
: data_(view.data()), size_(view.size())
|
||||
{
|
||||
if (!isNullTerminated(view))
|
||||
throw "xrpl::NullTerminatedView : view does not reach a null";
|
||||
}
|
||||
|
||||
constexpr
|
||||
operator std::string_view() const noexcept
|
||||
{
|
||||
return view();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The string as a view.
|
||||
*/
|
||||
[[nodiscard]] constexpr std::string_view
|
||||
view() const noexcept
|
||||
{
|
||||
return {data_, size_};
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The string as a C string. Never null.
|
||||
*/
|
||||
[[nodiscard]] constexpr char const*
|
||||
asCString() const noexcept
|
||||
{
|
||||
return data_;
|
||||
}
|
||||
|
||||
private:
|
||||
char const* data_;
|
||||
std::size_t size_;
|
||||
};
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -59,34 +59,34 @@ template <
|
||||
class T,
|
||||
bool IsKeyCache = false,
|
||||
class SharedWeakUnionPointerType = SharedWeakCachePointer<T>,
|
||||
class SharedPointerType = std::shared_ptr<T>,
|
||||
class SharedPointer = std::shared_ptr<T>,
|
||||
class Hash = HardenedHash<>,
|
||||
class KeyEqual = std::equal_to<Key>,
|
||||
class Mutex = std::recursive_mutex>
|
||||
class TaggedCache
|
||||
{
|
||||
public:
|
||||
using mutex_type = Mutex;
|
||||
using MutexType = Mutex;
|
||||
using key_type = Key;
|
||||
using mapped_type = T;
|
||||
using clock_type = beast::AbstractClock<std::chrono::steady_clock>;
|
||||
using shared_weak_combo_pointer_type = SharedWeakUnionPointerType;
|
||||
using shared_pointer_type = SharedPointerType;
|
||||
using ClockType = beast::AbstractClock<std::chrono::steady_clock>;
|
||||
using SharedWeakComboPointerType = SharedWeakUnionPointerType;
|
||||
using SharedPointerType = SharedPointer;
|
||||
|
||||
public:
|
||||
TaggedCache(
|
||||
std::string const& name,
|
||||
int size,
|
||||
clock_type::duration expiration,
|
||||
clock_type& clock,
|
||||
ClockType::duration expiration,
|
||||
ClockType& clock,
|
||||
beast::Journal journal,
|
||||
beast::insight::Collector::ptr const& collector = beast::insight::NullCollector::make());
|
||||
beast::insight::Collector::Ptr const& collector = beast::insight::NullCollector::make());
|
||||
|
||||
public:
|
||||
/**
|
||||
* Return the clock associated with the cache.
|
||||
*/
|
||||
clock_type&
|
||||
ClockType&
|
||||
clock();
|
||||
|
||||
/**
|
||||
@@ -239,7 +239,7 @@ public:
|
||||
bool
|
||||
retrieve(key_type const& key, T& data);
|
||||
|
||||
mutex_type&
|
||||
MutexType&
|
||||
peekMutex();
|
||||
|
||||
std::vector<key_type>
|
||||
@@ -265,7 +265,7 @@ public:
|
||||
|
||||
private:
|
||||
SharedPointerType
|
||||
initialFetch(key_type const& key, std::scoped_lock<mutex_type> const& l);
|
||||
initialFetch(key_type const& key, std::scoped_lock<MutexType> const& l);
|
||||
|
||||
void
|
||||
collectMetrics();
|
||||
@@ -277,7 +277,7 @@ private:
|
||||
Stats(
|
||||
std::string const& prefix,
|
||||
Handler const& handler,
|
||||
beast::insight::Collector::ptr const& collector)
|
||||
beast::insight::Collector::Ptr const& collector)
|
||||
: hook(collector->makeHook(handler))
|
||||
, size(collector->makeGauge(prefix, "size"))
|
||||
, hitRate(collector->makeGauge(prefix, "hit_rate"))
|
||||
@@ -296,14 +296,14 @@ private:
|
||||
class KeyOnlyEntry
|
||||
{
|
||||
public:
|
||||
clock_type::time_point lastAccess;
|
||||
ClockType::time_point lastAccess;
|
||||
|
||||
explicit KeyOnlyEntry(clock_type::time_point const& lastAccess) : lastAccess(lastAccess)
|
||||
explicit KeyOnlyEntry(ClockType::time_point const& lastAccess) : lastAccess(lastAccess)
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
touch(clock_type::time_point const& now)
|
||||
touch(ClockType::time_point const& now)
|
||||
{
|
||||
lastAccess = now;
|
||||
}
|
||||
@@ -312,10 +312,10 @@ private:
|
||||
class ValueEntry
|
||||
{
|
||||
public:
|
||||
shared_weak_combo_pointer_type ptr;
|
||||
clock_type::time_point lastAccess;
|
||||
SharedWeakComboPointerType ptr;
|
||||
ClockType::time_point lastAccess;
|
||||
|
||||
ValueEntry(clock_type::time_point const& lastAccess, shared_pointer_type const& ptr)
|
||||
ValueEntry(ClockType::time_point const& lastAccess, SharedPointerType const& ptr)
|
||||
: ptr(ptr), lastAccess(lastAccess)
|
||||
{
|
||||
}
|
||||
@@ -343,7 +343,7 @@ private:
|
||||
return ptr.lock();
|
||||
}
|
||||
void
|
||||
touch(clock_type::time_point const& now)
|
||||
touch(ClockType::time_point const& now)
|
||||
{
|
||||
lastAccess = now;
|
||||
}
|
||||
@@ -351,35 +351,35 @@ private:
|
||||
|
||||
using Entry = std::conditional_t<IsKeyCache, KeyOnlyEntry, ValueEntry>;
|
||||
|
||||
using KeyOnlyCacheType = hardened_partitioned_hash_map<key_type, KeyOnlyEntry, Hash, KeyEqual>;
|
||||
using KeyOnlyCacheType = HardenedPartitionedHashMap<key_type, KeyOnlyEntry, Hash, KeyEqual>;
|
||||
|
||||
using KeyValueCacheType = hardened_partitioned_hash_map<key_type, ValueEntry, Hash, KeyEqual>;
|
||||
using KeyValueCacheType = HardenedPartitionedHashMap<key_type, ValueEntry, Hash, KeyEqual>;
|
||||
|
||||
using cache_type = hardened_partitioned_hash_map<key_type, Entry, Hash, KeyEqual>;
|
||||
using CacheType = HardenedPartitionedHashMap<key_type, Entry, Hash, KeyEqual>;
|
||||
|
||||
[[nodiscard]] std::thread
|
||||
sweepHelper(
|
||||
clock_type::time_point const& whenExpire,
|
||||
[[maybe_unused]] clock_type::time_point const& now,
|
||||
KeyValueCacheType::map_type& partition,
|
||||
ClockType::time_point const& whenExpire,
|
||||
[[maybe_unused]] ClockType::time_point const& now,
|
||||
KeyValueCacheType::MapType& partition,
|
||||
SweptPointersVector& stuffToSweep,
|
||||
std::atomic<int>& allRemovals,
|
||||
std::scoped_lock<std::recursive_mutex> const&);
|
||||
|
||||
[[nodiscard]] std::thread
|
||||
sweepHelper(
|
||||
clock_type::time_point const& whenExpire,
|
||||
clock_type::time_point const& now,
|
||||
KeyOnlyCacheType::map_type& partition,
|
||||
ClockType::time_point const& whenExpire,
|
||||
ClockType::time_point const& now,
|
||||
KeyOnlyCacheType::MapType& partition,
|
||||
SweptPointersVector&,
|
||||
std::atomic<int>& allRemovals,
|
||||
std::scoped_lock<std::recursive_mutex> const&);
|
||||
|
||||
beast::Journal journal_;
|
||||
clock_type& clock_;
|
||||
ClockType& clock_;
|
||||
Stats stats_;
|
||||
|
||||
mutex_type mutable mutex_;
|
||||
MutexType mutable mutex_;
|
||||
|
||||
// Used for logging
|
||||
std::string name_;
|
||||
@@ -388,11 +388,11 @@ private:
|
||||
int const targetSize_;
|
||||
|
||||
// Desired maximum cache age
|
||||
clock_type::duration const targetAge_;
|
||||
ClockType::duration const targetAge_;
|
||||
|
||||
// Number of items cached
|
||||
int cacheCount_{0};
|
||||
cache_type cache_; // Hold strong reference to recent objects
|
||||
CacheType cache_; // Hold strong reference to recent objects
|
||||
std::uint64_t hits_{0};
|
||||
std::uint64_t misses_{0};
|
||||
};
|
||||
|
||||
Loaded 100 of 1259 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user