From 0072ced94c73c2bb13eb415b46bacfea4164dce8 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Thu, 23 Jul 2026 10:22:18 +0000 Subject: [PATCH 01/13] ci: Update XRPLF/actions (#7849) --- .github/workflows/build-nix-images.yml | 2 +- .github/workflows/build-packaging-images.yml | 2 +- .github/workflows/build-pre-commit-image.yml | 2 +- .github/workflows/check-pr-title.yml | 2 +- .github/workflows/pre-commit.yml | 2 +- .github/workflows/publish-docs.yml | 2 +- .github/workflows/reusable-build-test-config.yml | 2 +- .github/workflows/reusable-clang-tidy.yml | 2 +- .github/workflows/upload-conan-deps.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-nix-images.yml b/.github/workflows/build-nix-images.yml index fd480cff67..8574182a7e 100644 --- a/.github/workflows/build-nix-images.yml +++ b/.github/workflows/build-nix-images.yml @@ -56,7 +56,7 @@ jobs: base_image: debian:bookworm - name: rhel base_image: registry.access.redhat.com/ubi9/ubi:latest - uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@ee03d31bcc4501d7599dc1b1ecd7a34af582ad1c + uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32 with: image_name: xrpld/nix-${{ matrix.distro.name }} dockerfile: nix/docker/Dockerfile diff --git a/.github/workflows/build-packaging-images.yml b/.github/workflows/build-packaging-images.yml index e72ea876a7..43b276bdf1 100644 --- a/.github/workflows/build-packaging-images.yml +++ b/.github/workflows/build-packaging-images.yml @@ -38,7 +38,7 @@ jobs: base_image: debian:bookworm - name: rhel base_image: registry.access.redhat.com/ubi9/ubi:latest - uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@ee03d31bcc4501d7599dc1b1ecd7a34af582ad1c + uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32 with: image_name: xrpld/packaging-${{ matrix.distro.name }} dockerfile: package/Dockerfile diff --git a/.github/workflows/build-pre-commit-image.yml b/.github/workflows/build-pre-commit-image.yml index 9b20dc7951..d0eba6b495 100644 --- a/.github/workflows/build-pre-commit-image.yml +++ b/.github/workflows/build-pre-commit-image.yml @@ -30,7 +30,7 @@ jobs: permissions: contents: read packages: write - uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@ee03d31bcc4501d7599dc1b1ecd7a34af582ad1c + uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32 with: image_name: xrpld/pre-commit dockerfile: bin/pre-commit/Dockerfile diff --git a/.github/workflows/check-pr-title.yml b/.github/workflows/check-pr-title.yml index 4b5f679df1..cc80982440 100644 --- a/.github/workflows/check-pr-title.yml +++ b/.github/workflows/check-pr-title.yml @@ -20,4 +20,4 @@ on: jobs: check_title: if: ${{ github.event.pull_request.draft != true }} - uses: XRPLF/actions/.github/workflows/check-pr-title.yml@cba1f0891650baf1a9c88624dc2d72573be2eb81 + uses: XRPLF/actions/.github/workflows/check-pr-title.yml@d7c65e49225a38f6d8010eacf017bb5a98d7476c diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 9970e9a07d..ac5fe46722 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -14,7 +14,7 @@ on: jobs: # Call the workflow in the XRPLF/actions repo that runs the pre-commit hooks. run-hooks: - uses: XRPLF/actions/.github/workflows/pre-commit.yml@1bde119a1ab71305ba5d3716e7a82cea1c7bdede + uses: XRPLF/actions/.github/workflows/pre-commit.yml@3ba08d6ddf114092891d48491fc2e26c3ba15552 with: runs_on: ubuntu-latest container: '{ "image": "ghcr.io/xrplf/xrpld/pre-commit:sha-f56b79f" }' diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 90182e7cbb..f2ebc84732 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -47,7 +47,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Prepare runner - uses: XRPLF/actions/prepare-runner@ad188deb3dae79dc39816e16ddfdad1e06c6fab2 + uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606 with: enable_ccache: false diff --git a/.github/workflows/reusable-build-test-config.yml b/.github/workflows/reusable-build-test-config.yml index 69fa7a7bef..6372bb6328 100644 --- a/.github/workflows/reusable-build-test-config.yml +++ b/.github/workflows/reusable-build-test-config.yml @@ -113,7 +113,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Prepare runner - uses: XRPLF/actions/prepare-runner@ad188deb3dae79dc39816e16ddfdad1e06c6fab2 + uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606 with: enable_ccache: ${{ inputs.ccache_enabled }} diff --git a/.github/workflows/reusable-clang-tidy.yml b/.github/workflows/reusable-clang-tidy.yml index 4a10e4b0f7..615f8af198 100644 --- a/.github/workflows/reusable-clang-tidy.yml +++ b/.github/workflows/reusable-clang-tidy.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Prepare runner - uses: XRPLF/actions/prepare-runner@ad188deb3dae79dc39816e16ddfdad1e06c6fab2 + uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606 with: enable_ccache: false diff --git a/.github/workflows/upload-conan-deps.yml b/.github/workflows/upload-conan-deps.yml index abc0867b15..8a02c4c2db 100644 --- a/.github/workflows/upload-conan-deps.yml +++ b/.github/workflows/upload-conan-deps.yml @@ -68,7 +68,7 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Prepare runner - uses: XRPLF/actions/prepare-runner@ad188deb3dae79dc39816e16ddfdad1e06c6fab2 + uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606 with: enable_ccache: false From 74cfb3586a9be66dd1c65eb8730a86f314303b62 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 23 Jul 2026 11:21:49 +0000 Subject: [PATCH 02/13] ci: Update CI image (#7850) --- .github/scripts/strategy-matrix/linux.json | 2 +- .github/workflows/publish-docs.yml | 2 +- .github/workflows/reusable-clang-tidy.yml | 2 +- .github/workflows/reusable-upload-recipe.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/strategy-matrix/linux.json b/.github/scripts/strategy-matrix/linux.json index 7edbf96ef6..992f314686 100644 --- a/.github/scripts/strategy-matrix/linux.json +++ b/.github/scripts/strategy-matrix/linux.json @@ -1,5 +1,5 @@ { - "image_tag": "sha-2e25435", + "image_tag": "sha-3122de8", "configs": { "ubuntu": [ { diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index f2ebc84732..19c73f93d6 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -41,7 +41,7 @@ env: jobs: build: runs-on: ubuntu-latest - container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-2e25435 + container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-3122de8 steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/.github/workflows/reusable-clang-tidy.yml b/.github/workflows/reusable-clang-tidy.yml index 615f8af198..e81bbea367 100644 --- a/.github/workflows/reusable-clang-tidy.yml +++ b/.github/workflows/reusable-clang-tidy.yml @@ -34,7 +34,7 @@ jobs: needs: [determine-files] if: ${{ needs.determine-files.outputs.cpp_changed_files != '' || needs.determine-files.outputs.need_full_run == 'true' }} runs-on: ["self-hosted", "Linux", "X64", "heavy"] - container: "ghcr.io/xrplf/xrpld/nix-debian:sha-2e25435" + container: "ghcr.io/xrplf/xrpld/nix-debian:sha-3122de8" permissions: contents: read issues: write diff --git a/.github/workflows/reusable-upload-recipe.yml b/.github/workflows/reusable-upload-recipe.yml index 07077163ed..df4a2d9516 100644 --- a/.github/workflows/reusable-upload-recipe.yml +++ b/.github/workflows/reusable-upload-recipe.yml @@ -40,7 +40,7 @@ defaults: jobs: upload: runs-on: ubuntu-latest - container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-2e25435 + container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-3122de8 env: REMOTE_NAME: ${{ inputs.remote_name }} CONAN_LOGIN_USERNAME_XRPLF: ${{ secrets.remote_username }} From 95e1ffea6e7a3cb82f49a8371a1d7154b7ba011c Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Thu, 23 Jul 2026 17:49:44 +0100 Subject: [PATCH 03/13] ci: Add llvm-tools-preview to rust toolchain (#7853) --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 3206cf59c0..a82b4734d8 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] channel = "1.95" -components = ["rustfmt", "clippy", "rust-analyzer"] +components = ["rustfmt", "clippy", "rust-analyzer", "llvm-tools-preview"] profile = "minimal" From 38c54c3f36be1878c2bfbb7a86ebe4aee7c22141 Mon Sep 17 00:00:00 2001 From: Mayukha Vadari Date: Thu, 23 Jul 2026 14:50:59 -0400 Subject: [PATCH 04/13] feat: Add fixCleanup3_4_0 amendment (no functionality yet) (#7854) --- include/xrpl/protocol/detail/features.macro | 1 + 1 file changed, 1 insertion(+) diff --git a/include/xrpl/protocol/detail/features.macro b/include/xrpl/protocol/detail/features.macro index bfe03a6303..4f1fac82da 100644 --- a/include/xrpl/protocol/detail/features.macro +++ b/include/xrpl/protocol/detail/features.macro @@ -15,6 +15,7 @@ // Add new amendments to the top of this list. // Keep it sorted in reverse chronological order. +XRPL_FIX (Cleanup3_4_0, Supported::Yes, VoteBehavior::DefaultNo) XRPL_FEATURE(Sponsor, Supported::Yes, VoteBehavior::DefaultNo) XRPL_FEATURE(BatchV1_1, Supported::Yes, VoteBehavior::DefaultNo) XRPL_FEATURE(LendingProtocolV1_1, Supported::No, VoteBehavior::DefaultNo) From 40cdf49d155322595764ff9b7edc33f1f553ab83 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Thu, 23 Jul 2026 20:05:24 +0100 Subject: [PATCH 05/13] build: Use custom libc in a devshell by default (#7852) --- docs/build/nix.md | 30 ++++++++--- nix/ci-env.nix | 104 +++---------------------------------- nix/compilers.nix | 117 ++++++++++++++++++++++++++++++++++++++++++ nix/devshell.nix | 91 +++++++++++++++++++++++++------- nix/docker/Dockerfile | 1 + nix/packages.nix | 12 +++++ 6 files changed, 234 insertions(+), 121 deletions(-) create mode 100644 nix/compilers.nix diff --git a/docs/build/nix.md b/docs/build/nix.md index d6e53a254a..b95b82fc42 100644 --- a/docs/build/nix.md +++ b/docs/build/nix.md @@ -38,8 +38,10 @@ The first time you run this command, it will take a few minutes to download and ### Platform notes -- **Linux**: `nix develop` gives you a shell with all the tooling necessary to - develop xrpld and with GCC 15.2 (also provided by Nix). There are no caveats. +- **Linux**: `nix develop` gives you a shell with all the tooling necessary to develop xrpld + and with the same GCC/glibc toolchain that Nix builds for CI. + See [Choosing a different compiler](#choosing-a-different-compiler) + for the custom-vs-plain toolchain trade-off. - **macOS**: `nix develop` gives you a full environment too, with Clang (and every other tool, including Conan) provided by Nix. To use your system-wide Apple Clang instead, enter `nix develop .#apple-clang`. Conan has no binary in @@ -63,8 +65,16 @@ The first time you run this command, it will take a few minutes to download and ### Choosing a different compiler A compiler can be chosen by providing its name with the `.#` prefix, e.g. `nix develop .#clang`. -The `.#gcc` and `.#clang` shells provide the same GCC and Clang versions used in CI -(pinned in [`nix/packages.nix`](../../nix/packages.nix)). + +On Linux, `.#gcc` and `.#clang` provide the exact toolchain CI uses: +the compiler (pinned in [`nix/packages.nix`](../../nix/packages.nix)) +rebuilt against the pinned custom glibc (see [`nix/compilers.nix`](../../nix/compilers.nix)). +Building that toolchain the first time is slow unless it is fetched from a Nix binary cache. +If you don't need the custom glibc, the Linux-only `.#gcc-plain` and `.#clang-plain` +give you the stock nixpkgs compilers of the same versions. +On macOS there is no custom glibc, so `.#gcc` and `.#clang` are already the plain nixpkgs toolchain, +and the `-plain` variants do not exist. + Use `nix flake show` to see all the available development shells. Use `nix develop .#no-compiler` to use the compiler from your system. @@ -72,14 +82,18 @@ Use `nix develop .#no-compiler` to use the compiler from your system. ### Example Usage ```bash -# Use GCC (same version as CI) +# Use GCC — same toolchain as CI (custom glibc on Linux) nix develop .#gcc -# Use Clang (same version as CI) +# Use Clang — same toolchain as CI (custom glibc on Linux) nix develop .#clang # Use default for your platform nix develop + +# Stock nixpkgs GCC/Clang, Linux only — skips the custom-glibc build, but does not match CI +nix develop .#gcc-plain +nix develop .#clang-plain ``` ### Using a different shell @@ -110,6 +124,10 @@ nix develop -c "$SHELL" Once inside the Nix development shell, follow the standard [build instructions](../../BUILD.md#steps). The Nix shell provides all necessary tools (CMake, Ninja, Conan, etc.). +Coverage builds (`-Dcoverage=ON`) work in the `gcc` shell (and `gcc-plain` on Linux): +each ships a `gcov` matching its compiler, since Nix's cc-wrapper does not expose one. +The `clang` shells do not include `llvm-cov`, so use a `gcc` shell for coverage. + ## Automatic Activation with direnv [direnv](https://direnv.net/) or [nix-direnv](https://github.com/nix-community/nix-direnv) can automatically activate the Nix development shell when you enter the repository directory. diff --git a/nix/ci-env.nix b/nix/ci-env.nix index 63bddb46d8..787b94406e 100644 --- a/nix/ci-env.nix +++ b/nix/ci-env.nix @@ -6,108 +6,18 @@ let inherit (import ./packages.nix { inherit pkgs; }) commonPackages - gccPackage gccVersion - llvmPackages llvmVersion mkVersionedToolLinks ; - # Underlying compiler toolchains to wrap (versions pinned in packages.nix). - customGccPackage = gccPackage; - customLlvmPackages = llvmPackages; - - # binutils wrapped to emit binaries that reference the custom glibc - # (dynamic linker path, library search path, RPATH). - customBinutils = pkgs.wrapBintoolsWith { - bintools = pkgs.binutils-unwrapped; - libc = customGlibc; - }; - - # Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom - # glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses - # the existing gcc binary but links against the custom glibc, so the - # resulting compiler ships runtime libraries that only reference symbols - # available in that glibc. - customGccCc = customGccPackage.cc.override { - stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv ( - pkgs.wrapCCWith { - cc = customGccPackage.cc; - libc = customGlibc; - bintools = customBinutils; - } - ); - }; - - # cc-wrapper around the rebuilt compiler, pointing at the custom glibc - # headers and libraries. This is what we actually expose to users. - customGcc = pkgs.wrapCCWith { - cc = customGccCc; - libc = customGlibc; - bintools = customBinutils; - }; - - # gcov ships in gcc's `cc` output, but the cc-wrapper doesn't expose it. - # Surface the gcov from our rebuilt gcc (linked against the custom glibc, so - # it runs under the loader installed in the image) and matching the exact - # compiler version, so gcovr can produce coverage reports in the CI env. - customGcov = pkgs.runCommand "gcov-custom-for-ci-env" { } '' - mkdir -p "$out/bin" - ln -s "${customGccCc}/bin/gcov" "$out/bin/gcov" - ''; - - # stdenv built around the rebuilt gcc / custom glibc. Used to rebuild - # compiler-rt below so its sanitizer runtimes see the custom glibc - # headers. - customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc; - - # Rebuild compiler-rt against the custom glibc so the sanitizer runtimes - # don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that - # only exist in newer glibc versions. scudo is dropped because its CMake - # includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which - # makes std::atomic unfindable in our stdenv; we don't use scudo (only - # asan/ubsan/tsan etc.). - customCompilerRt = - (customLlvmPackages.compiler-rt.override { - stdenv = customStdenv; - }).overrideAttrs - (old: { - postPatch = (old.postPatch or "") + '' - substituteInPlace lib/CMakeLists.txt \ - --replace-quiet 'add_subdirectory(scudo/standalone)' \ - '# scudo/standalone disabled in xrpld ci-env' - ''; - }); - - # cc-wrapper around clang, pointing at the custom glibc headers and - # libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that - # C++ binaries produced by clang also only reference symbols available - # in the custom glibc. compiler-rt is wired into a resource-root so - # sanitizer runtimes (libclang_rt.*.a) are found at link time; this - # mirrors what nixpkgs does internally when building llvmPackages.clang. - customClang = pkgs.wrapCCWith { - cc = customLlvmPackages.clang-unwrapped; - libc = customGlibc; - bintools = customBinutils; - gccForLibs = customGccCc; - extraPackages = [ customCompilerRt ]; - extraBuildCommands = '' - rsrc="$out/resource-root" - mkdir "$rsrc" - ln -s "${customLlvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include" - ln -s "${customCompilerRt.out}/lib" "$rsrc/lib" - ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true - echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags - # compiler-rt ships the sanitizer/profile/xray interface headers (e.g. - # ) in its `dev` output. In a normal Nix - # build these reach the include path because compiler-rt is propagated - # via depsTargetTargetPropagated and stdenv's setup hooks add its - # dev/include. The CI image runs clang outside a Nix stdenv (binaries - # on PATH, no setup hooks), so that never happens; add the headers - # explicitly. gcc ships its own copy, which is why this is clang-only. - echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags - ''; - }; + # Custom-glibc toolchain, shared with the Linux dev shell (see compilers.nix). + inherit (import ./compilers.nix { inherit pkgs customGlibc; }) + customGcc + customClang + customBinutils + customGcov + ; # Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can # coexist with the gcc wrapper in buildEnv. gcc remains the default diff --git a/nix/compilers.nix b/nix/compilers.nix new file mode 100644 index 0000000000..90856afacc --- /dev/null +++ b/nix/compilers.nix @@ -0,0 +1,117 @@ +# Custom-glibc compiler toolchain shared by the CI environment (ci-env.nix) and +# the Linux dev shell (devshell.nix): gcc / clang / binutils rebuilt to target +# the pinned custom glibc. Linux only — the pinned glibc snapshot does not build +# on darwin, so callers must not evaluate this on macOS. +{ + pkgs, + customGlibc, +}: +let + inherit (import ./packages.nix { inherit pkgs; }) + gccPackage + llvmPackages + llvmVersion + mkGcov + ; + + # binutils wrapped to emit binaries that reference the custom glibc + # (dynamic linker path, library search path, RPATH). + customBinutils = pkgs.wrapBintoolsWith { + bintools = pkgs.binutils-unwrapped; + libc = customGlibc; + }; + + # Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom + # glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses + # the existing gcc binary but links against the custom glibc, so the + # resulting compiler ships runtime libraries that only reference symbols + # available in that glibc. + customGccCc = gccPackage.cc.override { + stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv ( + pkgs.wrapCCWith { + cc = gccPackage.cc; + libc = customGlibc; + bintools = customBinutils; + } + ); + }; + + # cc-wrapper around the rebuilt compiler, pointing at the custom glibc + # headers and libraries. This is what we actually expose to users. + customGcc = pkgs.wrapCCWith { + cc = customGccCc; + libc = customGlibc; + bintools = customBinutils; + }; + + # gcov matching the rebuilt gcc (linked against the custom glibc), so gcovr + # can produce coverage reports both in CI and in the dev shell. + customGcov = mkGcov { + name = "custom"; + cc = customGccCc; + }; + + # stdenv built around the rebuilt gcc / custom glibc. Exported as the dev + # shell's gcc stdenv, and used below to rebuild compiler-rt so its sanitizer + # runtimes see the custom glibc headers. + customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc; + + # Rebuild compiler-rt against the custom glibc so the sanitizer runtimes + # don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that + # only exist in newer glibc versions. scudo is dropped because its CMake + # includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which + # makes std::atomic unfindable in our stdenv; we don't use scudo (only + # asan/ubsan/tsan etc.). + customCompilerRt = + (llvmPackages.compiler-rt.override { + stdenv = customStdenv; + }).overrideAttrs + (old: { + postPatch = (old.postPatch or "") + '' + substituteInPlace lib/CMakeLists.txt \ + --replace-quiet 'add_subdirectory(scudo/standalone)' \ + '# scudo/standalone disabled in xrpld ci-env' + ''; + }); + + # cc-wrapper around clang, pointing at the custom glibc headers and + # libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that + # C++ binaries produced by clang also only reference symbols available + # in the custom glibc. compiler-rt is wired into a resource-root so + # sanitizer runtimes (libclang_rt.*.a) are found at link time; this + # mirrors what nixpkgs does internally when building llvmPackages.clang. + customClang = pkgs.wrapCCWith { + cc = llvmPackages.clang-unwrapped; + libc = customGlibc; + bintools = customBinutils; + gccForLibs = customGccCc; + extraPackages = [ customCompilerRt ]; + extraBuildCommands = '' + rsrc="$out/resource-root" + mkdir "$rsrc" + ln -s "${llvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include" + ln -s "${customCompilerRt.out}/lib" "$rsrc/lib" + ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true + echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags + # compiler-rt ships the sanitizer/profile/xray interface headers (e.g. + # ) in its `dev` output. In a normal Nix + # build these reach the include path because compiler-rt is propagated + # via depsTargetTargetPropagated and stdenv's setup hooks add its + # dev/include. The CI image runs clang outside a Nix stdenv (binaries + # on PATH, no setup hooks), so that never happens; add the headers + # explicitly. gcc ships its own copy, which is why this is clang-only. + echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags + ''; + }; +in +{ + inherit + customGcc + customClang + customBinutils + customStdenv + customGcov + ; + + customClangStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customClang; +} diff --git a/nix/devshell.nix b/nix/devshell.nix index 1316fe4234..9b453ddef8 100644 --- a/nix/devshell.nix +++ b/nix/devshell.nix @@ -1,16 +1,51 @@ -{ pkgs, ... }: +{ pkgs, customGlibc, ... }: let inherit (import ./packages.nix { inherit pkgs; }) commonPackages + gccPackage gccVersion llvmVersion llvmPackages mkVersionedToolLinks + mkGcov ; - # Plain nixpkgs stdenvs — no custom glibc, unlike ci-env.nix. - gccStdenv = pkgs."gcc${toString gccVersion}Stdenv"; - clangStdenv = llvmPackages.stdenv; + # Plain nixpkgs stdenvs — no custom glibc. + plainGccStdenv = pkgs."gcc${toString gccVersion}Stdenv"; + plainClangStdenv = llvmPackages.stdenv; + + # Custom-glibc stdenvs, matching the CI environment (see compilers.nix). The + # pinned glibc snapshot only builds on Linux, so on darwin these fall back to + # the plain stdenvs; the `if isLinux` guard keeps `customGlibc` from being + # forced (and erroring) on macOS. + customCompilers = import ./compilers.nix { inherit pkgs customGlibc; }; + customGccStdenv = if pkgs.stdenv.isLinux then customCompilers.customStdenv else plainGccStdenv; + customClangStdenv = + if pkgs.stdenv.isLinux then customCompilers.customClangStdenv else plainClangStdenv; + + # gcov matching each gcc shell, so `-Dcoverage=ON` builds work in the shell. + plainGcov = mkGcov { + name = "plain"; + cc = gccPackage.cc; + }; + customGccGcov = if pkgs.stdenv.isLinux then customCompilers.customGcov else plainGcov; + + # Shown when entering a *-plain shell. These exist only on Linux (see below), + # where the stock toolchain diverges from CI. + plainWarningHook = '' + echo "⚠️ WARNING: this is the stock nixpkgs toolchain and does not match CI's glibc. Prefer 'nix develop .#gcc' / '.#clang' unless you need to skip the custom-glibc build." + ''; + + # Tools to expose under version-suffixed names (see mkVersionedToolLinks). + gccVersionedTools = [ + "gcc" + "g++" + "cpp" + ]; + clangVersionedTools = [ + "clang" + "clang++" + ]; # compilerName is the command used to print the version, or null for none. makeShell = @@ -19,9 +54,11 @@ let compilerName, version ? null, versionedTools ? [ ], + extraPackages ? [ ], + warningHook ? "", }: let - compilerVersion = + compilerVersionHook = if compilerName == null then ''echo "No compiler specified - using system compiler"'' else @@ -37,10 +74,11 @@ let }); in (pkgs.mkShell.override { inherit stdenv; }) { - packages = commonPackages ++ versionedLinks; + packages = commonPackages ++ versionedLinks ++ extraPackages; shellHook = '' echo "Welcome to xrpld development shell"; - ${compilerVersion} + ${compilerVersionHook} + ${warningHook} ''; }; in @@ -48,25 +86,21 @@ rec { # macOS: Nix Clang. Linux: Nix GCC. default = if pkgs.stdenv.isDarwin then clang else gcc; + # gcc/clang use the custom-glibc toolchain, matching CI. On darwin there is no + # custom glibc, so they fall back to the plain nixpkgs toolchain. gcc = makeShell { - stdenv = gccStdenv; + stdenv = customGccStdenv; compilerName = "gcc"; version = gccVersion; - versionedTools = [ - "gcc" - "g++" - "cpp" - ]; + versionedTools = gccVersionedTools; + extraPackages = [ customGccGcov ]; }; clang = makeShell { - stdenv = clangStdenv; + stdenv = customClangStdenv; compilerName = "clang"; version = llvmVersion; - versionedTools = [ - "clang" - "clang++" - ]; + versionedTools = clangVersionedTools; }; # Nix provides no compiler; use the one from your system (e.g. Apple Clang). @@ -76,3 +110,24 @@ rec { }; apple-clang = no-compiler; } +# The *-plain shells (stock nixpkgs toolchain) exist only on Linux: on darwin +# gcc/clang are already plain, so these would be redundant and are omitted, which +# makes `nix develop .#gcc-plain` fail there rather than silently aliasing gcc. +// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { + gcc-plain = makeShell { + stdenv = plainGccStdenv; + compilerName = "gcc"; + version = gccVersion; + versionedTools = gccVersionedTools; + extraPackages = [ plainGcov ]; + warningHook = plainWarningHook; + }; + + clang-plain = makeShell { + stdenv = plainClangStdenv; + compilerName = "clang"; + version = llvmVersion; + versionedTools = clangVersionedTools; + warningHook = plainWarningHook; + }; +} diff --git a/nix/docker/Dockerfile b/nix/docker/Dockerfile index 8b851fd9e0..53b646ac7c 100644 --- a/nix/docker/Dockerfile +++ b/nix/docker/Dockerfile @@ -8,6 +8,7 @@ RUN mkdir -p ~/.config/nix && \ # Copy our source and setup our working dir. COPY nix/ci-env.nix /tmp/build/nix/ci-env.nix +COPY nix/compilers.nix /tmp/build/nix/compilers.nix COPY nix/packages.nix /tmp/build/nix/packages.nix COPY nix/utils.nix /tmp/build/nix/utils.nix COPY flake.nix /tmp/build/ diff --git a/nix/packages.nix b/nix/packages.nix index 1d9b6cd2f8..3cf0f57c3e 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -48,6 +48,17 @@ let }) tools ); + # The cc-wrapper doesn't re-export gcov, but coverage tooling (gcovr) needs a + # gcov that exactly matches the compiler. Surface it from a gcc `cc` output. + mkGcov = + { name, cc }: + pkgs.linkFarm "gcov-${name}" [ + { + name = "bin/gcov"; + path = "${cc}/bin/gcov"; + } + ]; + clangToolLinks = mkVersionedToolLinks { name = "clang-tools"; package = clangTools; @@ -72,6 +83,7 @@ in gccPackage llvmPackages mkVersionedToolLinks + mkGcov ; commonPackages = with pkgs; [ From b89d75a2d504c58f71813dc5b3d2437567193113 Mon Sep 17 00:00:00 2001 From: Ed Hennis Date: Thu, 23 Jul 2026 16:57:00 -0400 Subject: [PATCH 06/13] test: Add an RAII class to manage the env.parseFailureExpected flag (#7669) --- src/test/app/Batch_test.cpp | 3 +-- src/test/app/Vault_test.cpp | 3 +-- src/test/jtx/Env.h | 43 +++++++++++++++++++++++++++++++++++++ src/test/jtx/impl/Env.cpp | 2 +- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/test/app/Batch_test.cpp b/src/test/app/Batch_test.cpp index 5085ad6172..ffaf26b5a7 100644 --- a/src/test/app/Batch_test.cpp +++ b/src/test/app/Batch_test.cpp @@ -498,7 +498,7 @@ class Batch_test : public beast::unit_test::Suite auto const batchFee = batch::calcBatchFee(env, 0, 2); auto tx1 = batch::Inner(pay(alice, bob, XRP(1)), seq + 1); tx1[jss::Fee] = "1.5"; - env.setParseFailureExpected(true); + auto const g = env.getParseFailureGuard(true); try { env(batch::outer(alice, seq, batchFee, tfAllOrNothing), @@ -510,7 +510,6 @@ class Batch_test : public beast::unit_test::Suite { BEAST_EXPECT(true); } - env.setParseFailureExpected(false); } // temSEQ_AND_TICKET: Batch: inner txn cannot have both Sequence diff --git a/src/test/app/Vault_test.cpp b/src/test/app/Vault_test.cpp index 617820c89c..12ad7e6782 100644 --- a/src/test/app/Vault_test.cpp +++ b/src/test/app/Vault_test.cpp @@ -5524,9 +5524,9 @@ class Vault_test : public beast::unit_test::Suite env.close(); // 2. Mantissa larger than uint64 max - env.setParseFailureExpected(true); try { + auto const g = env.getParseFailureGuard(true); tx[sfAssetsMaximum] = "18446744073709551617e5"; // uint64 max + 1 env(tx); BEAST_EXPECTS(false, "Expected parse_error for mantissa larger than uint64 max"); @@ -5537,7 +5537,6 @@ class Vault_test : public beast::unit_test::Suite BEAST_EXPECT( e.what() == "invalidParamsField 'tx_json.AssetsMaximum' has invalid data."s); } - env.setParseFailureExpected(false); } } diff --git a/src/test/jtx/Env.h b/src/test/jtx/Env.h index 7e22cdd571..0df62c7e9b 100644 --- a/src/test/jtx/Env.h +++ b/src/test/jtx/Env.h @@ -514,6 +514,49 @@ public: parseFailureExpected_ = b; } + /** + * RAII class to set and restore the parse failure flag (setParseFailureExpected). + * + * Can be created directly, or through the `getParseFailureGuard(bool)` function. + */ + class ParseFailureGuard final + { + Env& self_; + bool const oldExpected_; + + public: + ParseFailureGuard(Env& self, bool b) + : self_(self), oldExpected_(self_.parseFailureExpected_) + { + self_.setParseFailureExpected(b); + } + + ~ParseFailureGuard() + { + self_.setParseFailureExpected(oldExpected_); + } + + // No copy, no move + ParseFailureGuard(ParseFailureGuard const&) = delete; + ParseFailureGuard& + operator=(ParseFailureGuard const&) = delete; + ParseFailureGuard(ParseFailureGuard&& other) = delete; + ParseFailureGuard& + operator=(ParseFailureGuard&&) = delete; + }; + + /** + * Gets an RAII guard to set and restore the parse failure flag + * + * Usage: + * auto const guard = env.getParseFailureGuard(true/false); + */ + [[nodiscard]] ParseFailureGuard + getParseFailureGuard(bool b) + { + return ParseFailureGuard{*this, b}; + } + /** * Turn off signature checks. */ diff --git a/src/test/jtx/impl/Env.cpp b/src/test/jtx/impl/Env.cpp index 3f6aca9fcb..4da2e2b521 100644 --- a/src/test/jtx/impl/Env.cpp +++ b/src/test/jtx/impl/Env.cpp @@ -629,7 +629,7 @@ Env::autofill(JTx& jt) catch (ParseError const&) { if (!parseFailureExpected_) - test.log << "parse failed:\n" << pretty(jv) << std::endl; + test.log << "parse failure:\n" << pretty(jv) << std::endl; rethrow(); } } From 4acccfeda8a097dd5355b16095f3950153563f12 Mon Sep 17 00:00:00 2001 From: Marek Foss Date: Thu, 23 Jul 2026 22:00:06 +0100 Subject: [PATCH 07/13] test: Modularize Peerfinder component and migrate Peerfinder tests from Beast to GTest and GMock (#7054) Co-authored-by: Alex Kremer --- .../scripts/levelization/results/loops.txt | 3 - .../scripts/levelization/results/ordering.txt | 20 +- cmake/XrplCore.cmake | 7 + .../detail/aged_unordered_container.h | 1 + include/xrpl/peerfinder/Config.h | 163 +++ include/xrpl/peerfinder/PeerfinderManager.h | 179 +++ {src/xrpld => include/xrpl}/peerfinder/Slot.h | 0 include/xrpl/peerfinder/Types.h | 46 + .../xrpl}/peerfinder/detail/Bootcache.h | 6 +- .../xrpl}/peerfinder/detail/Checker.h | 0 .../xrpl}/peerfinder/detail/Counts.h | 7 +- .../xrpl}/peerfinder/detail/Fixed.h | 4 +- .../xrpl}/peerfinder/detail/Handouts.h | 7 +- .../xrpl}/peerfinder/detail/Livecache.h | 18 +- .../xrpl}/peerfinder/detail/Logic.h | 95 +- .../xrpl}/peerfinder/detail/SlotImp.h | 7 +- .../xrpl}/peerfinder/detail/Source.h | 3 +- .../xrpl}/peerfinder/detail/SourceStrings.h | 2 +- .../xrpl}/peerfinder/detail/Store.h | 0 .../xrpl}/peerfinder/detail/Tuning.h | 0 include/xrpl/peerfinder/make_Manager.h | 36 + .../peerfinder}/Bootcache.cpp | 38 +- src/libxrpl/peerfinder/Config.cpp | 135 ++ .../peerfinder}/Endpoint.cpp | 4 +- .../peerfinder}/PeerfinderManager.cpp | 32 +- .../detail => libxrpl/peerfinder}/SlotImp.cpp | 10 +- .../peerfinder}/SourceStrings.cpp | 5 +- src/test/overlay/TMGetObjectByHash_test.cpp | 2 +- src/test/overlay/tx_reduce_relay_test.cpp | 2 +- src/test/peerfinder/Livecache_test.cpp | 212 --- src/test/peerfinder/PeerFinder_test.cpp | 789 ---------- src/tests/libxrpl/CMakeLists.txt | 3 +- src/tests/libxrpl/main.cpp | 3 +- src/tests/libxrpl/peerfinder/Livecache.cpp | 294 ++++ src/tests/libxrpl/peerfinder/PeerFinder.cpp | 1270 +++++++++++++++++ src/xrpld/app/rdb/PeerFinder.h | 3 +- src/xrpld/app/rdb/detail/PeerFinder.cpp | 3 +- src/xrpld/overlay/detail/ConnectAttempt.cpp | 4 +- src/xrpld/overlay/detail/ConnectAttempt.h | 2 +- src/xrpld/overlay/detail/OverlayImpl.cpp | 11 +- src/xrpld/overlay/detail/OverlayImpl.h | 6 +- src/xrpld/overlay/detail/PeerImp.cpp | 4 +- src/xrpld/overlay/detail/PeerImp.h | 4 +- src/xrpld/peerfinder/PeerfinderManager.h | 365 +---- .../peerfinder/detail/PeerfinderConfig.cpp | 113 +- src/xrpld/peerfinder/detail/StoreSqdb.h | 2 +- src/xrpld/peerfinder/detail/iosformat.h | 201 --- src/xrpld/peerfinder/make_Manager.h | 26 - 48 files changed, 2308 insertions(+), 1839 deletions(-) create mode 100644 include/xrpl/peerfinder/Config.h create mode 100644 include/xrpl/peerfinder/PeerfinderManager.h rename {src/xrpld => include/xrpl}/peerfinder/Slot.h (100%) create mode 100644 include/xrpl/peerfinder/Types.h rename {src/xrpld => include/xrpl}/peerfinder/detail/Bootcache.h (97%) rename {src/xrpld => include/xrpl}/peerfinder/detail/Checker.h (100%) rename {src/xrpld => include/xrpl}/peerfinder/detail/Counts.h (98%) rename {src/xrpld => include/xrpl}/peerfinder/detail/Fixed.h (92%) rename {src/xrpld => include/xrpl}/peerfinder/detail/Handouts.h (98%) rename {src/xrpld => include/xrpl}/peerfinder/detail/Livecache.h (95%) rename {src/xrpld => include/xrpl}/peerfinder/detail/Logic.h (91%) rename {src/xrpld => include/xrpl}/peerfinder/detail/SlotImp.h (96%) rename {src/xrpld => include/xrpl}/peerfinder/detail/Source.h (95%) rename {src/xrpld => include/xrpl}/peerfinder/detail/SourceStrings.h (90%) rename {src/xrpld => include/xrpl}/peerfinder/detail/Store.h (100%) rename {src/xrpld => include/xrpl}/peerfinder/detail/Tuning.h (100%) create mode 100644 include/xrpl/peerfinder/make_Manager.h rename src/{xrpld/peerfinder/detail => libxrpl/peerfinder}/Bootcache.cpp (80%) create mode 100644 src/libxrpl/peerfinder/Config.cpp rename src/{xrpld/peerfinder/detail => libxrpl/peerfinder}/Endpoint.cpp (76%) rename src/{xrpld/peerfinder/detail => libxrpl/peerfinder}/PeerfinderManager.cpp (92%) rename src/{xrpld/peerfinder/detail => libxrpl/peerfinder}/SlotImp.cpp (93%) rename src/{xrpld/peerfinder/detail => libxrpl/peerfinder}/SourceStrings.cpp (93%) delete mode 100644 src/test/peerfinder/Livecache_test.cpp delete mode 100644 src/test/peerfinder/PeerFinder_test.cpp create mode 100644 src/tests/libxrpl/peerfinder/Livecache.cpp create mode 100644 src/tests/libxrpl/peerfinder/PeerFinder.cpp delete mode 100644 src/xrpld/peerfinder/detail/iosformat.h delete mode 100644 src/xrpld/peerfinder/make_Manager.h diff --git a/.github/scripts/levelization/results/loops.txt b/.github/scripts/levelization/results/loops.txt index cf70468e32..ea7b8a372a 100644 --- a/.github/scripts/levelization/results/loops.txt +++ b/.github/scripts/levelization/results/loops.txt @@ -1,9 +1,6 @@ Loop: xrpld.app xrpld.overlay xrpld.app > xrpld.overlay -Loop: xrpld.app xrpld.peerfinder - xrpld.peerfinder ~= xrpld.app - Loop: xrpld.app xrpld.rpc xrpld.rpc > xrpld.app diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index 3c9c514516..fdd134dc8a 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -25,6 +25,9 @@ libxrpl.nodestore > xrpl.config libxrpl.nodestore > xrpl.json libxrpl.nodestore > xrpl.nodestore libxrpl.nodestore > xrpl.protocol +libxrpl.peerfinder > xrpl.basics +libxrpl.peerfinder > xrpl.peerfinder +libxrpl.peerfinder > xrpl.protocol libxrpl.protocol > xrpl.basics libxrpl.protocol > xrpl.json libxrpl.protocol > xrpl.protocol @@ -146,19 +149,13 @@ test.overlay > xrpl.config test.overlay > xrpld.app test.overlay > xrpld.core test.overlay > xrpld.overlay -test.overlay > xrpld.peerfinder test.overlay > xrpl.json test.overlay > xrpl.nodestore +test.overlay > xrpl.peerfinder test.overlay > xrpl.protocol test.overlay > xrpl.resource test.overlay > xrpl.server test.overlay > xrpl.shamap -test.peerfinder > test.beast -test.peerfinder > test.unit_test -test.peerfinder > xrpl.basics -test.peerfinder > xrpld.core -test.peerfinder > xrpld.peerfinder -test.peerfinder > xrpl.protocol test.protocol > test.jtx test.protocol > test.unit_test test.protocol > xrpl.basics @@ -197,6 +194,7 @@ tests.libxrpl > xrpl.json tests.libxrpl > xrpl.ledger tests.libxrpl > xrpl.net tests.libxrpl > xrpl.nodestore +tests.libxrpl > xrpl.peerfinder tests.libxrpl > xrpl.protocol tests.libxrpl > xrpl.protocol_autogen tests.libxrpl > xrpl.resource @@ -220,6 +218,8 @@ xrpl.nodestore > xrpl.basics xrpl.nodestore > xrpl.config xrpl.nodestore > xrpl.json xrpl.nodestore > xrpl.protocol +xrpl.peerfinder > xrpl.basics +xrpl.peerfinder > xrpl.protocol xrpl.protocol > xrpl.basics xrpl.protocol > xrpl.json xrpl.protocol_autogen > xrpl.json @@ -253,6 +253,7 @@ xrpld.app > xrpl.json xrpld.app > xrpl.ledger xrpld.app > xrpl.net xrpld.app > xrpl.nodestore +xrpld.app > xrpl.peerfinder xrpld.app > xrpl.protocol xrpld.app > xrpl.rdb xrpld.app > xrpl.resource @@ -277,15 +278,16 @@ xrpld.overlay > xrpld.core xrpld.overlay > xrpld.peerfinder xrpld.overlay > xrpl.json xrpld.overlay > xrpl.ledger +xrpld.overlay > xrpl.peerfinder xrpld.overlay > xrpl.protocol xrpld.overlay > xrpl.resource xrpld.overlay > xrpl.server xrpld.overlay > xrpl.shamap xrpld.overlay > xrpl.tx xrpld.peerfinder > xrpl.basics -xrpld.peerfinder > xrpl.config +xrpld.peerfinder > xrpld.app xrpld.peerfinder > xrpld.core -xrpld.peerfinder > xrpl.protocol +xrpld.peerfinder > xrpl.peerfinder xrpld.peerfinder > xrpl.rdb xrpld.perflog > xrpl.basics xrpld.perflog > xrpl.config diff --git a/cmake/XrplCore.cmake b/cmake/XrplCore.cmake index 3e49267715..62a8fe143b 100644 --- a/cmake/XrplCore.cmake +++ b/cmake/XrplCore.cmake @@ -133,6 +133,12 @@ target_link_libraries( add_module(xrpl resource) target_link_libraries(xrpl.libxrpl.resource PUBLIC xrpl.libxrpl.protocol) +add_module(xrpl peerfinder) +target_link_libraries( + xrpl.libxrpl.peerfinder + PUBLIC xrpl.libxrpl.basics xrpl.libxrpl.protocol +) + # Level 08 add_module(xrpl net) target_link_libraries( @@ -227,6 +233,7 @@ target_link_modules( ledger net nodestore + peerfinder protocol protocol_autogen rdb diff --git a/include/xrpl/beast/container/detail/aged_unordered_container.h b/include/xrpl/beast/container/detail/aged_unordered_container.h index db10e8cc23..c4287b1ca1 100644 --- a/include/xrpl/beast/container/detail/aged_unordered_container.h +++ b/include/xrpl/beast/container/detail/aged_unordered_container.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include diff --git a/include/xrpl/peerfinder/Config.h b/include/xrpl/peerfinder/Config.h new file mode 100644 index 0000000000..9ff0d342c3 --- /dev/null +++ b/include/xrpl/peerfinder/Config.h @@ -0,0 +1,163 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace xrpl::PeerFinder { + +struct PeerLimitConfig +{ + std::optional maxPeers; + std::optional inPeers; + std::optional outPeers; +}; + +/** + * PeerFinder configuration settings. + */ +struct Config +{ + /** + * The largest number of public peer slots to allow. + * This includes both inbound and outbound, but does not include + * fixed peers. + */ + std::size_t maxPeers{Tuning::kDefaultMaxPeers}; + + /** + * The number of automatic outbound connections to maintain. + * Outbound connections are only maintained if autoConnect + * is `true`. + */ + std::size_t outPeers = calcOutPeers(); // Note: relies on `maxPeers` being initialized + + /** + * The number of automatic inbound connections to maintain. + * Inbound connections are only maintained if wantIncoming + * is `true`. + */ + std::size_t inPeers{0}; + + /** + * `true` if we want our IP address kept private. + */ + bool peerPrivate = true; + + /** + * `true` if we want to accept incoming connections. + */ + bool wantIncoming{true}; + + /** + * `true` if we want to establish connections automatically + */ + bool autoConnect{true}; + + /** + * The listening port number. + */ + std::uint16_t listeningPort{0}; + + /** + * The set of features we advertise. + */ + std::string features; + + /** + * Limit how many incoming connections we allow per IP + */ + int ipLimit{0}; + + /** + * `true` if we want to verify endpoints in TMEndpoints messages + */ + bool verifyEndpoints = true; + + //-------------------------------------------------------------------------- + + /** + * Returns a suitable value for outPeers according to the rules. + */ + [[nodiscard]] std::size_t + calcOutPeers() const; + + /** + * Adjusts the values so they follow the business rules. + */ + void + applyTuning(); + + /** + * Write the configuration into a property stream + */ + void + onWrite(beast::PropertyStream::Map& map) const; + + /** + * Make PeerFinder::Config from peer limit and server mode parameters. + */ + static Config + makeConfig( + bool peerPrivate, + bool standalone, + PeerLimitConfig const& limits, + std::uint16_t port, + bool validationPublicKey, + int ipLimit, + bool verifyEndpoints); + + /** + * Compares two configurations for equality field by field. + */ + friend bool + operator==(Config const& lhs, Config const& rhs) = default; +}; + +//------------------------------------------------------------------------------ + +/** + * Possible results from activating a slot. + */ +enum class Result { InboundDisabled, DuplicatePeer, IpLimitExceeded, Full, Success }; + +/** + * @brief Converts a `Result` enum value to its string representation. + * + * This function provides a human-readable string for a given `Result` enum, + * which is useful for logging, debugging, or displaying status messages. + * + * @param result The `Result` enum value to convert. + * @return A `std::string_view` representing the enum value. Returns "unknown" + * if the enum value is not explicitly handled. + * + * @note This function returns a `std::string_view` for performance. + * A `std::string` would need to allocate memory on the heap and copy the + * string literal into it every time the function is called. + */ +inline std::string_view +to_string(Result result) noexcept +{ + switch (result) + { + case Result::InboundDisabled: + return "inbound disabled"; + case Result::DuplicatePeer: + return "peer already connected"; + case Result::IpLimitExceeded: + return "ip limit exceeded"; + case Result::Full: + return "slots full"; + case Result::Success: + return "success"; + } + + return "unknown"; +} + +} // namespace xrpl::PeerFinder diff --git a/include/xrpl/peerfinder/PeerfinderManager.h b/include/xrpl/peerfinder/PeerfinderManager.h new file mode 100644 index 0000000000..ed683520c1 --- /dev/null +++ b/include/xrpl/peerfinder/PeerfinderManager.h @@ -0,0 +1,179 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace xrpl::PeerFinder { + +/** + * Maintains a set of IP addresses used for getting into the network. + */ +class Manager : public beast::PropertyStream::Source +{ +protected: + Manager() noexcept; + +public: + /** + * Destroy the object. + * Any pending source fetch operations are aborted. + * There may be some listener calls made before the + * destructor returns. + */ + ~Manager() override = default; + + /** + * Set the configuration for the manager. + * The new settings will be applied asynchronously. + * Thread safety: + * Can be called from any threads at any time. + */ + virtual void + setConfig(Config const& config) = 0; + + /** + * Transition to the started state, synchronously. + */ + virtual void + start() = 0; + + /** + * Transition to the stopped state, synchronously. + */ + virtual void + stop() = 0; + + /** + * Returns the configuration for the manager. + */ + virtual Config + config() = 0; + + /** + * Add a peer that should always be connected. + * This is useful for maintaining a private cluster of peers. + * The string is the name as specified in the configuration + * file, along with the set of corresponding IP addresses. + */ + virtual void + addFixedPeer(std::string_view name, std::vector const& addresses) = 0; + + /** + * Add a set of strings as fallback IP::Endpoint sources. + * @param name A label used for diagnostics. + */ + virtual void + addFallbackStrings(std::string const& name, std::vector const& strings) = 0; + + /** + * Add a URL as a fallback location to obtain IP::Endpoint sources. + * @param name A label used for diagnostics. + */ + /* VFALCO NOTE Unimplemented + virtual void addFallbackURL (std::string const& name, + std::string const& url) = 0; + */ + + //-------------------------------------------------------------------------- + + /** + * Create a new inbound slot with the specified remote endpoint. + * If nullptr is returned, then the slot could not be assigned. + * Usually this is because of a detected self-connection. + */ + virtual std::pair, Result> + newInboundSlot( + beast::IP::Endpoint const& localEndpoint, + beast::IP::Endpoint const& remoteEndpoint) = 0; + + /** + * Create a new outbound slot with the specified remote endpoint. + * If nullptr is returned, then the slot could not be assigned. + * Usually this is because of a duplicate connection. + */ + virtual std::pair, Result> + newOutboundSlot(beast::IP::Endpoint const& remoteEndpoint) = 0; + + /** + * Called when mtENDPOINTS is received. + */ + virtual void + onEndpoints(std::shared_ptr const& slot, Endpoints const& endpoints) = 0; + + /** + * Called when the slot is closed. + * This always happens when the socket is closed, unless the socket + * was canceled. + */ + virtual void + onClosed(std::shared_ptr const& slot) = 0; + + /** + * Called when an outbound connection is deemed to have failed + */ + virtual void + onFailure(std::shared_ptr const& slot) = 0; + + /** + * Called when we received redirect IPs from a busy peer. + */ + virtual void + onRedirects( + boost::asio::ip::tcp::endpoint const& remoteAddress, + std::vector const& eps) = 0; + + //-------------------------------------------------------------------------- + + /** + * Called when an outbound connection attempt succeeds. + * The local endpoint must be valid. If the caller receives an error + * when retrieving the local endpoint from the socket, it should + * proceed as if the connection attempt failed by calling on_closed + * instead of on_connected. + * @return `true` if the connection should be kept + */ + virtual bool + onConnected(std::shared_ptr const& slot, beast::IP::Endpoint const& localEndpoint) = 0; + + /** + * Request an active slot type. + */ + virtual Result + activate(std::shared_ptr const& slot, PublicKey const& key, bool reserved) = 0; + + /** + * Returns a set of endpoints suitable for redirection. + */ + virtual std::vector + redirect(std::shared_ptr const& slot) = 0; + + /** + * Return a set of addresses we should connect to. + */ + virtual std::vector + autoconnect() = 0; + + virtual std::vector, std::vector>> + buildEndpointsForPeers() = 0; + + /** + * Perform periodic activity. + * This should be called once per second. + */ + virtual void + oncePerSecond() = 0; +}; + +} // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/Slot.h b/include/xrpl/peerfinder/Slot.h similarity index 100% rename from src/xrpld/peerfinder/Slot.h rename to include/xrpl/peerfinder/Slot.h diff --git a/include/xrpl/peerfinder/Types.h b/include/xrpl/peerfinder/Types.h new file mode 100644 index 0000000000..9e82d9d65c --- /dev/null +++ b/include/xrpl/peerfinder/Types.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace xrpl::PeerFinder { + +using clock_type = beast::AbstractClock; + +/** + * Represents a set of addresses. + */ +using IPAddresses = std::vector; + +//------------------------------------------------------------------------------ + +/** + * Describes a connectable peer address along with some metadata. + */ +struct Endpoint +{ + Endpoint() = default; + + Endpoint(beast::IP::Endpoint ep, std::uint32_t hops); + + std::uint32_t hops = 0; + beast::IP::Endpoint address; +}; + +inline bool +operator<(Endpoint const& lhs, Endpoint const& rhs) +{ + return lhs.address < rhs.address; +} + +/** + * A set of Endpoint used for connecting. + */ +using Endpoints = std::vector; + +} // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/Bootcache.h b/include/xrpl/peerfinder/detail/Bootcache.h similarity index 97% rename from src/xrpld/peerfinder/detail/Bootcache.h rename to include/xrpl/peerfinder/detail/Bootcache.h index c84fed42c7..2141a374fa 100644 --- a/src/xrpld/peerfinder/detail/Bootcache.h +++ b/include/xrpl/peerfinder/detail/Bootcache.h @@ -1,11 +1,11 @@ #pragma once -#include -#include - #include #include #include +#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Checker.h b/include/xrpl/peerfinder/detail/Checker.h similarity index 100% rename from src/xrpld/peerfinder/detail/Checker.h rename to include/xrpl/peerfinder/detail/Checker.h diff --git a/src/xrpld/peerfinder/detail/Counts.h b/include/xrpl/peerfinder/detail/Counts.h similarity index 98% rename from src/xrpld/peerfinder/detail/Counts.h rename to include/xrpl/peerfinder/detail/Counts.h index c90598c1a1..ce78eadce4 100644 --- a/src/xrpld/peerfinder/detail/Counts.h +++ b/include/xrpl/peerfinder/detail/Counts.h @@ -1,11 +1,10 @@ #pragma once -#include -#include -#include - #include #include +#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Fixed.h b/include/xrpl/peerfinder/detail/Fixed.h similarity index 92% rename from src/xrpld/peerfinder/detail/Fixed.h rename to include/xrpl/peerfinder/detail/Fixed.h index 24d54775ef..6754ec6dbd 100644 --- a/src/xrpld/peerfinder/detail/Fixed.h +++ b/include/xrpl/peerfinder/detail/Fixed.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Handouts.h b/include/xrpl/peerfinder/detail/Handouts.h similarity index 98% rename from src/xrpld/peerfinder/detail/Handouts.h rename to include/xrpl/peerfinder/detail/Handouts.h index 757f1a8e1b..cb5fd7f850 100644 --- a/src/xrpld/peerfinder/detail/Handouts.h +++ b/include/xrpl/peerfinder/detail/Handouts.h @@ -1,12 +1,11 @@ #pragma once -#include -#include -#include - #include #include #include +#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Livecache.h b/include/xrpl/peerfinder/detail/Livecache.h similarity index 95% rename from src/xrpld/peerfinder/detail/Livecache.h rename to include/xrpl/peerfinder/detail/Livecache.h index 2015098847..cac284d1cc 100644 --- a/src/xrpld/peerfinder/detail/Livecache.h +++ b/include/xrpl/peerfinder/detail/Livecache.h @@ -1,9 +1,5 @@ #pragma once -#include -#include -#include - #include #include #include @@ -12,6 +8,8 @@ #include #include #include +#include +#include #include #include @@ -22,6 +20,8 @@ #include #include #include +#include +#include #include #include #include @@ -411,7 +411,7 @@ Livecache::expire() } if (n > 0) { - JLOG(journal_.debug()) << beast::Leftw(18) << "Livecache expired " << n + JLOG(journal_.debug()) << std::left << std::setw(18) << "Livecache expired " << n << ((n > 1) ? " entries" : " entry"); } } @@ -434,7 +434,7 @@ Livecache::insert(Endpoint const& ep) if (result.second) { hops.insert(e); - JLOG(journal_.debug()) << beast::Leftw(18) << "Livecache insert " << ep.address + JLOG(journal_.debug()) << std::left << std::setw(18) << "Livecache insert " << ep.address << " at hops " << ep.hops; return; } @@ -442,7 +442,7 @@ Livecache::insert(Endpoint const& ep) { // Drop duplicates at higher hops std::size_t const excess(ep.hops - e.endpoint.hops); - JLOG(journal_.trace()) << beast::Leftw(18) << "Livecache drop " << ep.address + JLOG(journal_.trace()) << std::left << std::setw(18) << "Livecache drop " << ep.address << " at hops +" << excess; return; } @@ -453,12 +453,12 @@ Livecache::insert(Endpoint const& ep) if (ep.hops < e.endpoint.hops) { hops.reinsert(e, ep.hops); - JLOG(journal_.debug()) << beast::Leftw(18) << "Livecache update " << ep.address + JLOG(journal_.debug()) << std::left << std::setw(18) << "Livecache update " << ep.address << " at hops " << ep.hops; } else { - JLOG(journal_.trace()) << beast::Leftw(18) << "Livecache refresh " << ep.address + JLOG(journal_.trace()) << std::left << std::setw(18) << "Livecache refresh " << ep.address << " at hops " << ep.hops; } } diff --git a/src/xrpld/peerfinder/detail/Logic.h b/include/xrpl/peerfinder/detail/Logic.h similarity index 91% rename from src/xrpld/peerfinder/detail/Logic.h rename to include/xrpl/peerfinder/detail/Logic.h index a7dbbf850d..c623263884 100644 --- a/src/xrpld/peerfinder/detail/Logic.h +++ b/include/xrpl/peerfinder/detail/Logic.h @@ -1,17 +1,5 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include @@ -22,18 +10,34 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include + #include #include #include #include +#include +#include #include #include #include #include #include #include +#include #include #include #include @@ -197,8 +201,8 @@ public: if (result.second) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Logic add fixed '" << name << "' at " << remoteAddress; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic add fixed '" << name + << "' at " << remoteAddress; return; } } @@ -221,7 +225,7 @@ public: if (iter == slots.end()) { // The slot disconnected before we finished the check - JLOG(journal.debug()) << beast::Leftw(18) << "Logic tested " << checkedAddress + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic tested " << checkedAddress << " but the connection was closed"; return; } @@ -255,7 +259,7 @@ public: beast::IP::Endpoint const& localEndpoint, beast::IP::Endpoint const& remoteEndpoint) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic accept" << remoteEndpoint + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic accept" << remoteEndpoint << " on local " << localEndpoint; std::scoped_lock const _(lock); @@ -266,7 +270,7 @@ public: auto const count = connectedAddresses.count(remoteEndpoint.address()); if (count + 1 > config_.ipLimit) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic dropping inbound " + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic dropping inbound " << remoteEndpoint << " because of ip limits."; return {SlotImp::ptr(), Result::IpLimitExceeded}; } @@ -275,8 +279,8 @@ public: // Check for duplicate connection if (slots.contains(remoteEndpoint)) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic dropping " << remoteEndpoint - << " as duplicate incoming"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic dropping " + << remoteEndpoint << " as duplicate incoming"; return {SlotImp::ptr(), Result::DuplicatePeer}; } @@ -304,15 +308,15 @@ public: std::pair newOutboundSlot(beast::IP::Endpoint const& remoteEndpoint) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic connect " << remoteEndpoint; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic connect " << remoteEndpoint; std::scoped_lock const _(lock); // Check for duplicate connection if (slots.contains(remoteEndpoint)) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic dropping " << remoteEndpoint - << " as duplicate connect"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic dropping " + << remoteEndpoint << " as duplicate connect"; return {SlotImp::ptr(), Result::DuplicatePeer}; } @@ -506,15 +510,15 @@ public: if (!h.list().empty()) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Logic connect " << h.list().size() << " fixed"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic connect " + << h.list().size() << " fixed"; return h.list(); } if (counts_.attempts() > 0) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Logic waiting on " << counts_.attempts() << " attempts"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic waiting on " + << counts_.attempts() << " attempts"; return none; } } @@ -535,14 +539,14 @@ public: if (!h.list().empty()) { JLOG(journal.debug()) - << beast::Leftw(18) << "Logic connect " << h.list().size() << " live " + << std::left << std::setw(18) << "Logic connect " << h.list().size() << " live " << ((h.list().size() > 1) ? "endpoints" : "endpoint"); return h.list(); } if (counts_.attempts() > 0) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Logic waiting on " << counts_.attempts() << " attempts"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic waiting on " + << counts_.attempts() << " attempts"; return none; } } @@ -568,8 +572,9 @@ public: if (!h.list().empty()) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic connect " << h.list().size() - << " boot " << ((h.list().size() > 1) ? "addresses" : "address"); + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic connect " + << h.list().size() << " boot " + << ((h.list().size() > 1) ? "addresses" : "address"); return h.list(); } @@ -689,8 +694,8 @@ public: // Enforce hop limit if (ep.hops > Tuning::kMaxHops) { - JLOG(journal.debug()) << beast::Leftw(18) << "Endpoints drop " << ep.address - << " for excess hops " << ep.hops; + JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop " + << ep.address << " for excess hops " << ep.hops; iter = list.erase(iter); continue; } @@ -706,18 +711,18 @@ public: } else { - JLOG(journal.debug()) - << beast::Leftw(18) << "Endpoints drop " << ep.address << " for extra self"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop " + << ep.address << " for extra self"; iter = list.erase(iter); continue; } } // Discard invalid addresses - if (config_.verifyEndpoints && !isValidAddress(ep.address)) + if (!isValidAddress(ep.address)) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Endpoints drop " << ep.address << " as invalid"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop " + << ep.address << " as invalid"; iter = list.erase(iter); continue; } @@ -727,8 +732,8 @@ public: return ep.address == other.address; })) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Endpoints drop " << ep.address << " as duplicate"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop " + << ep.address << " as duplicate"; iter = list.erase(iter); continue; } @@ -1074,13 +1079,13 @@ public: if (!results.error) { int const count(addBootcacheAddresses(results.addresses)); - JLOG(journal.info()) << beast::Leftw(18) << "Logic added " << count << " new " + JLOG(journal.info()) << std::left << std::setw(18) << "Logic added " << count << " new " << ((count == 1) ? "address" : "addresses") << " from " << source->name(); } else { - JLOG(journal.error()) << beast::Leftw(18) << "Logic failed " + JLOG(journal.error()) << std::left << std::setw(18) << "Logic failed " << "'" << source->name() << "' fetch, " << results.error.message(); } @@ -1098,8 +1103,6 @@ public: { if (isUnspecified(address)) return false; - if (isLoopback(address)) - return false; if (!isPublic(address)) return false; if (address.port() == 0) @@ -1221,8 +1224,8 @@ Logic::onRedirects( bootcache.insert(beast::IPAddressConversion::fromAsio(*first)); if (n > 0) { - JLOG(journal.trace()) << beast::Leftw(18) << "Logic add " << n << " redirect IPs from " - << remoteAddress; + JLOG(journal.trace()) << std::left << std::setw(18) << "Logic add " << n + << " redirect IPs from " << remoteAddress; } } diff --git a/src/xrpld/peerfinder/detail/SlotImp.h b/include/xrpl/peerfinder/detail/SlotImp.h similarity index 96% rename from src/xrpld/peerfinder/detail/SlotImp.h rename to include/xrpl/peerfinder/detail/SlotImp.h index 898941b157..35c61b13cf 100644 --- a/src/xrpld/peerfinder/detail/SlotImp.h +++ b/include/xrpl/peerfinder/detail/SlotImp.h @@ -1,10 +1,9 @@ #pragma once -#include -#include - #include #include +#include +#include #include #include @@ -172,7 +171,7 @@ private: std::optional localEndpoint_; std::optional publicKey_; - static constexpr std::int32_t kUnknownPort = -1; + static std::int32_t constexpr kUnknownPort = -1; std::atomic listeningPort_; public: diff --git a/src/xrpld/peerfinder/detail/Source.h b/include/xrpl/peerfinder/detail/Source.h similarity index 95% rename from src/xrpld/peerfinder/detail/Source.h rename to include/xrpl/peerfinder/detail/Source.h index b205dc8dfb..5cdb535bdd 100644 --- a/src/xrpld/peerfinder/detail/Source.h +++ b/include/xrpl/peerfinder/detail/Source.h @@ -1,8 +1,7 @@ #pragma once -#include - #include +#include #include diff --git a/src/xrpld/peerfinder/detail/SourceStrings.h b/include/xrpl/peerfinder/detail/SourceStrings.h similarity index 90% rename from src/xrpld/peerfinder/detail/SourceStrings.h rename to include/xrpl/peerfinder/detail/SourceStrings.h index b79cf0df03..325a024764 100644 --- a/src/xrpld/peerfinder/detail/SourceStrings.h +++ b/include/xrpl/peerfinder/detail/SourceStrings.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Store.h b/include/xrpl/peerfinder/detail/Store.h similarity index 100% rename from src/xrpld/peerfinder/detail/Store.h rename to include/xrpl/peerfinder/detail/Store.h diff --git a/src/xrpld/peerfinder/detail/Tuning.h b/include/xrpl/peerfinder/detail/Tuning.h similarity index 100% rename from src/xrpld/peerfinder/detail/Tuning.h rename to include/xrpl/peerfinder/detail/Tuning.h diff --git a/include/xrpl/peerfinder/make_Manager.h b/include/xrpl/peerfinder/make_Manager.h new file mode 100644 index 0000000000..5da372e588 --- /dev/null +++ b/include/xrpl/peerfinder/make_Manager.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include + +namespace xrpl::PeerFinder { + +/** + * @brief Create a new Manager. + * + * @param ioContext The io_context used to schedule asynchronous work. + * @param clock The clock used for timekeeping. + * @param journal The journal used for logging. + * @param store The persistence backend for the bootstrap cache. The caller + * retains ownership and must keep it alive (and opened) for the lifetime of + * the returned Manager. This lets consumers supply their own Store + * implementation (e.g. the SQLite-backed StoreSqdb in xrpld). + * @param collector The collector used to report metrics. + * @return The newly created Manager. + */ +std::unique_ptr +makeManager( + boost::asio::io_context& ioContext, + clock_type& clock, + beast::Journal journal, + Store& store, + beast::insight::Collector::ptr const& collector); + +} // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/Bootcache.cpp b/src/libxrpl/peerfinder/Bootcache.cpp similarity index 80% rename from src/xrpld/peerfinder/detail/Bootcache.cpp rename to src/libxrpl/peerfinder/Bootcache.cpp index a0a753530b..a2a56b4d01 100644 --- a/src/xrpld/peerfinder/detail/Bootcache.cpp +++ b/src/libxrpl/peerfinder/Bootcache.cpp @@ -1,19 +1,19 @@ -#include - -#include -#include -#include -#include +#include #include #include #include #include #include +#include +#include +#include #include #include #include +#include +#include #include namespace xrpl::PeerFinder { @@ -82,13 +82,14 @@ Bootcache::load() auto const result(this->map_.insert(value_type(endpoint, valence))); if (!result.second) { - JLOG(this->journal_.error()) << beast::Leftw(18) << "Bootcache discard " << endpoint; + JLOG(this->journal_.error()) + << std::left << std::setw(18) << "Bootcache discard " << endpoint; } })); if (n > 0) { - JLOG(journal_.info()) << beast::Leftw(18) << "Bootcache loaded " << n + JLOG(journal_.info()) << std::left << std::setw(18) << "Bootcache loaded " << n << ((n > 1) ? " addresses" : " address"); prune(); } @@ -100,7 +101,7 @@ Bootcache::insert(beast::IP::Endpoint const& endpoint) auto const result(map_.insert(value_type(endpoint, 0))); if (result.second) { - JLOG(journal_.trace()) << beast::Leftw(18) << "Bootcache insert " << endpoint; + JLOG(journal_.trace()) << std::left << std::setw(18) << "Bootcache insert " << endpoint; prune(); flagForUpdate(); } @@ -121,7 +122,7 @@ Bootcache::insertStatic(beast::IP::Endpoint const& endpoint) if (result.second) { - JLOG(journal_.trace()) << beast::Leftw(18) << "Bootcache insert " << endpoint; + JLOG(journal_.trace()) << std::left << std::setw(18) << "Bootcache insert " << endpoint; prune(); flagForUpdate(); } @@ -146,8 +147,9 @@ Bootcache::onSuccess(beast::IP::Endpoint const& endpoint) XRPL_ASSERT(result.second, "xrpl::PeerFinder::Bootcache::onSuccess : endpoint inserted"); } Entry const& entry(result.first->right); - JLOG(journal_.info()) << beast::Leftw(18) << "Bootcache connect " << endpoint << " with " - << entry.valence() << ((entry.valence() > 1) ? " successes" : " success"); + JLOG(journal_.info()) << std::left << std::setw(18) << "Bootcache connect " << endpoint + << " with " << entry.valence() + << ((entry.valence() > 1) ? " successes" : " success"); flagForUpdate(); } @@ -170,8 +172,8 @@ Bootcache::onFailure(beast::IP::Endpoint const& endpoint) } Entry const& entry(result.first->right); auto const n(std::abs(entry.valence())); - JLOG(journal_.debug()) << beast::Leftw(18) << "Bootcache failed " << endpoint << " with " << n - << ((n > 1) ? " attempts" : " attempt"); + JLOG(journal_.debug()) << std::left << std::setw(18) << "Bootcache failed " << endpoint + << " with " << n << ((n > 1) ? " attempts" : " attempt"); flagForUpdate(); } @@ -209,17 +211,19 @@ Bootcache::prune() // Work backwards because bimap doesn't handle // erasing using a reverse iterator very well. // - for (auto iter(map_.right.end()); count-- > 0 && iter != map_.right.begin(); ++pruned) + for (auto iter(map_.right.end()); count > 0 && iter != map_.right.begin(); ++pruned) { + --count; --iter; beast::IP::Endpoint const& endpoint(iter->get_left()); Entry const& entry(iter->get_right()); - JLOG(journal_.trace()) << beast::Leftw(18) << "Bootcache pruned" << endpoint + JLOG(journal_.trace()) << std::left << std::setw(18) << "Bootcache pruned" << endpoint << " at valence " << entry.valence(); iter = map_.right.erase(iter); } - JLOG(journal_.debug()) << beast::Leftw(18) << "Bootcache pruned " << pruned << " entries total"; + JLOG(journal_.debug()) << std::left << std::setw(18) << "Bootcache pruned " << pruned + << " entries total"; } // Updates the Store with the current set of entries if needed. diff --git a/src/libxrpl/peerfinder/Config.cpp b/src/libxrpl/peerfinder/Config.cpp new file mode 100644 index 0000000000..3e2f74f42b --- /dev/null +++ b/src/libxrpl/peerfinder/Config.cpp @@ -0,0 +1,135 @@ +#include + +#include + +#include +#include +#include +#include + +namespace xrpl::PeerFinder { + +std::size_t +Config::calcOutPeers() const +{ + return std::max( + ((maxPeers * Tuning::kOutPercent) + 50) / 100, std::size_t(Tuning::kMinOutCount)); +} + +void +Config::applyTuning() +{ + if (ipLimit == 0) + { + // Unless a limit is explicitly set, we allow between + // 2 and 5 connections from non RFC-1918 "private" + // IP addresses. + ipLimit = 2; + + if (inPeers > Tuning::kDefaultMaxPeers) + ipLimit += std::min(5, static_cast(inPeers / Tuning::kDefaultMaxPeers)); + } + + // We don't allow a single IP to consume all incoming slots, + // unless we only have one incoming slot available. + ipLimit = std::max(1, std::min(ipLimit, static_cast(inPeers / 2))); +} + +void +Config::onWrite(beast::PropertyStream::Map& map) const +{ + map["max_peers"] = maxPeers; + map["out_peers"] = outPeers; + map["want_incoming"] = wantIncoming; + map["auto_connect"] = autoConnect; + map["port"] = listeningPort; + map["features"] = features; + map["ip_limit"] = ipLimit; + map["verify_endpoints"] = verifyEndpoints; +} + +Config +Config::makeConfig( + bool peerPrivate, + bool standalone, + PeerLimitConfig const& limits, + std::uint16_t port, + bool validationPublicKey, + int ipLimit, + bool verifyEndpoints) +{ + PeerFinder::Config config; + + if (!limits.maxPeers) + { + if (limits.inPeers && !limits.outPeers) + throw std::runtime_error("Both inbound and outbound peer limits must be configured"); + + if (limits.outPeers && !limits.inPeers) + throw std::runtime_error("Both inbound and outbound peer limits must be configured"); + + if (limits.inPeers && *limits.inPeers > 1000) + throw std::runtime_error("Inbound peer limit must be less than or equal to 1000"); + + if (limits.outPeers && (*limits.outPeers < 10 || *limits.outPeers > 1000)) + throw std::runtime_error("Outbound peer limit must be in the range 10-1000"); + } + + config.peerPrivate = peerPrivate; + + // Servers with peer privacy don't want to allow incoming connections + config.wantIncoming = (!config.peerPrivate) && (port != 0); + + if (limits.maxPeers || (!limits.inPeers && !limits.outPeers)) + { + if (limits.maxPeers && *limits.maxPeers != 0) + config.maxPeers = *limits.maxPeers; + + config.maxPeers = std::max(config.maxPeers, Tuning::kMinOutCount); + config.outPeers = config.calcOutPeers(); + + // Calculate the number of outbound peers we want. If we dont want + // or can't accept incoming, this will simply be equal to maxPeers. + if (!config.wantIncoming) + config.outPeers = config.maxPeers; + + // Calculate the largest number of inbound connections we could + // take. + if (config.maxPeers >= config.outPeers) + { + config.inPeers = config.maxPeers - config.outPeers; + } + else + { + config.inPeers = 0; + } + } + else + { + config.outPeers = *limits.outPeers; + config.inPeers = *limits.inPeers; + config.maxPeers = 0; + } + + // This will cause servers configured as validators to request that + // peers they connect to never report their IP address. We set this + // after we set the 'wantIncoming' because we want a "soft" version + // of peer privacy unless the operator explicitly asks for it. + if (validationPublicKey) + config.peerPrivate = true; + + // if it's a private peer or we are running as standalone + // automatic connections would defeat the purpose. + config.autoConnect = !standalone && !peerPrivate; + config.listeningPort = port; + config.features = ""; + config.ipLimit = ipLimit; + config.verifyEndpoints = verifyEndpoints; + + // Enforce business rules + config.applyTuning(); + + return config; +} + +} // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/Endpoint.cpp b/src/libxrpl/peerfinder/Endpoint.cpp similarity index 76% rename from src/xrpld/peerfinder/detail/Endpoint.cpp rename to src/libxrpl/peerfinder/Endpoint.cpp index 15de5cd153..12f2725ea5 100644 --- a/src/xrpld/peerfinder/detail/Endpoint.cpp +++ b/src/libxrpl/peerfinder/Endpoint.cpp @@ -1,7 +1,5 @@ -#include -#include - #include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/PeerfinderManager.cpp b/src/libxrpl/peerfinder/PeerfinderManager.cpp similarity index 92% rename from src/xrpld/peerfinder/detail/PeerfinderManager.cpp rename to src/libxrpl/peerfinder/PeerfinderManager.cpp index 2727a03013..2219627f09 100644 --- a/src/xrpld/peerfinder/detail/PeerfinderManager.cpp +++ b/src/libxrpl/peerfinder/PeerfinderManager.cpp @@ -1,11 +1,4 @@ -#include - -#include -#include -#include -#include -#include -#include +#include #include #include @@ -13,7 +6,15 @@ #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -38,10 +39,9 @@ public: std::optional> work_; clock_type& clock_; beast::Journal journal_; - StoreSqdb store_; + Store& store_; Checker checker_; Logic logic_; - BasicConfig const& config_; // NOLINTEND(readability-identifier-naming) //-------------------------------------------------------------------------- @@ -50,16 +50,15 @@ public: boost::asio::io_context& ioContext, clock_type& clock, beast::Journal journal, - BasicConfig const& config, + Store& store, beast::insight::Collector::ptr const& collector) : io_context_(ioContext) , work_(std::in_place, boost::asio::make_work_guard(io_context_)) , clock_(clock) , journal_(journal) - , store_(journal) + , store_(store) , checker_(io_context_) , logic_(clock, store_, checker_, journal) - , config_(config) , stats_([this] { collectMetrics(); }, collector) { } @@ -206,7 +205,6 @@ public: void start() override { - store_.open(config_); logic_.load(); } @@ -261,10 +259,10 @@ makeManager( boost::asio::io_context& ioContext, clock_type& clock, beast::Journal journal, - BasicConfig const& config, + Store& store, beast::insight::Collector::ptr const& collector) { - return std::make_unique(ioContext, clock, journal, config, collector); + return std::make_unique(ioContext, clock, journal, store, collector); } } // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/SlotImp.cpp b/src/libxrpl/peerfinder/SlotImp.cpp similarity index 93% rename from src/xrpld/peerfinder/detail/SlotImp.cpp rename to src/libxrpl/peerfinder/SlotImp.cpp index a54ddb56e7..0a4f32fd62 100644 --- a/src/xrpld/peerfinder/detail/SlotImp.cpp +++ b/src/libxrpl/peerfinder/SlotImp.cpp @@ -1,12 +1,10 @@ -#include +#include -#include -#include -#include - -#include #include #include +#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/SourceStrings.cpp b/src/libxrpl/peerfinder/SourceStrings.cpp similarity index 93% rename from src/xrpld/peerfinder/detail/SourceStrings.cpp rename to src/libxrpl/peerfinder/SourceStrings.cpp index 7b28db4306..f47e0cd51d 100644 --- a/src/xrpld/peerfinder/detail/SourceStrings.cpp +++ b/src/libxrpl/peerfinder/SourceStrings.cpp @@ -1,9 +1,8 @@ -#include - -#include +#include #include #include +#include #include #include diff --git a/src/test/overlay/TMGetObjectByHash_test.cpp b/src/test/overlay/TMGetObjectByHash_test.cpp index e579989181..f52220a90c 100644 --- a/src/test/overlay/TMGetObjectByHash_test.cpp +++ b/src/test/overlay/TMGetObjectByHash_test.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -16,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/src/test/overlay/tx_reduce_relay_test.cpp b/src/test/overlay/tx_reduce_relay_test.cpp index a0d91d3aed..43f6ef2506 100644 --- a/src/test/overlay/tx_reduce_relay_test.cpp +++ b/src/test/overlay/tx_reduce_relay_test.cpp @@ -9,12 +9,12 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include diff --git a/src/test/peerfinder/Livecache_test.cpp b/src/test/peerfinder/Livecache_test.cpp deleted file mode 100644 index 4f2d6e97e1..0000000000 --- a/src/test/peerfinder/Livecache_test.cpp +++ /dev/null @@ -1,212 +0,0 @@ -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace xrpl::PeerFinder { - -bool -operator==(Endpoint const& a, Endpoint const& b) -{ - return (a.hops == b.hops && a.address == b.address); -} - -class Livecache_test : public beast::unit_test::Suite -{ - TestStopwatch clock_; - test::SuiteJournal journal_; - -public: - Livecache_test() : journal_("Livecache_test", *this) - { - } - - // Add the address as an endpoint - template - void - add(beast::IP::Endpoint ep, C& c, std::uint32_t hops = 0) - { - Endpoint const cep{ep, hops}; - c.insert(cep); - } - - void - testBasicInsert() - { - testcase("Basic Insert"); - Livecache<> c(clock_, journal_); - BEAST_EXPECT(c.empty()); - - for (auto i = 0; i < 10; ++i) - add(beast::IP::randomEP(true), c); - - BEAST_EXPECT(!c.empty()); - BEAST_EXPECT(c.size() == 10); - - for (auto i = 0; i < 10; ++i) - add(beast::IP::randomEP(false), c); - - BEAST_EXPECT(!c.empty()); - BEAST_EXPECT(c.size() == 20); - } - - void - testInsertUpdate() - { - testcase("Insert/Update"); - Livecache<> c(clock_, journal_); - - auto ep1 = Endpoint{beast::IP::randomEP(), 2}; - c.insert(ep1); - BEAST_EXPECT(c.size() == 1); - // third position list will contain the entry - BEAST_EXPECT((c.hops.begin() + 2)->begin()->hops == 2); - - auto ep2 = Endpoint{ep1.address, 4}; - // this will not change the entry has higher hops - c.insert(ep2); - BEAST_EXPECT(c.size() == 1); - // still in third position list - BEAST_EXPECT((c.hops.begin() + 2)->begin()->hops == 2); - - auto ep3 = Endpoint{ep1.address, 2}; - // this will not change the entry has the same hops as existing - c.insert(ep3); - BEAST_EXPECT(c.size() == 1); - // still in third position list - BEAST_EXPECT((c.hops.begin() + 2)->begin()->hops == 2); - - auto ep4 = Endpoint{ep1.address, 1}; - c.insert(ep4); - BEAST_EXPECT(c.size() == 1); - // now at second position list - BEAST_EXPECT((c.hops.begin() + 1)->begin()->hops == 1); - } - - void - testExpire() - { - testcase("Expire"); - using namespace std::chrono_literals; - Livecache<> c(clock_, journal_); - - auto ep1 = Endpoint{beast::IP::randomEP(), 1}; - c.insert(ep1); - BEAST_EXPECT(c.size() == 1); - c.expire(); - BEAST_EXPECT(c.size() == 1); - // verify that advancing to 1 sec before expiration - // leaves our entry intact - clock_.advance(Tuning::kLiveCacheSecondsToLive - 1s); - c.expire(); - BEAST_EXPECT(c.size() == 1); - // now advance to the point of expiration - clock_.advance(1s); - c.expire(); - BEAST_EXPECT(c.empty()); - } - - void - testHistogram() - { - testcase("Histogram"); - static constexpr auto kNumEps = 40; - Livecache<> c(clock_, journal_); - for (auto i = 0; i < kNumEps; ++i) - add(beast::IP::randomEP(true), c, xrpl::randInt()); - auto h = c.hops.histogram(); - if (!BEAST_EXPECT(!h.empty())) - return; - std::vector v; - boost::split(v, h, boost::algorithm::is_any_of(",")); - auto sum = 0; - for (auto const& n : v) - { - auto val = boost::lexical_cast(boost::trim_copy(n)); - sum += val; - BEAST_EXPECT(val >= 0); - } - BEAST_EXPECT(sum == kNumEps); - } - - void - testShuffle() - { - testcase("Shuffle"); - Livecache<> c(clock_, journal_); - for (auto i = 0; i < 100; ++i) - add(beast::IP::randomEP(true), c, xrpl::randInt(Tuning::kMaxHops + 1)); - - using at_hop = std::vector; - using all_hops = std::array; - - auto cmpEp = [](Endpoint const& a, Endpoint const& b) { - return (b.hops < a.hops || (b.hops == a.hops && b.address < a.address)); - }; - all_hops before; - all_hops beforeSorted; - for (auto i = std::make_pair(0, c.hops.begin()); i.second != c.hops.end(); - ++i.first, ++i.second) - { - std::ranges::copy(*i.second, std::back_inserter(before[i.first])); - std::ranges::copy(*i.second, std::back_inserter(beforeSorted[i.first])); - std::ranges::sort(beforeSorted[i.first], cmpEp); - } - - c.hops.shuffle(); - - all_hops after; - all_hops afterSorted; - for (auto i = std::make_pair(0, c.hops.begin()); i.second != c.hops.end(); - ++i.first, ++i.second) - { - std::ranges::copy(*i.second, std::back_inserter(after[i.first])); - std::ranges::copy(*i.second, std::back_inserter(afterSorted[i.first])); - std::ranges::sort(afterSorted[i.first], cmpEp); - } - - // each hop bucket should contain the same items - // before and after sort, albeit in different order - bool allMatch = true; - for (auto i = 0; i < before.size(); ++i) - { - BEAST_EXPECT(before[i].size() == after[i].size()); - allMatch = allMatch && (before[i] == after[i]); - BEAST_EXPECT(beforeSorted[i] == afterSorted[i]); - } - BEAST_EXPECT(!allMatch); - } - - void - run() override - { - testBasicInsert(); - testInsertUpdate(); - testExpire(); - testHistogram(); - testShuffle(); - } -}; - -BEAST_DEFINE_TESTSUITE(Livecache, peerfinder, xrpl); - -} // namespace xrpl::PeerFinder diff --git a/src/test/peerfinder/PeerFinder_test.cpp b/src/test/peerfinder/PeerFinder_test.cpp deleted file mode 100644 index cf91800951..0000000000 --- a/src/test/peerfinder/PeerFinder_test.cpp +++ /dev/null @@ -1,789 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace xrpl::PeerFinder { - -class PeerFinder_test : public beast::unit_test::Suite -{ - test::SuiteJournal journal_; - -public: - PeerFinder_test() : journal_("PeerFinder_test", *this) - { - } - - struct TestStore : Store - { - std::size_t - load(load_callback const& cb) override - { - return 0; - } - - void - save(std::vector const&) override - { - } - }; - - struct TestChecker - { - void - stop() - { - } - - void - wait() - { - } - - template - void - asyncConnect(beast::IP::Endpoint const& ep, Handler&& handler) - { - // NOLINTNEXTLINE(misc-const-correctness) - boost::system::error_code ec; - handler(ec); - } - }; - - void - testBackoff1() - { - auto const seconds = 10000; - testcase("backoff 1"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - logic.addFixedPeer("test", beast::IP::Endpoint::fromString("65.0.0.1:5")); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - logic.config(c); - } - std::size_t n = 0; - for (std::size_t i = 0; i < seconds; ++i) - { - auto const list = logic.autoconnect(); - if (!list.empty()) - { - BEAST_EXPECT(list.size() == 1); - auto const [slot, _] = logic.newOutboundSlot(list.front()); - BEAST_EXPECT( - logic.onConnected(slot, beast::IP::Endpoint::fromString("65.0.0.2:5"))); - logic.onClosed(slot); - ++n; - } - clock.advance(std::chrono::seconds(1)); - logic.oncePerSecond(); - } - // Less than 20 attempts - BEAST_EXPECT(n < 20); - } - - // with activate - void - testBackoff2() - { - auto const seconds = 10000; - testcase("backoff 2"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - logic.addFixedPeer("test", beast::IP::Endpoint::fromString("65.0.0.1:5")); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - logic.config(c); - } - - PublicKey const pk(randomKeyPair(KeyType::Secp256k1).first); - std::size_t n = 0; - - for (std::size_t i = 0; i < seconds; ++i) - { - auto const list = logic.autoconnect(); - if (!list.empty()) - { - BEAST_EXPECT(list.size() == 1); - auto const [slot, _] = logic.newOutboundSlot(list.front()); - if (!BEAST_EXPECT( - logic.onConnected(slot, beast::IP::Endpoint::fromString("65.0.0.2:5")))) - return; - if (!BEAST_EXPECT(logic.activate(slot, pk, false) == PeerFinder::Result::Success)) - return; - logic.onClosed(slot); - ++n; - } - clock.advance(std::chrono::seconds(1)); - logic.oncePerSecond(); - } - // No more often than once per minute - BEAST_EXPECT(n <= (seconds + 59) / 60); - } - - // test accepting an incoming slot for an already existing outgoing slot - void - testDuplicateOutIn() - { - testcase("duplicate out/in"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - auto const remote = beast::IP::Endpoint::fromString("65.0.0.1:5"); - auto const [slot1, r] = logic.newOutboundSlot(remote); - BEAST_EXPECT(slot1 != nullptr); - BEAST_EXPECT(r == Result::Success); - BEAST_EXPECT(logic.connectedAddresses.count(remote.address()) == 1); - - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - auto const [slot2, r2] = logic.newInboundSlot(local, remote); - BEAST_EXPECT(logic.connectedAddresses.count(remote.address()) == 1); - BEAST_EXPECT(r2 == Result::DuplicatePeer); - - if (!BEAST_EXPECT(slot2 == nullptr)) - logic.onClosed(slot2); - - logic.onClosed(slot1); - } - - // test establishing outgoing slot for an already existing incoming slot - void - testDuplicateInOut() - { - testcase("duplicate in/out"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - auto const remote = beast::IP::Endpoint::fromString("65.0.0.1:5"); - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - - auto const [slot1, r] = logic.newInboundSlot(local, remote); - BEAST_EXPECT(slot1 != nullptr); - BEAST_EXPECT(r == Result::Success); - BEAST_EXPECT(logic.connectedAddresses.count(remote.address()) == 1); - - auto const [slot2, r2] = logic.newOutboundSlot(remote); - BEAST_EXPECT(r2 == Result::DuplicatePeer); - BEAST_EXPECT(logic.connectedAddresses.count(remote.address()) == 1); - if (!BEAST_EXPECT(slot2 == nullptr)) - logic.onClosed(slot2); - logic.onClosed(slot1); - } - - void - testPeerLimitExceeded() - { - testcase("peer limit exceeded"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - auto const [slot, r] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1025")); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(r == Result::Success); - - auto const [slot1, r1] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1026")); - BEAST_EXPECT(slot1 != nullptr); - BEAST_EXPECT(r1 == Result::Success); - - auto const [slot2, r2] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1027")); - BEAST_EXPECT(r2 == Result::IpLimitExceeded); - - if (!BEAST_EXPECT(slot2 == nullptr)) - logic.onClosed(slot2); - logic.onClosed(slot1); - logic.onClosed(slot); - } - - void - testActivateDuplicatePeer() - { - testcase("test activate duplicate peer"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - - PublicKey const pk1(randomKeyPair(KeyType::Secp256k1).first); - - auto const [slot, rSlot] = - logic.newOutboundSlot(beast::IP::Endpoint::fromString("55.104.0.2:1025")); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(rSlot == Result::Success); - - auto const [slot2, r2Slot] = - logic.newOutboundSlot(beast::IP::Endpoint::fromString("55.104.0.2:1026")); - BEAST_EXPECT(slot2 != nullptr); - BEAST_EXPECT(r2Slot == Result::Success); - - BEAST_EXPECT(logic.onConnected(slot, local)); - BEAST_EXPECT(logic.onConnected(slot2, local)); - - BEAST_EXPECT(logic.activate(slot, pk1, false) == Result::Success); - - // activating a different slot with the same node ID (pk) must fail - BEAST_EXPECT(logic.activate(slot2, pk1, false) == Result::DuplicatePeer); - - logic.onClosed(slot); - - // accept the same key for a new slot after removing the old slot - BEAST_EXPECT(logic.activate(slot2, pk1, false) == Result::Success); - logic.onClosed(slot2); - } - - void - testActivateInboundDisabled() - { - testcase("test activate inbound disabled"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - PublicKey const pk1(randomKeyPair(KeyType::Secp256k1).first); - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - - auto const [slot, rSlot] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1025")); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(rSlot == Result::Success); - - BEAST_EXPECT(logic.activate(slot, pk1, false) == Result::InboundDisabled); - - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - c.inPeers = 1; - logic.config(c); - } - // new inbound slot must succeed when inbound connections are enabled - BEAST_EXPECT(logic.activate(slot, pk1, false) == Result::Success); - - // creating a new inbound slot must succeed as IP Limit is not exceeded - auto const [slot2, r2Slot] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1026")); - BEAST_EXPECT(slot2 != nullptr); - BEAST_EXPECT(r2Slot == Result::Success); - - PublicKey const pk2(randomKeyPair(KeyType::Secp256k1).first); - - // an inbound slot exceeding inPeers limit must fail - BEAST_EXPECT(logic.activate(slot2, pk2, false) == Result::Full); - - logic.onClosed(slot2); - logic.onClosed(slot); - } - - void - testAddFixedPeerNoPort() - { - testcase("test addFixedPeer no port"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - try - { - logic.addFixedPeer("test", beast::IP::Endpoint::fromString("65.0.0.2")); - fail("invalid endpoint successfully added"); - } - catch (std::runtime_error const& e) - { - pass(); - } - } - - void - testIsValidAddress() - { - testcase("is_valid_address"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - - auto const pass = [&](std::string const& s) { - BEAST_EXPECT(logic.isValidAddress(beast::IP::Endpoint::fromString(s))); - }; - auto const fail = [&](std::string const& s) { - BEAST_EXPECT(!logic.isValidAddress(beast::IP::Endpoint::fromString(s))); - }; - - // Invalid: port 0 - fail("65.0.0.1:0"); - - // --- IPv4 ranges --- - // For each range: 1 before (pass), first (fail), last (fail), - // 1 after (pass) - - // 0.0.0.0/8 - "This network" - // No "before" - nothing before 0.0.0.0 - fail("0.0.0.0:8080"); - fail("0.255.255.255:8080"); - pass("1.0.0.0:8080"); - - // 10.0.0.0/8 - Private (RFC 1918) - pass("9.255.255.255:8080"); - fail("10.0.0.0:8080"); - fail("10.255.255.255:8080"); - pass("11.0.0.0:8080"); - - // 100.64.0.0/10 - Shared Address Space / CGNAT (RFC 6598) - pass("100.63.255.255:8080"); - fail("100.64.0.0:8080"); - fail("100.127.255.255:8080"); - pass("100.128.0.0:8080"); - - // 127.0.0.0/8 - Loopback - pass("126.255.255.255:8080"); - fail("127.0.0.0:8080"); - fail("127.255.255.255:8080"); - pass("128.0.0.0:8080"); - - // 169.254.0.0/16 - Link-local - pass("169.253.255.255:8080"); - fail("169.254.0.0:8080"); - fail("169.254.255.255:8080"); - pass("169.255.0.0:8080"); - - // 172.16.0.0/12 - Private (RFC 1918) - pass("172.15.255.255:8080"); - fail("172.16.0.0:8080"); - fail("172.31.255.255:8080"); - pass("172.32.0.0:8080"); - - // 192.0.0.0/24 - IETF Protocol Assignments (RFC 6890) - pass("191.255.255.255:8080"); - fail("192.0.0.0:8080"); - fail("192.0.0.255:8080"); - pass("192.0.1.0:8080"); - - // 192.0.2.0/24 - TEST-NET-1 (RFC 5737) - pass("192.0.1.255:8080"); - fail("192.0.2.0:8080"); - fail("192.0.2.255:8080"); - pass("192.0.3.0:8080"); - - // 192.88.99.0/24 - 6to4 Relay Anycast (RFC 7526) - pass("192.88.98.255:8080"); - fail("192.88.99.0:8080"); - fail("192.88.99.255:8080"); - pass("192.88.100.0:8080"); - - // 192.168.0.0/16 - Private (RFC 1918) - pass("192.167.255.255:8080"); - fail("192.168.0.0:8080"); - fail("192.168.255.255:8080"); - pass("192.169.0.0:8080"); - - // 198.18.0.0/15 - Benchmarking (RFC 2544) - pass("198.17.255.255:8080"); - fail("198.18.0.0:8080"); - fail("198.19.255.255:8080"); - pass("198.20.0.0:8080"); - - // 198.51.100.0/24 - TEST-NET-2 (RFC 5737) - pass("198.51.99.255:8080"); - fail("198.51.100.0:8080"); - fail("198.51.100.255:8080"); - pass("198.51.101.0:8080"); - - // 203.0.113.0/24 - TEST-NET-3 (RFC 5737) - pass("203.0.112.255:8080"); - fail("203.0.113.0:8080"); - fail("203.0.113.255:8080"); - pass("203.0.114.0:8080"); - - // 224.0.0.0/4 - Multicast - pass("223.255.255.255:8080"); - fail("224.0.0.0:8080"); - fail("239.255.255.255:8080"); - // 240.0.0.0 (after multicast) is also blocked (reserved) - - // 240.0.0.0/4 - Reserved (RFC 1112) - // 239.255.255.255 (before reserved) is also blocked (multicast) - fail("240.0.0.0:8080"); - fail("255.255.255.255:8080"); - - // --- IPv6 ranges --- - - // ::1 - Loopback (single address) - fail("[::1]:8080"); - - // :: - Unspecified (single address) - fail("[::]:8080"); - - // fc00::/7 - Unique Local Address (ULA) - pass("[fb00::1]:8080"); - fail("[fc00::1]:8080"); - fail("[fdff::1]:8080"); - pass("[fe00::1]:8080"); - - // fe80::/10 - Link-local - pass("[fe7f::1]:8080"); - fail("[fe80::1]:8080"); - fail("[febf::1]:8080"); - pass("[fec0::1]:8080"); - - // ff00::/8 - Multicast - pass("[feff::1]:8080"); - fail("[ff00::1]:8080"); - fail("[ffff::1]:8080"); - // No "after" - ffff:... is the highest IPv6 range - - // 100::/64 - Discard prefix (RFC 6666) - pass("[ff::1]:8080"); - fail("[100::]:8080"); - fail("[100::ffff:ffff:ffff:ffff]:8080"); - pass("[100:0:0:1::1]:8080"); - - // 2001::/32 - IETF Protocol Assignments / Teredo (RFC 4380) - pass("[2000:ffff::1]:8080"); - fail("[2001::]:8080"); - fail("[2001:0:ffff::1]:8080"); - pass("[2001:1::1]:8080"); - - // 2001:20::/28 - ORCHIDv2 (RFC 7343) - pass("[2001:1f::1]:8080"); - fail("[2001:20::1]:8080"); - fail("[2001:2f::1]:8080"); - pass("[2001:30::1]:8080"); - - // 2001:db8::/32 - Documentation (RFC 3849) - pass("[2001:db7::1]:8080"); - fail("[2001:db8::1]:8080"); - fail("[2001:db8:ffff::1]:8080"); - pass("[2001:db9::1]:8080"); - - // 2002::/16 - 6to4 (RFC 3056, deprecated) - pass("[2001:ffff::1]:8080"); - fail("[2002::1]:8080"); - fail("[2002:ffff::1]:8080"); - pass("[2003::1]:8080"); - - // --- IPv6 v4-mapped (delegates to IPv4 checks) --- - fail("[::ffff:10.0.0.1]:8080"); - fail("[::ffff:100.64.0.1]:8080"); - fail("[::ffff:169.254.1.1]:8080"); - fail("[::ffff:192.0.2.1]:8080"); - fail("[::ffff:198.18.0.1]:8080"); - fail("[::ffff:224.0.0.1]:8080"); - fail("[::ffff:240.0.0.1]:8080"); - - // --- Valid public addresses --- - pass("8.8.8.8:443"); - pass("65.0.0.1:8080"); - pass("[2001:4860:4860::8888]:8080"); - pass("[2606:4700:4700::1111]:8080"); - } - - void - testVerifyEndpoints() - { - // Helper that sets up a Logic instance, creates and activates a slot, - // then calls on_endpoints with the given list and returns the - // livecache size afterwards. - auto run = [&](bool verifyEndpoints, Endpoints eps) -> std::size_t { - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - c.verifyEndpoints = verifyEndpoints; - logic.config(c); - } - - auto const remote = beast::IP::Endpoint::fromString("65.0.0.1:5"); - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - - auto const [slot, r] = logic.newOutboundSlot(remote); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(r == Result::Success); - BEAST_EXPECT(logic.onConnected(slot, local)); - - PublicKey const pk(randomKeyPair(KeyType::Secp256k1).first); - BEAST_EXPECT(logic.activate(slot, pk, false) == Result::Success); - - logic.onEndpoints(slot, std::move(eps)); - - auto const size = logic.livecache.size(); - logic.onClosed(slot); - return size; - }; - - { - testcase("verify_endpoints enabled"); - - // Valid public addresses - Endpoints eps; - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.1:5"), 1); - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.2:6"), 1); - // Invalid: private address - eps.emplace_back(beast::IP::Endpoint::fromString("10.0.0.1:5"), 1); - // Invalid: port 0 - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.3:0"), 1); - - // With verification enabled, only the 2 valid endpoints survive - BEAST_EXPECT(run(true, eps) == 2); - } - { - testcase("verify_endpoints disabled"); - - Endpoints eps; - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.1:5"), 1); - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.2:6"), 1); - // Private address — kept when verification is off - eps.emplace_back(beast::IP::Endpoint::fromString("10.0.0.1:5"), 1); - // Port 0 — kept when verification is off - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.3:0"), 1); - - // Without verification, all 4 endpoints survive - BEAST_EXPECT(run(false, eps) == 4); - } - } - - void - testOnConnectedSelfConnection() - { - testcase("test onConnected self connection"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1234"); - auto const [slot, r] = logic.newOutboundSlot(local); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(r == Result::Success); - - // Must fail when a slot is to our own IP address - BEAST_EXPECT(!logic.onConnected(slot, local)); - logic.onClosed(slot); - } - - void - testConfig() - { - // if peers_max is configured then peers_in_max and peers_out_max - // are ignored - auto run = [&](std::string const& test, - std::optional maxPeers, - std::optional maxIn, - std::optional maxOut, - std::uint16_t port, - std::uint16_t expectOut, - std::uint16_t expectIn, - std::uint16_t expectIpLimit) { - xrpl::Config c; - - testcase(test); - - std::string toLoad; - int max = 0; - if (maxPeers) - { - max = maxPeers.value(); - toLoad += "[peers_max]\n" + std::to_string(max) + "\n" + "[peers_in_max]\n" + - std::to_string(maxIn.value_or(0)) + "\n" + "[peers_out_max]\n" + - std::to_string(maxOut.value_or(0)) + "\n"; - } - else if (maxIn && maxOut) - { - toLoad += "[peers_in_max]\n" + std::to_string(*maxIn) + "\n" + "[peers_out_max]\n" + - std::to_string(*maxOut) + "\n"; - } - - c.loadFromString(toLoad); - BEAST_EXPECT( - (c.peersMax == max && c.peersInMax == 0 && c.peersOutMax == 0) || - (c.peersInMax == *maxIn && c.peersOutMax == *maxOut)); - - Config const config = Config::makeConfig(c, port, false, 0, true); - - Counts counts; - counts.onConfig(config); - BEAST_EXPECT( - counts.outMax() == expectOut && counts.inMax() == expectIn && - config.ipLimit == expectIpLimit); - - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - logic.config(config); - - BEAST_EXPECT(logic.config() == config); - }; - - // if max_peers == 0 => maxPeers = 21, - // else if max_peers < 10 => maxPeers = 10 else maxPeers = - // max_peers - // expectOut => if legacy => max(0.15 * maxPeers, 10), - // if legacy && !wantIncoming => maxPeers else max_out_peers - // expectIn => if legacy && wantIncoming => maxPeers - outPeers - // else if !wantIncoming => 0 else max_in_peers - // ipLimit => if expectIn <= 21 => 2 else 2 + min(5, expectIn/21) - // ipLimit = max(1, min(ipLimit, expectIn/2)) - - // legacy test with max_peers - run("legacy no config", {}, {}, {}, 4000, 10, 11, 2); - run("legacy max_peers 0", 0, 100, 10, 4000, 10, 11, 2); - run("legacy max_peers 5", 5, 100, 10, 4000, 10, 0, 1); - run("legacy max_peers 20", 20, 100, 10, 4000, 10, 10, 2); - run("legacy max_peers 100", 100, 100, 10, 4000, 15, 85, 6); - run("legacy max_peers 20, private", 20, 100, 10, 0, 20, 0, 1); - - // test with max_in_peers and max_out_peers - run("new in 100/out 10", {}, 100, 10, 4000, 10, 100, 6); - run("new in 0/out 10", {}, 0, 10, 4000, 10, 0, 1); - run("new in 100/out 10, private", {}, 100, 10, 0, 10, 0, 6); - } - - void - testInvalidConfig() - { - testcase("invalid config"); - - auto run = [&](std::string const& toLoad) { - xrpl::Config c; - try - { - c.loadFromString(toLoad); - fail(); - } - catch (...) - { - pass(); - } - }; - run(R"xrpldConfig( -[peers_in_max] -100 -)xrpldConfig"); - run(R"xrpldConfig( -[peers_out_max] -100 -)xrpldConfig"); - run(R"xrpldConfig( -[peers_in_max] -100 -[peers_out_max] -5 -)xrpldConfig"); - run(R"xrpldConfig( -[peers_in_max] -1001 -[peers_out_max] -10 -)xrpldConfig"); - run(R"xrpldConfig( -[peers_in_max] -10 -[peers_out_max] -1001 -)xrpldConfig"); - } - - void - run() override - { - testBackoff1(); - testBackoff2(); - testDuplicateOutIn(); - testDuplicateInOut(); - testConfig(); - testInvalidConfig(); - testPeerLimitExceeded(); - testActivateDuplicatePeer(); - testActivateInboundDisabled(); - testAddFixedPeerNoPort(); - testOnConnectedSelfConnection(); - testIsValidAddress(); - testVerifyEndpoints(); - } -}; - -BEAST_DEFINE_TESTSUITE(PeerFinder, peerfinder, xrpl); - -} // namespace xrpl::PeerFinder diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index cafe72eff9..2e131b895e 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -21,7 +21,7 @@ set_target_properties( ) # Lets test sources include the shared helpers as . target_include_directories(xrpl_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(xrpl_tests PRIVATE GTest::gtest xrpl.libxrpl) +target_link_libraries(xrpl_tests PRIVATE GTest::gtest GTest::gmock xrpl.libxrpl) # One source subdirectory per module. Network unit tests are currently not # supported on Windows. @@ -29,6 +29,7 @@ set(test_modules basics crypto json + peerfinder resource shamap tx diff --git a/src/tests/libxrpl/main.cpp b/src/tests/libxrpl/main.cpp index 5142bbe08a..f9114bffc4 100644 --- a/src/tests/libxrpl/main.cpp +++ b/src/tests/libxrpl/main.cpp @@ -1,8 +1,9 @@ +#include #include int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/src/tests/libxrpl/peerfinder/Livecache.cpp b/src/tests/libxrpl/peerfinder/Livecache.cpp new file mode 100644 index 0000000000..464ec0e5da --- /dev/null +++ b/src/tests/libxrpl/peerfinder/Livecache.cpp @@ -0,0 +1,294 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::PeerFinder { +namespace { + +class LivecacheTest : public ::testing::Test +{ +protected: + static beast::Journal + journal() + { + return beast::Journal{TestSink::instance()}; + } + + static beast::IP::Endpoint + endpoint(std::uint16_t index, bool v4 = true) + { + auto const port = static_cast(10000 + index); + + if (v4) + { + auto bytes = beast::IP::AddressV4::bytes_type{ + {54, + static_cast((index / 256) % 256), + static_cast(index % 256), + 1}}; + return beast::IP::Endpoint{beast::IP::Address{beast::IP::AddressV4{bytes}}, port}; + } + + auto bytes = beast::IP::AddressV6::bytes_type{ + {0x20, + 0x01, + 0x0d, + 0xb8, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + static_cast((index / 256) % 256), + static_cast(index % 256), + 1}}; + return beast::IP::Endpoint{beast::IP::Address{beast::IP::AddressV6{bytes}}, port}; + } + + void + addEndpoint(beast::IP::Endpoint const& ep, std::uint32_t hops = 0) + { + cache_.insert(Endpoint{ep, hops}); + } + + TestStopwatch clock_; + Livecache<> cache_{clock_, journal()}; +}; + +} // namespace + +TEST_F(LivecacheTest, basic_insert) +{ + EXPECT_TRUE(cache_.empty()); + + for (auto i = 0; i < 10; ++i) + addEndpoint(endpoint(i, true)); + + EXPECT_FALSE(cache_.empty()); + EXPECT_EQ(cache_.size(), 10u); + + for (auto i = 10; i < 20; ++i) + addEndpoint(endpoint(i, false)); + + EXPECT_FALSE(cache_.empty()); + EXPECT_EQ(cache_.size(), 20u); +} + +TEST_F(LivecacheTest, insert_update_keeps_lowest_hop_count) +{ + auto const ep1 = Endpoint{endpoint(1), 2}; + cache_.insert(ep1); + ASSERT_EQ(cache_.size(), 1u); + EXPECT_EQ((cache_.hops.begin() + 2)->begin()->hops, 2u); + + auto const ep2 = Endpoint{ep1.address, 4}; + cache_.insert(ep2); + EXPECT_EQ(cache_.size(), 1u); + EXPECT_EQ((cache_.hops.begin() + 2)->begin()->hops, 2u); + + auto const ep3 = Endpoint{ep1.address, 2}; + cache_.insert(ep3); + EXPECT_EQ(cache_.size(), 1u); + EXPECT_EQ((cache_.hops.begin() + 2)->begin()->hops, 2u); + + auto const ep4 = Endpoint{ep1.address, 1}; + cache_.insert(ep4); + EXPECT_EQ(cache_.size(), 1u); + EXPECT_EQ((cache_.hops.begin() + 1)->begin()->hops, 1u); +} + +TEST_F(LivecacheTest, hop_iterators_support_const_reverse_and_move_back) +{ + auto const ep1 = Endpoint{endpoint(1), 1}; + auto const ep2 = Endpoint{endpoint(2), 1}; + cache_.insert(ep1); + cache_.insert(ep2); + + auto hop = *(cache_.hops.begin() + 1); + ASSERT_NE(hop.begin(), hop.end()); + ASSERT_NE(hop.cbegin(), hop.cend()); + ASSERT_NE(hop.rbegin(), hop.rend()); + ASSERT_NE(hop.crbegin(), hop.crend()); + + auto const firstAddress = hop.begin()->address; + hop.moveBack(hop.begin()); + EXPECT_EQ(hop.rbegin()->address, firstAddress); + + auto const& constHops = cache_.hops; + EXPECT_NE(constHops.begin(), constHops.end()); + EXPECT_NE(constHops.cbegin(), constHops.cend()); + EXPECT_NE(constHops.rbegin(), constHops.rend()); + EXPECT_NE(constHops.crbegin(), constHops.crend()); + + auto const constHop = *(constHops.cbegin() + 1); + EXPECT_EQ(std::distance(constHop.begin(), constHop.end()), 2); + EXPECT_EQ(std::distance(constHop.cbegin(), constHop.cend()), 2); + EXPECT_EQ(std::distance(constHop.rbegin(), constHop.rend()), 2); + EXPECT_EQ(std::distance(constHop.crbegin(), constHop.crend()), 2); +} + +TEST_F(LivecacheTest, on_write_reports_entries_and_expiration) +{ + cache_.insert(Endpoint{endpoint(1), 1}); + cache_.insert(Endpoint{endpoint(2), Tuning::kMaxHops + 1}); + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + cache_.onWrite(map); + } + + auto const& top = stream.top(); + EXPECT_EQ(top["size"].asUInt(), 2u); + EXPECT_FALSE(top["hist"].asString().empty()); + ASSERT_TRUE(top.isMember("entries")); + ASSERT_EQ(top["entries"].size(), 2u); + auto const& entry = top["entries"][json::UInt{0}]; + EXPECT_TRUE(entry.isMember("hops")); + EXPECT_TRUE(entry.isMember("address")); + EXPECT_TRUE(entry.isMember("expires")); +} + +TEST_F(LivecacheTest, expire_removes_entries_after_ttl) +{ + using namespace std::chrono_literals; + + cache_.insert(Endpoint{endpoint(1), 1}); + ASSERT_EQ(cache_.size(), 1u); + + cache_.expire(); + EXPECT_EQ(cache_.size(), 1u); + + clock_.advance(Tuning::kLiveCacheSecondsToLive - 1s); + cache_.expire(); + EXPECT_EQ(cache_.size(), 1u); + + clock_.advance(1s); + cache_.expire(); + EXPECT_TRUE(cache_.empty()); +} + +TEST_F(LivecacheTest, expire_removes_multiple_entries_after_ttl) +{ + using namespace std::chrono_literals; + + cache_.insert(Endpoint{endpoint(1), 1}); + cache_.insert(Endpoint{endpoint(2), 2}); + + clock_.advance(Tuning::kLiveCacheSecondsToLive); + cache_.expire(); + EXPECT_TRUE(cache_.empty()); +} + +TEST_F(LivecacheTest, histogram_counts_all_entries) +{ + constexpr auto kNumEndpoints = 40; + + for (auto i = 0; i < kNumEndpoints; ++i) + { + addEndpoint(endpoint(static_cast(i)), xrpl::randInt()); + } + + auto const histogram = cache_.hops.histogram(); + ASSERT_FALSE(histogram.empty()); + + std::vector values; + boost::split(values, histogram, boost::algorithm::is_any_of(",")); + + auto sum = 0; + for (auto const& value : values) + { + auto const count = boost::lexical_cast(boost::trim_copy(value)); + sum += count; + EXPECT_GE(count, 0); + } + EXPECT_EQ(sum, kNumEndpoints); +} + +TEST_F(LivecacheTest, shuffle_preserves_bucket_contents) +{ + for (auto i = 0; i < 100; ++i) + { + addEndpoint(endpoint(static_cast(i)), xrpl::randInt(Tuning::kMaxHops + 1)); + } + + using AtHop = std::vector; + using AllHops = std::array; + + auto const compareEndpoint = [](Endpoint const& lhs, Endpoint const& rhs) { + return rhs.hops < lhs.hops || (rhs.hops == lhs.hops && rhs.address < lhs.address); + }; + auto const sameEndpoint = [](Endpoint const& lhs, Endpoint const& rhs) { + return lhs.hops == rhs.hops && lhs.address == rhs.address; + }; + auto const sameEndpoints = + [&sameEndpoint](std::vector const& lhs, std::vector const& rhs) { + return lhs.size() == rhs.size() && + std::equal(lhs.begin(), lhs.end(), rhs.begin(), sameEndpoint); + }; + + AllHops before; + AllHops beforeSorted; + for (auto i = std::make_pair(0, cache_.hops.begin()); i.second != cache_.hops.end(); + ++i.first, ++i.second) + { + std::ranges::copy(*i.second, std::back_inserter(before[i.first])); + std::ranges::copy(*i.second, std::back_inserter(beforeSorted[i.first])); + std::ranges::sort(beforeSorted[i.first], compareEndpoint); + } + + cache_.hops.shuffle(); + + AllHops after; + AllHops afterSorted; + for (auto i = std::make_pair(0, cache_.hops.begin()); i.second != cache_.hops.end(); + ++i.first, ++i.second) + { + std::ranges::copy(*i.second, std::back_inserter(after[i.first])); + std::ranges::copy(*i.second, std::back_inserter(afterSorted[i.first])); + std::ranges::sort(afterSorted[i.first], compareEndpoint); + } + + auto allBucketsKeptOriginalOrder = true; + for (auto i = 0u; i < before.size(); ++i) + { + EXPECT_EQ(before[i].size(), after[i].size()); + allBucketsKeptOriginalOrder = + allBucketsKeptOriginalOrder && sameEndpoints(before[i], after[i]); + EXPECT_TRUE(sameEndpoints(beforeSorted[i], afterSorted[i])); + } + EXPECT_FALSE(allBucketsKeptOriginalOrder); +} + +} // namespace xrpl::PeerFinder diff --git a/src/tests/libxrpl/peerfinder/PeerFinder.cpp b/src/tests/libxrpl/peerfinder/PeerFinder.cpp new file mode 100644 index 0000000000..31fa59d1ce --- /dev/null +++ b/src/tests/libxrpl/peerfinder/PeerFinder.cpp @@ -0,0 +1,1270 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::PeerFinder { +namespace { + +using ::testing::_; +using ::testing::NiceMock; +using ::testing::Return; + +beast::Journal +journal() +{ + return beast::Journal{TestSink::instance()}; +} + +beast::IP::Endpoint +endpoint(std::string const& value) +{ + return beast::IP::Endpoint::fromString(value); +} + +class MockStore : public Store +{ +public: + MOCK_METHOD(std::size_t, load, (Store::load_callback const& cb), (override)); + MOCK_METHOD(void, save, (std::vector const& entries), (override)); +}; + +class CapturingStore : public Store +{ +public: + std::vector entriesToLoad; + std::vector> saves; + + std::size_t + load(Store::load_callback const& cb) override + { + for (auto const& entry : entriesToLoad) + cb(entry.endpoint, entry.valence); + return entriesToLoad.size(); + } + + void + save(std::vector const& entries) override + { + saves.push_back(entries); + } +}; + +Store::Entry +storeEntry(beast::IP::Endpoint const& endpoint, int valence) +{ + Store::Entry entry; + entry.endpoint = endpoint; + entry.valence = valence; + return entry; +} + +void +allowEmptyStore(MockStore& store) +{ + ON_CALL(store, load(_)).WillByDefault(Return(0)); + ON_CALL(store, save(_)).WillByDefault([](std::vector const&) {}); +} + +class MockChecker +{ +public: + MOCK_METHOD(void, stop, ()); + MOCK_METHOD(void, wait, ()); + MOCK_METHOD(void, recordAsyncConnect, (beast::IP::Endpoint const& ep)); + + boost::system::error_code nextError; + bool completeAsync = true; + std::vector asyncConnects; + + template + void + asyncConnect(beast::IP::Endpoint const& ep, Handler&& handler) + { + asyncConnects.push_back(ep); + recordAsyncConnect(ep); + if (completeAsync) + std::forward(handler)(nextError); + } +}; + +class TestSource : public Source +{ +public: + explicit TestSource(std::string name) : name_(std::move(name)) + { + } + + std::string const& + name() override + { + return name_; + } + + void + cancel() override + { + ++cancelCount; + } + + void + fetch(Results& results, beast::Journal) override + { + ++fetchCount; + results = resultsToFetch; + } + + Results resultsToFetch; + int fetchCount = 0; + int cancelCount = 0; + +private: + std::string name_; +}; + +class DefaultCancelSource : public Source +{ +public: + std::string const& + name() override + { + return name_; + } + + void + fetch(Results& results, beast::Journal) override + { + results = resultsToFetch; + } + + Results resultsToFetch; + +private: + std::string name_{"default"}; +}; + +class PeerFinderTest : public ::testing::Test +{ +public: + PeerFinderTest() + { + allowEmptyStore(store_); + } + +protected: + void + configure(std::size_t ipLimit = 2) + { + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = static_cast(ipLimit); + logic_.config(config); + } + + NiceMock store_; + NiceMock checker_; + TestStopwatch clock_; + Logic> logic_{clock_, store_, checker_, journal()}; +}; + +int +savedValence(std::vector const& entries, beast::IP::Endpoint const& endpoint) +{ + for (auto const& entry : entries) + { + if (entry.endpoint == endpoint) + return entry.valence; + } + + ADD_FAILURE() << "missing saved endpoint " << endpoint.toString(); + return 0; +} + +TEST_F(PeerFinderTest, backoff_limits_repeated_connection_attempts) +{ + auto constexpr kSECONDS = 10000; + + logic_.addFixedPeer("test", endpoint("65.0.0.1:5")); + configure(); + + std::size_t attempts = 0; + for (std::size_t i = 0; i < kSECONDS; ++i) + { + auto const list = logic_.autoconnect(); + if (!list.empty()) + { + ASSERT_EQ(list.size(), 1u); + auto const [slot, result] = logic_.newOutboundSlot(list.front()); + ASSERT_NE(slot, nullptr); + ASSERT_EQ(result, Result::Success); + EXPECT_TRUE(logic_.onConnected(slot, endpoint("65.0.0.2:5"))); + logic_.onClosed(slot); + ++attempts; + } + clock_.advance(std::chrono::seconds(1)); + logic_.oncePerSecond(); + } + + EXPECT_LT(attempts, 20u); +} + +TEST_F(PeerFinderTest, activated_peer_backoff_allows_at_most_one_attempt_per_minute) +{ + auto constexpr kSECONDS = 10000; + + logic_.addFixedPeer("test", endpoint("65.0.0.1:5")); + configure(); + + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + + std::size_t attempts = 0; + for (std::size_t i = 0; i < kSECONDS; ++i) + { + auto const list = logic_.autoconnect(); + if (!list.empty()) + { + ASSERT_EQ(list.size(), 1u); + auto const [slot, result] = logic_.newOutboundSlot(list.front()); + ASSERT_NE(slot, nullptr); + ASSERT_EQ(result, Result::Success); + ASSERT_TRUE(logic_.onConnected(slot, endpoint("65.0.0.2:5"))); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + logic_.onClosed(slot); + ++attempts; + } + clock_.advance(std::chrono::seconds(1)); + logic_.oncePerSecond(); + } + + EXPECT_LE(attempts, (kSECONDS + 59u) / 60u); +} + +TEST_F(PeerFinderTest, duplicate_inbound_slot_is_rejected_for_existing_outbound_slot) +{ + configure(); + + auto const remote = endpoint("65.0.0.1:5"); + auto const [slot1, result1] = logic_.newOutboundSlot(remote); + ASSERT_NE(slot1, nullptr); + EXPECT_EQ(result1, Result::Success); + EXPECT_EQ(logic_.connectedAddresses.count(remote.address()), 1u); + + auto const local = endpoint("65.0.0.2:1024"); + auto const [slot2, result2] = logic_.newInboundSlot(local, remote); + EXPECT_EQ(logic_.connectedAddresses.count(remote.address()), 1u); + EXPECT_EQ(result2, Result::DuplicatePeer); + EXPECT_EQ(slot2, nullptr); + + if (slot2) + logic_.onClosed(slot2); + logic_.onClosed(slot1); +} + +TEST_F(PeerFinderTest, duplicate_outbound_slot_is_rejected_for_existing_inbound_slot) +{ + configure(); + + auto const remote = endpoint("65.0.0.1:5"); + auto const local = endpoint("65.0.0.2:1024"); + + auto const [slot1, result1] = logic_.newInboundSlot(local, remote); + ASSERT_NE(slot1, nullptr); + EXPECT_EQ(result1, Result::Success); + EXPECT_EQ(logic_.connectedAddresses.count(remote.address()), 1u); + + auto const [slot2, result2] = logic_.newOutboundSlot(remote); + EXPECT_EQ(result2, Result::DuplicatePeer); + EXPECT_EQ(logic_.connectedAddresses.count(remote.address()), 1u); + EXPECT_EQ(slot2, nullptr); + + if (slot2) + logic_.onClosed(slot2); + logic_.onClosed(slot1); +} + +TEST_F(PeerFinderTest, peer_limit_exceeded_rejects_additional_inbound_slot) +{ + configure(); + + auto const local = endpoint("65.0.0.2:1024"); + auto const [slot, result] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1025")); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + + auto const [slot1, result1] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1026")); + ASSERT_NE(slot1, nullptr); + EXPECT_EQ(result1, Result::Success); + + auto const [slot2, result2] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1027")); + EXPECT_EQ(result2, Result::IpLimitExceeded); + EXPECT_EQ(slot2, nullptr); + + if (slot2) + logic_.onClosed(slot2); + logic_.onClosed(slot1); + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, activate_rejects_duplicate_public_key) +{ + configure(); + + auto const local = endpoint("65.0.0.2:1024"); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + + auto const [slot, result] = logic_.newOutboundSlot(endpoint("55.104.0.2:1025")); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + + auto const [slot2, result2] = logic_.newOutboundSlot(endpoint("55.104.0.2:1026")); + ASSERT_NE(slot2, nullptr); + EXPECT_EQ(result2, Result::Success); + + EXPECT_TRUE(logic_.onConnected(slot, local)); + EXPECT_TRUE(logic_.onConnected(slot2, local)); + + EXPECT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + EXPECT_EQ(logic_.activate(slot2, publicKey, false), Result::DuplicatePeer); + + logic_.onClosed(slot); + + EXPECT_EQ(logic_.activate(slot2, publicKey, false), Result::Success); + logic_.onClosed(slot2); +} + +TEST_F(PeerFinderTest, activate_rejects_inbound_when_inbound_connections_are_disabled) +{ + configure(); + + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + auto const local = endpoint("65.0.0.2:1024"); + + auto const [slot, result] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1025")); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + + EXPECT_EQ(logic_.activate(slot, publicKey, false), Result::InboundDisabled); + + { + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.inPeers = 1; + logic_.config(config); + } + + EXPECT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + auto const [slot2, result2] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1026")); + ASSERT_NE(slot2, nullptr); + EXPECT_EQ(result2, Result::Success); + + PublicKey const publicKey2(randomKeyPair(KeyType::Secp256k1).first); + EXPECT_EQ(logic_.activate(slot2, publicKey2, false), Result::Full); + + logic_.onClosed(slot2); + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, add_fixed_peer_rejects_endpoint_without_port) +{ + EXPECT_THROW(logic_.addFixedPeer("test", endpoint("65.0.0.2")), std::runtime_error); +} + +TEST_F(PeerFinderTest, on_connected_rejects_self_connection) +{ + auto const local = endpoint("65.0.0.2:1234"); + auto const [slot, result] = logic_.newOutboundSlot(local); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + + EXPECT_FALSE(logic_.onConnected(slot, local)); + logic_.onClosed(slot); +} + +TEST(PeerFinderResult, converts_all_result_values_to_strings) +{ + EXPECT_EQ(to_string(Result::InboundDisabled), "inbound disabled"); + EXPECT_EQ(to_string(Result::DuplicatePeer), "peer already connected"); + EXPECT_EQ(to_string(Result::IpLimitExceeded), "ip limit exceeded"); + EXPECT_EQ(to_string(Result::Full), "slots full"); + EXPECT_EQ(to_string(Result::Success), "success"); + EXPECT_EQ(to_string(static_cast(-1)), "unknown"); +} + +TEST(PeerFinderEndpoint, orders_by_address) +{ + Endpoint const high{endpoint("65.0.0.2:10002"), 1}; + Endpoint const low{endpoint("65.0.0.1:10001"), 2}; + std::vector endpoints{high, low}; + + std::ranges::sort( + endpoints, [](Endpoint const& lhs, Endpoint const& rhs) { return lhs < rhs; }); + + EXPECT_EQ(endpoints.front().address, low.address); + EXPECT_EQ(endpoints.back().address, high.address); +} + +TEST(PeerFinderCounts, tracks_slot_states_and_capacity) +{ + TestStopwatch clock; + Counts counts; + Config config; + config.outPeers = 1; + config.inPeers = 1; + config.wantIncoming = true; + counts.onConfig(config); + + EXPECT_EQ(counts.outMax(), 1); + EXPECT_EQ(counts.inMax(), 1); + EXPECT_EQ(counts.inboundSlotsFree(), 1); + EXPECT_EQ(counts.outboundSlotsFree(), 1); + EXPECT_EQ(counts.totalActive(), 0); + EXPECT_FALSE(counts.isConnectedToNetwork()); + EXPECT_EQ(counts.attemptsNeeded(), Tuning::kMaxConnectAttempts); + EXPECT_EQ(counts.stateString(), "0/1 out, 0/1 in, 0 connecting, 0 closing"); + + SlotImp inbound(endpoint("65.0.0.1:10001"), endpoint("65.0.0.2:10002"), false, clock); + counts.add(inbound); + EXPECT_EQ(counts.acceptCount(), 1); + EXPECT_TRUE(counts.canActivate(inbound)); + counts.remove(inbound); + EXPECT_EQ(counts.acceptCount(), 0); + + inbound.activate(clock.now()); + counts.add(inbound); + EXPECT_EQ(counts.inboundActive(), 1); + EXPECT_EQ(counts.totalActive(), 1); + EXPECT_EQ(counts.inboundSlotsFree(), 0); + + SlotImp const extraInbound( + endpoint("65.0.0.3:10003"), endpoint("65.0.0.4:10004"), false, clock); + EXPECT_FALSE(counts.canActivate(extraInbound)); + counts.remove(inbound); + + SlotImp outbound(endpoint("65.0.0.5:10005"), false, clock); + counts.add(outbound); + EXPECT_EQ(counts.attempts(), 1); + EXPECT_EQ(counts.connectCount(), 1); + EXPECT_EQ(counts.attemptsNeeded(), Tuning::kMaxConnectAttempts - 1); + counts.remove(outbound); + + outbound.state(Slot::State::Connected); + EXPECT_TRUE(counts.canActivate(outbound)); + outbound.activate(clock.now()); + counts.add(outbound); + EXPECT_EQ(counts.outActive(), 1); + EXPECT_EQ(counts.outboundSlotsFree(), 0); + + SlotImp extraOutbound(endpoint("65.0.0.6:10006"), false, clock); + extraOutbound.state(Slot::State::Connected); + EXPECT_FALSE(counts.canActivate(extraOutbound)); + + SlotImp fixedOutbound(endpoint("65.0.0.7:10007"), true, clock); + fixedOutbound.state(Slot::State::Connected); + EXPECT_TRUE(counts.canActivate(fixedOutbound)); + fixedOutbound.activate(clock.now()); + counts.add(fixedOutbound); + EXPECT_EQ(counts.fixed(), 1u); + EXPECT_EQ(counts.fixedActive(), 1u); + counts.remove(fixedOutbound); + + SlotImp reservedOutbound(endpoint("65.0.0.8:10008"), false, clock); + reservedOutbound.reserved(true); + reservedOutbound.state(Slot::State::Connected); + EXPECT_TRUE(counts.canActivate(reservedOutbound)); + reservedOutbound.activate(clock.now()); + counts.add(reservedOutbound); + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + counts.onWrite(map); + } + EXPECT_TRUE(stream.top().isMember("accept")); + EXPECT_TRUE(stream.top().isMember("connect")); + EXPECT_TRUE(stream.top().isMember("close")); + EXPECT_TRUE(stream.top().isMember("reserved")); + EXPECT_TRUE(stream.top().isMember("total")); + counts.remove(reservedOutbound); + counts.remove(outbound); + + SlotImp closing(endpoint("65.0.0.9:10009"), endpoint("65.0.0.10:10010"), false, clock); + closing.state(Slot::State::Closing); + counts.add(closing); + EXPECT_EQ(counts.closingCount(), 1); + counts.remove(closing); + + Counts saturatedAttempts; + saturatedAttempts.onConfig(config); + std::vector> attempts; + for (int i = 0; i < Tuning::kMaxConnectAttempts; ++i) + { + attempts.push_back( + std::make_unique( + endpoint("65.1.0." + std::to_string(i + 1) + ":" + std::to_string(11000 + i)), + false, + clock)); + saturatedAttempts.add(*attempts.back()); + } + EXPECT_EQ(saturatedAttempts.attempts(), Tuning::kMaxConnectAttempts); + EXPECT_EQ(saturatedAttempts.attemptsNeeded(), 0u); + + Config disconnected; + disconnected.outPeers = 0; + counts.onConfig(disconnected); + EXPECT_TRUE(counts.isConnectedToNetwork()); +} + +TEST(PeerFinderHandouts, filters_redirect_slot_and_connect_targets) +{ + TestStopwatch clock; + auto const remote = endpoint("65.0.0.2:10002"); + auto const slot = std::make_shared(endpoint("65.0.0.1:10001"), remote, false, clock); + + RedirectHandouts redirects(slot); + EXPECT_EQ(redirects.slot(), slot); + EXPECT_TRUE(redirects.list().empty()); + EXPECT_FALSE(redirects.full()); + EXPECT_FALSE(redirects.tryInsert(Endpoint{endpoint("65.0.0.3:10003"), Tuning::kMaxHops + 1})); + EXPECT_FALSE(redirects.tryInsert(Endpoint{endpoint("65.0.0.3:10003"), 0})); + EXPECT_FALSE(redirects.tryInsert(Endpoint{remote.atPort(12000), 1})); + EXPECT_TRUE(redirects.tryInsert(Endpoint{endpoint("65.0.0.3:10003"), 1})); + EXPECT_FALSE(redirects.tryInsert(Endpoint{endpoint("65.0.0.3:12000"), 1})); + EXPECT_EQ(redirects.list().size(), 1u); + + SlotHandouts slotHandouts(slot); + EXPECT_EQ(slotHandouts.slot(), slot); + EXPECT_FALSE(slotHandouts.full()); + EXPECT_FALSE( + slotHandouts.tryInsert(Endpoint{endpoint("65.0.0.4:10004"), Tuning::kMaxHops + 1})); + EXPECT_FALSE(slotHandouts.tryInsert(Endpoint{remote.atPort(12001), 1})); + + auto const recent = endpoint("65.0.0.5:10005"); + slot->recent.insert(recent, 2); + EXPECT_FALSE(slotHandouts.tryInsert(Endpoint{recent, 2})); + EXPECT_TRUE(slotHandouts.tryInsert(Endpoint{endpoint("65.0.0.6:10006"), 2})); + EXPECT_FALSE(slotHandouts.tryInsert(Endpoint{endpoint("65.0.0.6:12000"), 2})); + slotHandouts.insert(Endpoint{endpoint("65.0.0.7:10007"), 1}); + EXPECT_EQ(slotHandouts.list().size(), 2u); + + ConnectHandouts::Squelches squelches(clock); + ConnectHandouts connects(2, squelches); + EXPECT_TRUE(connects.empty()); + EXPECT_TRUE(connects.tryInsert(endpoint("65.0.0.8:10008"))); + EXPECT_FALSE(connects.empty()); + EXPECT_FALSE(connects.tryInsert(endpoint("65.0.0.8:12000"))); + EXPECT_TRUE(connects.tryInsert(Endpoint{endpoint("65.0.0.9:10009"), 1})); + EXPECT_TRUE(connects.full()); + EXPECT_FALSE(connects.tryInsert(endpoint("65.0.0.10:10010"))); + EXPECT_EQ(connects.list().size(), 2u); + + ConnectHandouts squelched(1, squelches); + EXPECT_FALSE(squelched.tryInsert(endpoint("65.0.0.9:12000"))); +} + +TEST(PeerFinderHandouts, distributes_livecache_entries) +{ + TestStopwatch clock; + Livecache<> cache(clock, journal()); + cache.insert(Endpoint{endpoint("65.0.0.10:10010"), 1}); + cache.insert(Endpoint{endpoint("65.0.0.11:10011"), 2}); + + auto const slot1 = std::make_shared( + endpoint("65.0.0.1:10001"), endpoint("65.0.0.2:10002"), false, clock); + auto const slot2 = std::make_shared( + endpoint("65.0.0.3:10003"), endpoint("65.0.0.4:10004"), false, clock); + std::vector targets; + targets.emplace_back(slot1); + targets.emplace_back(slot2); + + handout(targets.begin(), targets.end(), cache.hops.begin(), cache.hops.end()); + + EXPECT_FALSE(targets.front().list().empty()); + EXPECT_FALSE(targets.back().list().empty()); + + for (std::uint32_t i = 0; i < Tuning::kNumberOfEndpoints; ++i) + targets.front().insert(Endpoint{endpoint("65.1.0." + std::to_string(i + 1) + ":12000"), 1}); + + handout(targets.begin(), targets.begin() + 1, cache.hops.begin(), cache.hops.end()); + EXPECT_TRUE(targets.front().full()); +} + +TEST_F(PeerFinderTest, preprocess_filters_invalid_duplicate_and_extra_self_endpoints) +{ + auto const local = endpoint("65.0.0.1:10001"); + auto const remote = endpoint("65.0.0.2:10002"); + auto const slot = std::make_shared(local, remote, false, clock_); + Endpoints endpoints{ + Endpoint{endpoint("65.0.0.3:10003"), Tuning::kMaxHops + 1}, + Endpoint{endpoint("0.0.0.0:2459"), 0}, + Endpoint{endpoint("0.0.0.0:2460"), 0}, + Endpoint{endpoint("10.0.0.1:10004"), 1}, + Endpoint{endpoint("65.0.0.5"), 1}, + Endpoint{endpoint("65.0.0.6:10006"), 1}, + Endpoint{endpoint("65.0.0.6:10006"), 2}}; + + logic_.preprocess(slot, endpoints); + + ASSERT_EQ(endpoints.size(), 2u); + EXPECT_EQ(endpoints.front().address, remote.atPort(2459)); + EXPECT_EQ(endpoints.front().hops, 1u); + EXPECT_EQ(endpoints.back().address, endpoint("65.0.0.6:10006")); + EXPECT_EQ(endpoints.back().hops, 2u); +} + +TEST_F(PeerFinderTest, on_endpoints_checks_neighbor_before_caching_it) +{ + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.inPeers = 1; + logic_.config(config); + + auto const local = endpoint("65.0.0.1:10001"); + auto const remote = endpoint("55.104.0.2:1025"); + auto const [slot, result] = logic_.newInboundSlot(local, remote); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + Endpoints const advertised{Endpoint{endpoint("0.0.0.0:2459"), 0}}; + logic_.onEndpoints(slot, advertised); + + ASSERT_EQ(checker_.asyncConnects.size(), 1u); + EXPECT_EQ(checker_.asyncConnects.front(), remote.atPort(2459)); + EXPECT_EQ(slot->listeningPort(), std::optional{2459}); + EXPECT_TRUE(slot->checked); + EXPECT_TRUE(slot->canAccept); + EXPECT_TRUE(logic_.livecache.empty()); + + clock_.advance(Tuning::kSecondsPerMessage); + logic_.onEndpoints(slot, advertised); + EXPECT_EQ(logic_.livecache.size(), 1u); + EXPECT_EQ(logic_.bootcache.size(), 1u); + + logic_.onEndpoints(slot, Endpoints{Endpoint{endpoint("65.0.0.9:10009"), 1}}); + EXPECT_EQ(logic_.livecache.size(), 1u); + + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, on_endpoints_skips_failed_neighbor_connectivity_checks) +{ + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.inPeers = 1; + logic_.config(config); + + checker_.nextError = boost::asio::error::host_unreachable; + auto const local = endpoint("65.0.0.1:10001"); + auto const remote = endpoint("55.104.0.3:1025"); + auto const [slot, result] = logic_.newInboundSlot(local, remote); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + Endpoints const advertised{Endpoint{endpoint("0.0.0.0:2459"), 0}}; + logic_.onEndpoints(slot, advertised); + EXPECT_TRUE(slot->checked); + EXPECT_FALSE(slot->canAccept); + + clock_.advance(Tuning::kSecondsPerMessage); + logic_.onEndpoints(slot, advertised); + EXPECT_TRUE(logic_.livecache.empty()); + + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, on_endpoints_waits_for_pending_connectivity_check) +{ + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.inPeers = 1; + logic_.config(config); + + checker_.completeAsync = false; + auto const local = endpoint("65.0.0.1:10001"); + auto const remote = endpoint("55.104.0.4:1025"); + auto const [slot, result] = logic_.newInboundSlot(local, remote); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + Endpoints const advertised{Endpoint{endpoint("0.0.0.0:2459"), 0}}; + logic_.onEndpoints(slot, advertised); + EXPECT_TRUE(slot->connectivityCheckInProgress); + + clock_.advance(Tuning::kSecondsPerMessage); + logic_.onEndpoints(slot, advertised); + EXPECT_EQ(checker_.asyncConnects.size(), 1u); + EXPECT_TRUE(logic_.livecache.empty()); + + checker_.completeAsync = true; + logic_.checkComplete(remote, remote.atPort(2459), boost::asio::error::operation_aborted); + slot->connectivityCheckInProgress = false; + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, builds_endpoint_messages_and_redirects_from_livecache) +{ + Config config; + config.autoConnect = false; + config.wantIncoming = true; + config.listeningPort = 2459; + config.inPeers = 2; + config.outPeers = 2; + config.ipLimit = 2; + logic_.config(config); + + auto const remote = endpoint("55.104.0.5:1025"); + auto const live = endpoint("65.0.0.10:10010"); + logic_.livecache.insert(Endpoint{live, 1}); + + auto const [slot, result] = logic_.newOutboundSlot(remote); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + ASSERT_TRUE(logic_.onConnected(slot, endpoint("65.0.0.1:10001"))); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + auto const messages = logic_.buildEndpointsForPeers(); + ASSERT_EQ(messages.size(), 1u); + auto const& sent = messages.front().second; + EXPECT_TRUE(std::ranges::any_of(sent, [](Endpoint const& ep) { return ep.hops == 0; })); + EXPECT_TRUE( + std::ranges::any_of(sent, [&live](Endpoint const& ep) { return ep.address == live; })); + EXPECT_TRUE(logic_.buildEndpointsForPeers().empty()); + + auto const redirects = logic_.redirect(slot); + EXPECT_FALSE(redirects.empty()); + + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, autoconnect_uses_livecache_then_bootcache) +{ + Config config; + config.autoConnect = true; + config.wantIncoming = false; + config.outPeers = 1; + config.inPeers = 0; + config.ipLimit = 1; + logic_.config(config); + + auto const live = endpoint("65.0.0.11:10011"); + logic_.livecache.insert(Endpoint{live, 1}); + auto const liveAddresses = logic_.autoconnect(); + ASSERT_EQ(liveAddresses.size(), 1u); + EXPECT_EQ(liveAddresses.front(), live); + + auto const boot = endpoint("65.0.0.12:10012"); + EXPECT_TRUE(logic_.bootcache.insertStatic(boot)); + auto const bootAddresses = logic_.autoconnect(); + ASSERT_EQ(bootAddresses.size(), 1u); + EXPECT_EQ(bootAddresses.front(), boot); +} + +TEST_F(PeerFinderTest, sources_redirects_status_and_validation_paths_are_exercised) +{ + auto const source = std::make_shared("static"); + source->resultsToFetch.addresses = {endpoint("65.0.0.13:10013")}; + logic_.addStaticSource(source); + EXPECT_EQ(source->fetchCount, 1); + EXPECT_EQ(logic_.bootcache.size(), 1u); + + auto const failing = std::make_shared("failing"); + failing->resultsToFetch.error = boost::asio::error::host_unreachable; + logic_.fetch(failing); + EXPECT_EQ(failing->fetchCount, 1); + + auto const dynamic = std::make_shared("dynamic"); + logic_.addSource(dynamic); + ASSERT_EQ(logic_.sources.size(), 1u); + EXPECT_EQ(logic_.sources.front(), dynamic); + + std::vector redirects{ + {boost::asio::ip::make_address("65.0.0.14"), 10014}, + {boost::asio::ip::make_address("65.0.0.15"), 10015}}; + logic_.onRedirects(redirects.begin(), redirects.end(), redirects.front()); + EXPECT_EQ(logic_.bootcache.size(), 3u); + + EXPECT_FALSE(logic_.isValidAddress(endpoint("0.0.0.0:10016"))); + EXPECT_FALSE(logic_.isValidAddress(endpoint("10.0.0.1:10017"))); + EXPECT_FALSE(logic_.isValidAddress(endpoint("65.0.0.16"))); + EXPECT_TRUE(logic_.isValidAddress(endpoint("65.0.0.16:10016"))); + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + logic_.onWrite(map); + } + EXPECT_TRUE(stream.top().isMember("peers")); + EXPECT_TRUE(stream.top().isMember("counts")); + EXPECT_TRUE(stream.top().isMember("config")); + EXPECT_TRUE(stream.top().isMember("livecache")); + EXPECT_TRUE(stream.top().isMember("bootcache")); + + DefaultCancelSource defaultCancel; + Source::Results results; + EXPECT_TRUE(results.addresses.empty()); + defaultCancel.cancel(); + defaultCancel.fetch(results, journal()); + + logic_.fetchSource = dynamic; + logic_.stop(); + EXPECT_TRUE(logic_.stopping); + EXPECT_EQ(dynamic->cancelCount, 1); + + auto const ignored = std::make_shared("ignored"); + logic_.fetch(ignored); + EXPECT_EQ(ignored->fetchCount, 0); + + logic_.checkComplete( + endpoint("65.0.0.18:10018"), endpoint("65.0.0.19:10019"), boost::system::error_code{}); +} + +TEST(PeerFinderBootcache, loads_unique_entries_and_clears_cache) +{ + CapturingStore store; + TestStopwatch clock; + auto const ep1 = endpoint("65.0.0.1:10001"); + auto const ep2 = endpoint("65.0.0.2:10002"); + store.entriesToLoad = {storeEntry(ep1, 3), storeEntry(ep2, -2), storeEntry(ep1, 4)}; + + Bootcache cache(store, clock, journal()); + cache.load(); + + EXPECT_FALSE(cache.empty()); + EXPECT_EQ(cache.size(), 2u); + EXPECT_EQ(*cache.begin(), ep1); + EXPECT_EQ(*cache.cbegin(), ep1); + EXPECT_NE(cache.begin(), cache.end()); + EXPECT_NE(cache.cbegin(), cache.cend()); + + cache.clear(); + EXPECT_TRUE(cache.empty()); + EXPECT_EQ(cache.begin(), cache.end()); +} + +TEST(PeerFinderBootcache, records_connection_outcomes_and_persists_pending_updates) +{ + CapturingStore store; + TestStopwatch clock; + auto const ep1 = endpoint("65.0.0.1:10001"); + auto const ep2 = endpoint("65.0.0.2:10002"); + auto const ep3 = endpoint("65.0.0.3:10003"); + auto const ep4 = endpoint("65.0.0.4:10004"); + + { + Bootcache cache(store, clock, journal()); + + EXPECT_TRUE(cache.insert(ep1)); + EXPECT_FALSE(cache.insert(ep1)); + + cache.onSuccess(ep1); + EXPECT_TRUE(cache.insertStatic(ep1)); + EXPECT_FALSE(cache.insertStatic(ep1)); + + EXPECT_TRUE(cache.insertStatic(ep2)); + cache.onSuccess(ep3); + cache.onFailure(ep3); + cache.onFailure(ep4); + + EXPECT_EQ(cache.size(), 4u); + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + cache.onWrite(map); + } + EXPECT_TRUE(stream.top().isMember("entries")); + EXPECT_EQ(stream.top()["entries"].size(), 4u); + } + + ASSERT_EQ(store.saves.size(), 1u); + auto const& saved = store.saves.front(); + ASSERT_EQ(saved.size(), 4u); + EXPECT_EQ(savedValence(saved, ep1), Bootcache::kStaticValence); + EXPECT_EQ(savedValence(saved, ep2), Bootcache::kStaticValence); + EXPECT_EQ(savedValence(saved, ep3), -1); + EXPECT_EQ(savedValence(saved, ep4), -1); +} + +TEST(PeerFinderBootcache, periodic_activity_saves_after_cooldown) +{ + using namespace std::chrono_literals; + + CapturingStore store; + TestStopwatch clock; + + { + Bootcache cache(store, clock, journal()); + EXPECT_TRUE(cache.insert(endpoint("65.0.0.1:10001"))); + + cache.periodicActivity(); + EXPECT_TRUE(store.saves.empty()); + + clock.advance(Tuning::kBootcacheCooldownTime + 1s); + cache.periodicActivity(); + ASSERT_EQ(store.saves.size(), 1u); + + cache.periodicActivity(); + EXPECT_EQ(store.saves.size(), 1u); + } + + EXPECT_EQ(store.saves.size(), 1u); +} + +TEST(PeerFinderBootcache, prunes_when_cache_exceeds_limit) +{ + CapturingStore store; + TestStopwatch clock; + Bootcache cache(store, clock, journal()); + + for (std::uint16_t i = 0; i <= Tuning::kBootcacheSize; ++i) + { + EXPECT_TRUE(cache.insert(endpoint( + "65.0." + std::to_string((i / 256) % 256) + "." + std::to_string(i % 256) + ":" + + std::to_string(10000 + i)))); + } + + EXPECT_LE(cache.size(), Tuning::kBootcacheSize); +} + +TEST(PeerFinderEndpoint, clamps_hops_to_overflow_bucket) +{ + auto const address = endpoint("65.0.0.1:10001"); + Endpoint const ep(address, Tuning::kMaxHops + 10); + + EXPECT_EQ(ep.address, address); + EXPECT_EQ(ep.hops, Tuning::kMaxHops + 1); +} + +TEST(PeerFinderSlotImp, tracks_state_and_recent_endpoints) +{ + using State = Slot::State; + using namespace std::chrono_literals; + + TestStopwatch clock; + auto const local = endpoint("65.0.0.1:10000"); + auto const remote = endpoint("65.0.0.2:10001"); + SlotImp inbound(local, remote, true, clock); + + EXPECT_TRUE(inbound.inbound()); + EXPECT_TRUE(inbound.fixed()); + EXPECT_FALSE(inbound.reserved()); + EXPECT_EQ(inbound.state(), State::Accept); + EXPECT_EQ(inbound.remoteEndpoint(), remote); + EXPECT_EQ(inbound.localEndpoint(), std::optional{local}); + EXPECT_FALSE(inbound.publicKey()); + EXPECT_FALSE(inbound.listeningPort()); + EXPECT_FALSE(inbound.checked); + EXPECT_FALSE(inbound.canAccept); + EXPECT_FALSE(inbound.connectivityCheckInProgress); + + auto const newLocal = endpoint("65.0.0.3:10002"); + auto const newRemote = endpoint("65.0.0.4:10003"); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + + inbound.localEndpoint(newLocal); + inbound.remoteEndpoint(newRemote); + inbound.publicKey(publicKey); + inbound.reserved(true); + inbound.setListeningPort(2459); + + EXPECT_EQ(inbound.localEndpoint(), std::optional{newLocal}); + EXPECT_EQ(inbound.remoteEndpoint(), newRemote); + EXPECT_EQ(inbound.publicKey(), std::optional{publicKey}); + EXPECT_TRUE(inbound.reserved()); + EXPECT_EQ(inbound.listeningPort(), std::optional{2459}); + EXPECT_FALSE(inbound.prefix().empty()); + + inbound.state(State::Closing); + EXPECT_EQ(inbound.state(), State::Closing); + + SlotImp outbound(remote, false, clock); + EXPECT_FALSE(outbound.inbound()); + EXPECT_FALSE(outbound.fixed()); + EXPECT_EQ(outbound.state(), State::Connect); + EXPECT_TRUE(outbound.checked); + EXPECT_TRUE(outbound.canAccept); + + outbound.state(State::Connected); + outbound.activate(clock.now()); + EXPECT_EQ(outbound.state(), State::Active); + EXPECT_EQ(outbound.whenAcceptEndpoints, clock.now()); + + auto const recent = endpoint("65.0.0.5:10004"); + EXPECT_FALSE(outbound.recent.filter(recent, 2)); + + outbound.recent.insert(recent, 2); + EXPECT_TRUE(outbound.recent.filter(recent, 2)); + EXPECT_TRUE(outbound.recent.filter(recent, 3)); + EXPECT_FALSE(outbound.recent.filter(recent, 1)); + + outbound.recent.insert(recent, 4); + EXPECT_FALSE(outbound.recent.filter(recent, 1)); + + outbound.recent.insert(recent, 1); + EXPECT_TRUE(outbound.recent.filter(recent, 1)); + EXPECT_FALSE(outbound.recent.filter(recent, 0)); + + clock.advance(Tuning::kLiveCacheSecondsToLive + 1s); + outbound.expire(); + EXPECT_FALSE(outbound.recent.filter(recent, 1)); +} + +TEST(PeerFinderConfig, writes_property_stream_and_compares_verify_endpoints) +{ + Config config; + config.maxPeers = 42; + config.outPeers = 12; + config.inPeers = 30; + config.peerPrivate = false; + config.wantIncoming = true; + config.autoConnect = false; + config.listeningPort = 2459; + config.features = "feature"; + config.ipLimit = 4; + config.verifyEndpoints = false; + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + config.onWrite(map); + } + + auto const& json = stream.top(); + EXPECT_EQ(json["max_peers"].asUInt(), config.maxPeers); + EXPECT_EQ(json["out_peers"].asUInt(), config.outPeers); + EXPECT_TRUE(json.isMember("want_incoming")); + EXPECT_TRUE(json.isMember("auto_connect")); + EXPECT_EQ(json["port"].asUInt(), config.listeningPort); + EXPECT_EQ(json["features"].asString(), config.features); + EXPECT_EQ(json["ip_limit"].asInt(), config.ipLimit); + EXPECT_TRUE(json.isMember("verify_endpoints")); + + Config same = config; + EXPECT_EQ(config, same); + same.verifyEndpoints = true; + EXPECT_NE(config, same); +} + +TEST(PeerFinderConfig, validator_and_standalone_settings_disable_auto_connect) +{ + PeerLimitConfig const limits{.maxPeers = 50, .inPeers = {}, .outPeers = {}}; + + Config const config = Config::makeConfig(false, true, limits, 2459, true, 7, false); + + EXPECT_TRUE(config.peerPrivate); + EXPECT_FALSE(config.autoConnect); + EXPECT_FALSE(config.verifyEndpoints); + EXPECT_EQ(config.ipLimit, 7); +} + +TEST(PeerFinderConfig, calculates_outbound_peers_and_clamps_ip_limits) +{ + Config config; + config.maxPeers = 1; + EXPECT_EQ(config.calcOutPeers(), Tuning::kMinOutCount); + + config.maxPeers = 100; + EXPECT_EQ(config.calcOutPeers(), 15u); + + config.inPeers = 1; + config.ipLimit = 0; + config.applyTuning(); + EXPECT_EQ(config.ipLimit, 1); + + Config explicitLimit; + explicitLimit.inPeers = 8; + explicitLimit.ipLimit = 99; + explicitLimit.applyTuning(); + EXPECT_EQ(explicitLimit.ipLimit, 4); + + Config largeInbound; + largeInbound.inPeers = 200; + largeInbound.ipLimit = 0; + largeInbound.applyTuning(); + EXPECT_EQ(largeInbound.ipLimit, 7); +} + +TEST(PeerFinderConfig, applies_legacy_and_explicit_peer_limits) +{ + struct ConfigCase + { + std::string name; + std::optional maxPeers; + std::optional maxIn; + std::optional maxOut; + std::uint16_t port; + std::uint16_t expectedOut; + std::uint16_t expectedIn; + std::uint16_t expectedIpLimit; + }; + + std::vector const cases{ + {.name = "legacy no config", + .maxPeers = {}, + .maxIn = {}, + .maxOut = {}, + .port = 4000, + .expectedOut = 10, + .expectedIn = 11, + .expectedIpLimit = 2}, + {.name = "legacy max_peers 0", + .maxPeers = 0, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 11, + .expectedIpLimit = 2}, + {.name = "legacy max_peers 5", + .maxPeers = 5, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 0, + .expectedIpLimit = 1}, + {.name = "legacy max_peers 20", + .maxPeers = 20, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 10, + .expectedIpLimit = 2}, + {.name = "legacy max_peers 100", + .maxPeers = 100, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 15, + .expectedIn = 85, + .expectedIpLimit = 6}, + {.name = "legacy max_peers 20, private", + .maxPeers = 20, + .maxIn = 100, + .maxOut = 10, + .port = 0, + .expectedOut = 20, + .expectedIn = 0, + .expectedIpLimit = 1}, + {.name = "new in 100/out 10", + .maxPeers = {}, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 100, + .expectedIpLimit = 6}, + {.name = "new in 0/out 10", + .maxPeers = {}, + .maxIn = 0, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 0, + .expectedIpLimit = 1}, + {.name = "new in 100/out 10, private", + .maxPeers = {}, + .maxIn = 100, + .maxOut = 10, + .port = 0, + .expectedOut = 10, + .expectedIn = 0, + .expectedIpLimit = 6}}; + + for (auto const& testCase : cases) + { + SCOPED_TRACE(testCase.name); + + PeerLimitConfig const limits{ + .maxPeers = testCase.maxPeers, .inPeers = testCase.maxIn, .outPeers = testCase.maxOut}; + + Config const config = + Config::makeConfig(false, false, limits, testCase.port, false, 0, true); + + Counts counts; + counts.onConfig(config); + EXPECT_EQ(counts.outMax(), testCase.expectedOut); + EXPECT_EQ(counts.inMax(), testCase.expectedIn); + EXPECT_EQ(config.ipLimit, testCase.expectedIpLimit); + + NiceMock store; + allowEmptyStore(store); + NiceMock checker; + TestStopwatch clock; + Logic> logic(clock, store, checker, journal()); + logic.config(config); + + EXPECT_EQ(logic.config(), config); + } +} + +TEST(PeerFinderConfig, rejects_incomplete_or_out_of_range_peer_limits) +{ + std::vector const configs{ + {.maxPeers = {}, .inPeers = 100, .outPeers = {}}, + {.maxPeers = {}, .inPeers = {}, .outPeers = 100}, + {.maxPeers = {}, .inPeers = 100, .outPeers = 5}, + {.maxPeers = {}, .inPeers = 1001, .outPeers = 10}, + {.maxPeers = {}, .inPeers = 10, .outPeers = 1001}}; + + for (auto const& limits : configs) + { + EXPECT_THROW( + Config::makeConfig(false, false, limits, 4000, false, 0, true), std::exception); + } +} + +} // namespace +} // namespace xrpl::PeerFinder diff --git a/src/xrpld/app/rdb/PeerFinder.h b/src/xrpld/app/rdb/PeerFinder.h index 4f186ff7e2..5d916000a3 100644 --- a/src/xrpld/app/rdb/PeerFinder.h +++ b/src/xrpld/app/rdb/PeerFinder.h @@ -1,9 +1,8 @@ #pragma once -#include - #include #include +#include #include #include diff --git a/src/xrpld/app/rdb/detail/PeerFinder.cpp b/src/xrpld/app/rdb/detail/PeerFinder.cpp index abdcd1c61a..72a275c7cd 100644 --- a/src/xrpld/app/rdb/detail/PeerFinder.cpp +++ b/src/xrpld/app/rdb/detail/PeerFinder.cpp @@ -1,12 +1,11 @@ #include -#include - #include #include #include #include #include +#include #include #include // IWYU pragma: keep diff --git a/src/xrpld/overlay/detail/ConnectAttempt.cpp b/src/xrpld/overlay/detail/ConnectAttempt.cpp index 064b4ecd3e..0f0b3242de 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.cpp +++ b/src/xrpld/overlay/detail/ConnectAttempt.cpp @@ -7,8 +7,6 @@ #include #include #include -#include -#include #include #include @@ -16,6 +14,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/xrpld/overlay/detail/ConnectAttempt.h b/src/xrpld/overlay/detail/ConnectAttempt.h index d7836e3c84..f9ba33571f 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.h +++ b/src/xrpld/overlay/detail/ConnectAttempt.h @@ -3,12 +3,12 @@ #include #include #include -#include #include #include #include #include +#include #include #include diff --git a/src/xrpld/overlay/detail/OverlayImpl.cpp b/src/xrpld/overlay/detail/OverlayImpl.cpp index 6a6a6edace..f9972548d1 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.cpp +++ b/src/xrpld/overlay/detail/OverlayImpl.cpp @@ -10,8 +10,6 @@ #include #include #include -#include -#include #include #include #include @@ -39,6 +37,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -179,12 +180,13 @@ OverlayImpl::OverlayImpl( , journal_(app_.getJournal("Overlay")) , serverHandler_(serverHandler) , resourceManager_(resourceManager) + , store_(app_.getJournal("PeerFinder")) , peerFinder_( PeerFinder::makeManager( ioContext, stopwatch(), app_.getJournal("PeerFinder"), - config, + store_, collector)) , resolver_(resolver) , nextId_(1) @@ -201,6 +203,7 @@ OverlayImpl::OverlayImpl( return ret; }()) { + store_.open(config); beast::PropertyStream::Source::add(peerFinder_.get()); } @@ -505,7 +508,7 @@ OverlayImpl::remove(std::shared_ptr const& slot) void OverlayImpl::start() { - PeerFinder::Config const config = PeerFinder::Config::makeConfig( + PeerFinder::Config const config = PeerFinder::makeConfig( app_.config(), serverHandler_.setup().overlay.port(), app_.getValidationPublicKey().has_value(), diff --git a/src/xrpld/overlay/detail/OverlayImpl.h b/src/xrpld/overlay/detail/OverlayImpl.h index 092ac86a6d..cd2c7d630b 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.h +++ b/src/xrpld/overlay/detail/OverlayImpl.h @@ -8,8 +8,7 @@ #include #include #include -#include -#include +#include #include #include @@ -24,6 +23,8 @@ #include #include #include +#include +#include #include #include #include @@ -109,6 +110,7 @@ private: beast::Journal const journal_; ServerHandler& serverHandler_; Resource::Manager& resourceManager_; + PeerFinder::StoreSqdb store_; std::unique_ptr peerFinder_; TrafficCount traffic_; hash_map, std::weak_ptr> peers_; diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 8838970b5f..c720bdf30b 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -19,8 +19,6 @@ #include #include #include -#include -#include #include #include @@ -43,6 +41,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index ea6eccd656..90f8a917f4 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -9,8 +9,6 @@ #include #include #include -#include -#include #include #include @@ -24,6 +22,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/xrpld/peerfinder/PeerfinderManager.h b/src/xrpld/peerfinder/PeerfinderManager.h index 0530343641..f96ea31943 100644 --- a/src/xrpld/peerfinder/PeerfinderManager.h +++ b/src/xrpld/peerfinder/PeerfinderManager.h @@ -1,368 +1,19 @@ #pragma once #include -#include -#include -#include -#include -#include -#include +#include -#include - -#include -#include #include -#include -#include -#include -#include -#include namespace xrpl::PeerFinder { -using clock_type = beast::AbstractClock; - -/** - * Represents a set of addresses. - */ -using IPAddresses = std::vector; - -//------------------------------------------------------------------------------ - -/** - * PeerFinder configuration settings. - */ -struct Config -{ - /** - * The largest number of public peer slots to allow. - * This includes both inbound and outbound, but does not include - * fixed peers. - */ - std::size_t maxPeers{Tuning::kDefaultMaxPeers}; - - /** - * The number of automatic outbound connections to maintain. - * Outbound connections are only maintained if autoConnect - * is `true`. - */ - std::size_t outPeers; - - /** - * The number of automatic inbound connections to maintain. - * Inbound connections are only maintained if wantIncoming - * is `true`. - */ - std::size_t inPeers{0}; - - /** - * `true` if we want our IP address kept private. - */ - bool peerPrivate = true; - - /** - * `true` if we want to accept incoming connections. - */ - bool wantIncoming{true}; - - /** - * `true` if we want to establish connections automatically - */ - bool autoConnect{true}; - - /** - * The listening port number. - */ - std::uint16_t listeningPort{0}; - - /** - * The set of features we advertise. - */ - std::string features; - - /** - * Limit how many incoming connections we allow per IP - */ - int ipLimit{0}; - - /** - * `true` if we want to verify endpoints in TMEndpoints messages - */ - bool verifyEndpoints = true; - - //-------------------------------------------------------------------------- - - /** - * Create a configuration with default values. - */ - Config(); - - /** - * Returns a suitable value for outPeers according to the rules. - */ - [[nodiscard]] std::size_t - calcOutPeers() const; - - /** - * Adjusts the values so they follow the business rules. - */ - void - applyTuning(); - - /** - * Write the configuration into a property stream - */ - void - onWrite(beast::PropertyStream::Map& map) const; - - /** - * Make PeerFinder::Config from configuration parameters - * @param config server's configuration - * @param port server's listening port - * @param validationPublicKey true if validation public key is not empty - * @param ipLimit limit of incoming connections per IP - * @param verifyEndpoints `true` if we want to verify endpoints in - * TMEndpoints messages - * @return PeerFinder::Config - */ - static Config - makeConfig( - xrpl::Config const& config, - std::uint16_t port, - bool validationPublicKey, - int ipLimit, - bool verifyEndpoints); - - friend bool - operator==(Config const& lhs, Config const& rhs) = default; -}; - -//------------------------------------------------------------------------------ - -/** - * Describes a connectable peer address along with some metadata. - */ -struct Endpoint -{ - Endpoint() = default; - - Endpoint(beast::IP::Endpoint ep, std::uint32_t hops); - - std::uint32_t hops = 0; - beast::IP::Endpoint address; -}; - -inline bool -operator<(Endpoint const& lhs, Endpoint const& rhs) -{ - return lhs.address < rhs.address; -} - -/** - * A set of Endpoint used for connecting. - */ -using Endpoints = std::vector; - -//------------------------------------------------------------------------------ - -/** - * Possible results from activating a slot. - */ -enum class Result { InboundDisabled, DuplicatePeer, IpLimitExceeded, Full, Success }; - -/** - * @brief Converts a `Result` enum value to its string representation. - * - * This function provides a human-readable string for a given `Result` enum, - * which is useful for logging, debugging, or displaying status messages. - * - * @param result The `Result` enum value to convert. - * @return A `std::string_view` representing the enum value. Returns "unknown" - * if the enum value is not explicitly handled. - * - * @note This function returns a `std::string_view` for performance. - * A `std::string` would need to allocate memory on the heap and copy the - * string literal into it every time the function is called. - */ -inline std::string_view -to_string(Result result) noexcept -{ - switch (result) - { - case Result::InboundDisabled: - return "inbound disabled"; - case Result::DuplicatePeer: - return "peer already connected"; - case Result::IpLimitExceeded: - return "ip limit exceeded"; - case Result::Full: - return "slots full"; - case Result::Success: - return "success"; - } - - return "unknown"; -} - -/** - * Maintains a set of IP addresses used for getting into the network. - */ -class Manager : public beast::PropertyStream::Source -{ -protected: - Manager() noexcept; - -public: - /** - * Destroy the object. - * Any pending source fetch operations are aborted. - * There may be some listener calls made before the - * destructor returns. - */ - ~Manager() override = default; - - /** - * Set the configuration for the manager. - * The new settings will be applied asynchronously. - * Thread safety: - * Can be called from any threads at any time. - */ - virtual void - setConfig(Config const& config) = 0; - - /** - * Transition to the started state, synchronously. - */ - virtual void - start() = 0; - - /** - * Transition to the stopped state, synchronously. - */ - virtual void - stop() = 0; - - /** - * Returns the configuration for the manager. - */ - virtual Config - config() = 0; - - /** - * Add a peer that should always be connected. - * This is useful for maintaining a private cluster of peers. - * The string is the name as specified in the configuration - * file, along with the set of corresponding IP addresses. - */ - virtual void - addFixedPeer(std::string_view name, std::vector const& addresses) = 0; - - /** - * Add a set of strings as fallback IP::Endpoint sources. - * @param name A label used for diagnostics. - */ - virtual void - addFallbackStrings(std::string const& name, std::vector const& strings) = 0; - - /** - * Add a URL as a fallback location to obtain IP::Endpoint sources. - * @param name A label used for diagnostics. - */ - /* VFALCO NOTE Unimplemented - virtual void addFallbackURL (std::string const& name, - std::string const& url) = 0; - */ - - //-------------------------------------------------------------------------- - - /** - * Create a new inbound slot with the specified remote endpoint. - * If nullptr is returned, then the slot could not be assigned. - * Usually this is because of a detected self-connection. - */ - virtual std::pair, Result> - newInboundSlot( - beast::IP::Endpoint const& localEndpoint, - beast::IP::Endpoint const& remoteEndpoint) = 0; - - /** - * Create a new outbound slot with the specified remote endpoint. - * If nullptr is returned, then the slot could not be assigned. - * Usually this is because of a duplicate connection. - */ - virtual std::pair, Result> - newOutboundSlot(beast::IP::Endpoint const& remoteEndpoint) = 0; - - /** - * Called when mtENDPOINTS is received. - */ - virtual void - onEndpoints(std::shared_ptr const& slot, Endpoints const& endpoints) = 0; - - /** - * Called when the slot is closed. - * This always happens when the socket is closed, unless the socket - * was canceled. - */ - virtual void - onClosed(std::shared_ptr const& slot) = 0; - - /** - * Called when an outbound connection is deemed to have failed - */ - virtual void - onFailure(std::shared_ptr const& slot) = 0; - - /** - * Called when we received redirect IPs from a busy peer. - */ - virtual void - onRedirects( - boost::asio::ip::tcp::endpoint const& remoteAddress, - std::vector const& eps) = 0; - - //-------------------------------------------------------------------------- - - /** - * Called when an outbound connection attempt succeeds. - * The local endpoint must be valid. If the caller receives an error - * when retrieving the local endpoint from the socket, it should - * proceed as if the connection attempt failed by calling on_closed - * instead of on_connected. - * @return `true` if the connection should be kept - */ - virtual bool - onConnected(std::shared_ptr const& slot, beast::IP::Endpoint const& localEndpoint) = 0; - - /** - * Request an active slot type. - */ - virtual Result - activate(std::shared_ptr const& slot, PublicKey const& key, bool reserved) = 0; - - /** - * Returns a set of endpoints suitable for redirection. - */ - virtual std::vector - redirect(std::shared_ptr const& slot) = 0; - - /** - * Return a set of addresses we should connect to. - */ - virtual std::vector - autoconnect() = 0; - - virtual std::vector, std::vector>> - buildEndpointsForPeers() = 0; - - /** - * Perform periodic activity. - * This should be called once per second. - */ - virtual void - oncePerSecond() = 0; -}; +Config +makeConfig( + xrpl::Config const& config, + std::uint16_t port, + bool validationPublicKey, + int ipLimit, + bool verifyEndpoints); } // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/PeerfinderConfig.cpp b/src/xrpld/peerfinder/detail/PeerfinderConfig.cpp index 5d276dc9c5..a893b969e0 100644 --- a/src/xrpld/peerfinder/detail/PeerfinderConfig.cpp +++ b/src/xrpld/peerfinder/detail/PeerfinderConfig.cpp @@ -1,124 +1,39 @@ #include #include -#include -#include +#include -#include -#include #include namespace xrpl::PeerFinder { -Config::Config() : outPeers(calcOutPeers()) - -{ -} - -std::size_t -Config::calcOutPeers() const -{ - return std::max( - ((maxPeers * Tuning::kOutPercent) + 50) / 100, std::size_t(Tuning::kMinOutCount)); -} - -void -Config::applyTuning() -{ - if (ipLimit == 0) - { - // Unless a limit is explicitly set, we allow between - // 2 and 5 connections from non RFC-1918 "private" - // IP addresses. - ipLimit = 2; - - if (inPeers > Tuning::kDefaultMaxPeers) - ipLimit += std::min(5, static_cast(inPeers / Tuning::kDefaultMaxPeers)); - } - - // We don't allow a single IP to consume all incoming slots, - // unless we only have one incoming slot available. - ipLimit = std::max(1, std::min(ipLimit, static_cast(inPeers / 2))); -} - -void -Config::onWrite(beast::PropertyStream::Map& map) const -{ - map["max_peers"] = maxPeers; - map["out_peers"] = outPeers; - map["want_incoming"] = wantIncoming; - map["auto_connect"] = autoConnect; - map["port"] = listeningPort; - map["features"] = features; - map["ip_limit"] = ipLimit; - map["verify_endpoints"] = verifyEndpoints; -} - Config -Config::makeConfig( +makeConfig( xrpl::Config const& cfg, std::uint16_t port, bool validationPublicKey, int ipLimit, bool verifyEndpoints) { - PeerFinder::Config config; - - config.peerPrivate = cfg.peerPrivate; - - // Servers with peer privacy don't want to allow incoming connections - config.wantIncoming = (!config.peerPrivate) && (port != 0); - + PeerLimitConfig limits; if ((cfg.peersOutMax == 0u) && (cfg.peersInMax == 0u)) { - if (cfg.peersMax != 0) - config.maxPeers = cfg.peersMax; - - config.maxPeers = std::max(config.maxPeers, Tuning::kMinOutCount); - config.outPeers = config.calcOutPeers(); - - // Calculate the number of outbound peers we want. If we dont want - // or can't accept incoming, this will simply be equal to maxPeers. - if (!config.wantIncoming) - config.outPeers = config.maxPeers; - - // Calculate the largest number of inbound connections we could - // take. - if (config.maxPeers >= config.outPeers) - { - config.inPeers = config.maxPeers - config.outPeers; - } - else - { - config.inPeers = 0; - } + limits.maxPeers = cfg.peersMax; } else { - config.outPeers = cfg.peersOutMax; - config.inPeers = cfg.peersInMax; - config.maxPeers = 0; + limits.inPeers = cfg.peersInMax; + limits.outPeers = cfg.peersOutMax; } - // This will cause servers configured as validators to request that - // peers they connect to never report their IP address. We set this - // after we set the 'wantIncoming' because we want a "soft" version - // of peer privacy unless the operator explicitly asks for it. - if (validationPublicKey) - config.peerPrivate = true; - - // if it's a private peer or we are running as standalone - // automatic connections would defeat the purpose. - config.autoConnect = !cfg.standalone() && !cfg.peerPrivate; - config.listeningPort = port; - config.features = ""; - config.ipLimit = ipLimit; - config.verifyEndpoints = verifyEndpoints; - - // Enforce business rules - config.applyTuning(); - - return config; + return Config::makeConfig( + cfg.peerPrivate, + cfg.standalone(), + limits, + port, + validationPublicKey, + ipLimit, + verifyEndpoints); } } // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/StoreSqdb.h b/src/xrpld/peerfinder/detail/StoreSqdb.h index b17d2fdc5b..b0973a42b3 100644 --- a/src/xrpld/peerfinder/detail/StoreSqdb.h +++ b/src/xrpld/peerfinder/detail/StoreSqdb.h @@ -1,11 +1,11 @@ #pragma once #include -#include #include #include #include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/iosformat.h b/src/xrpld/peerfinder/detail/iosformat.h deleted file mode 100644 index 46c69ef602..0000000000 --- a/src/xrpld/peerfinder/detail/iosformat.h +++ /dev/null @@ -1,201 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace beast { - -// A collection of handy stream manipulators and -// functions to produce nice looking log output. - -/** - * Left justifies a field at the specified width. - */ -struct Leftw -{ - explicit Leftw(int width) : width(width) - { - } - int const width; - template - friend std::basic_ios& - operator<<(std::basic_ios& ios, Leftw const& p) - { - ios.setf(std::ios_base::left, std::ios_base::adjustfield); - ios.width(p.width); - return ios; - } -}; - -/** - * Produce a section heading and fill the rest of the line with dashes. - */ -template -std::basic_string -heading(std::basic_string title, int width = 80, CharT fill = CharT('-')) -{ - title.reserve(width); - title.push_back(CharT(' ')); - title.resize(width, fill); - return title; -} - -/** - * Produce a dashed line separator, with a specified or default size. - */ -struct Divider -{ - using CharT = char; - explicit Divider(int width = 80, CharT fill = CharT('-')) : width(width), fill(fill) - { - } - int const width; - CharT const fill; - template - friend std::basic_ostream& - operator<<(std::basic_ostream& os, Divider const& d) - { - os << std::basic_string(d.width, d.fill); - return os; - } -}; - -/** - * Creates a padded field with an optional fill character. - */ -struct Fpad -{ - explicit Fpad(int width, int pad = 0, char fill = ' ') : width(width + pad), fill(fill) - { - } - int const width; - char const fill; - template - friend std::basic_ostream& - operator<<(std::basic_ostream& os, Fpad const& f) - { - os << std::basic_string(f.width, f.fill); - return os; - } -}; - -//------------------------------------------------------------------------------ - -namespace detail { - -template -std::string -to_string(T const& t) -{ - std::stringstream ss; - ss << t; - return ss.str(); -} - -} // namespace detail - -/** - * Justifies a field at the specified width. - */ -/** @{ */ -template < - class CharT, - class Traits = std::char_traits, - class Allocator = std::allocator> -class FieldT -{ -public: - using string_t = std::basic_string; - FieldT(string_t const& text, int width, int pad, bool right) - : text(text), width(width), pad(pad), right(right) - { - } - string_t const text; - int const width; - int const pad; - bool const right; - template - friend std::basic_ostream& - operator<<(std::basic_ostream& os, FieldT const& f) - { - std::size_t const length(f.text.length()); - if (f.right) - { - if (length < f.width) - os << std::basic_string(f.width - length, CharT2(' ')); - os << f.text; - } - else - { - os << f.text; - if (length < f.width) - os << std::basic_string(f.width - length, CharT2(' ')); - } - if (f.pad != 0) - os << string_t(f.pad, CharT(' ')); - return os; - } -}; - -template -FieldT -field( - std::basic_string const& text, - int width = 8, - int pad = 0, - bool right = false) -{ - return FieldT(text, width, pad, right); -} - -template -FieldT -field(CharT const* text, int width = 8, int pad = 0, bool right = false) -{ - return FieldT, std::allocator>( - std::basic_string, std::allocator>(text), - width, - pad, - right); -} - -template -FieldT -field(T const& t, int width = 8, int pad = 0, bool right = false) -{ - std::string const text(detail::to_string(t)); - return field(text, width, pad, right); -} - -template -FieldT -rField(std::basic_string const& text, int width = 8, int pad = 0) -{ - return FieldT(text, width, pad, true); -} - -template -FieldT -rField(CharT const* text, int width = 8, int pad = 0) -{ - return FieldT, std::allocator>( - std::basic_string, std::allocator>(text), - width, - pad, - true); -} - -template -FieldT -rField(T const& t, int width = 8, int pad = 0) -{ - std::string const text(detail::to_string(t)); - return field(text, width, pad, true); -} -/** @} */ - -} // namespace beast diff --git a/src/xrpld/peerfinder/make_Manager.h b/src/xrpld/peerfinder/make_Manager.h deleted file mode 100644 index 1c13d7a4ca..0000000000 --- a/src/xrpld/peerfinder/make_Manager.h +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -#include -#include -#include - -#include - -#include - -namespace xrpl::PeerFinder { - -/** - * Create a new Manager. - */ -std::unique_ptr -makeManager( - boost::asio::io_context& ioContext, - clock_type& clock, - beast::Journal journal, - BasicConfig const& config, - beast::insight::Collector::ptr const& collector); - -} // namespace xrpl::PeerFinder From 4c0180b3dbb6eee7443ac02124601d09107d9dc8 Mon Sep 17 00:00:00 2001 From: Marek Foss Date: Thu, 23 Jul 2026 22:38:21 +0100 Subject: [PATCH 08/13] test: Migrate csf and xrpld-consensus Beast non-JTx tests to GTest (#7046) Co-authored-by: Alex Kremer --- .cspell.config.yaml | 4 +- .github/scripts/levelization/README.md | 18 +- .../scripts/levelization/results/ordering.txt | 24 +- cmake/XrplCore.cmake | 11 + .../xrpl/consensus/CensorshipDetector.h | 4 +- .../xrpl}/consensus/Consensus.h | 7 +- .../xrpl}/consensus/ConsensusParms.h | 0 .../xrpl}/consensus/ConsensusProposal.h | 0 .../xrpl}/consensus/ConsensusTypes.h | 5 +- .../xrpl}/consensus/DisputedTx.h | 3 +- .../xrpl}/consensus/LedgerTrie.h | 0 .../xrpl}/consensus/README.md | 0 .../xrpl}/consensus/Validations.h | 3 +- .../consensus/Consensus.cpp | 7 +- src/test/app/RCLValidations_test.cpp | 2 +- .../consensus/ByzantineFailureSim_test.cpp | 88 - src/test/consensus/Consensus_test.cpp | 1432 ---------------- .../DistributedValidatorsSim_test.cpp | 253 --- src/test/consensus/LedgerTiming_test.cpp | 118 -- src/test/consensus/LedgerTrie_test.cpp | 716 -------- .../consensus/RCLCensorshipDetector_test.cpp | 83 - src/test/consensus/ScaleFreeSim_test.cpp | 109 -- src/test/consensus/Validations_test.cpp | 1059 ------------ src/test/csf/BasicNetwork_test.cpp | 134 -- src/test/csf/Digraph_test.cpp | 81 - src/test/csf/Histogram_test.cpp | 66 - src/test/csf/Scheduler_test.cpp | 68 - src/tests/libxrpl/CMakeLists.txt | 11 + .../{base_uint_test.cpp => base_uint.cpp} | 3 +- .../libxrpl/consensus/ByzantineFailureSim.cpp | 81 + .../libxrpl/consensus/CensorshipDetector.cpp | 81 + src/tests/libxrpl/consensus/Consensus.cpp | 1455 +++++++++++++++++ .../consensus/DistributedValidatorsSim.cpp | 252 +++ src/tests/libxrpl/consensus/LedgerTiming.cpp | 105 ++ src/tests/libxrpl/consensus/LedgerTrie.cpp | 693 ++++++++ src/tests/libxrpl/consensus/ScaleFreeSim.cpp | 100 ++ src/tests/libxrpl/consensus/Validations.cpp | 1030 ++++++++++++ src/tests/libxrpl/csf/BasicNetwork.cpp | 122 ++ .../libxrpl}/csf/BasicNetwork.h | 4 +- .../libxrpl}/csf/CollectorRef.h | 12 +- src/tests/libxrpl/csf/Digraph.cpp | 72 + src/{test => tests/libxrpl}/csf/Digraph.h | 0 src/tests/libxrpl/csf/Histogram.cpp | 59 + src/{test => tests/libxrpl}/csf/Histogram.h | 0 src/{test => tests/libxrpl}/csf/Peer.h | 31 +- src/{test => tests/libxrpl}/csf/PeerGroup.h | 8 +- src/{test => tests/libxrpl}/csf/Proposal.h | 8 +- src/{test => tests/libxrpl}/csf/README.md | 0 src/tests/libxrpl/csf/Scheduler.cpp | 61 + src/{test => tests/libxrpl}/csf/Scheduler.h | 0 src/{test => tests/libxrpl}/csf/Sim.h | 18 +- src/{test => tests/libxrpl}/csf/SimTime.h | 0 src/{test => tests/libxrpl}/csf/TrustGraph.h | 4 +- src/{test => tests/libxrpl}/csf/Tx.h | 0 src/{test => tests/libxrpl}/csf/Validation.h | 4 +- src/{test => tests/libxrpl}/csf/collectors.h | 12 +- src/{test => tests/libxrpl}/csf/csf_graph.png | Bin .../libxrpl}/csf/csf_overview.png | Bin src/{test => tests/libxrpl}/csf/events.h | 6 +- src/{test => tests/libxrpl}/csf/impl/Sim.cpp | 6 +- .../libxrpl}/csf/impl/ledgers.cpp | 6 +- src/{test => tests/libxrpl}/csf/ledgers.h | 4 +- src/{test => tests/libxrpl}/csf/random.h | 4 +- src/{test => tests/libxrpl}/csf/submitters.h | 6 +- src/{test => tests/libxrpl}/csf/timers.h | 4 +- src/xrpld/app/consensus/RCLConsensus.cpp | 8 +- src/xrpld/app/consensus/RCLConsensus.h | 10 +- src/xrpld/app/consensus/RCLCxPeerPos.h | 3 +- src/xrpld/app/consensus/RCLValidations.cpp | 2 +- src/xrpld/app/consensus/RCLValidations.h | 3 +- src/xrpld/app/misc/NetworkOPs.cpp | 4 +- src/xrpld/overlay/detail/PeerImp.cpp | 2 +- 72 files changed, 4253 insertions(+), 4336 deletions(-) rename src/xrpld/app/consensus/RCLCensorshipDetector.h => include/xrpl/consensus/CensorshipDetector.h (98%) rename {src/xrpld => include/xrpl}/consensus/Consensus.h (99%) rename {src/xrpld => include/xrpl}/consensus/ConsensusParms.h (100%) rename {src/xrpld => include/xrpl}/consensus/ConsensusProposal.h (100%) rename {src/xrpld => include/xrpl}/consensus/ConsensusTypes.h (98%) rename {src/xrpld => include/xrpl}/consensus/DisputedTx.h (99%) rename {src/xrpld => include/xrpl}/consensus/LedgerTrie.h (100%) rename {src/xrpld => include/xrpl}/consensus/README.md (100%) rename {src/xrpld => include/xrpl}/consensus/Validations.h (99%) rename src/{xrpld => libxrpl}/consensus/Consensus.cpp (98%) delete mode 100644 src/test/consensus/ByzantineFailureSim_test.cpp delete mode 100644 src/test/consensus/Consensus_test.cpp delete mode 100644 src/test/consensus/DistributedValidatorsSim_test.cpp delete mode 100644 src/test/consensus/LedgerTiming_test.cpp delete mode 100644 src/test/consensus/LedgerTrie_test.cpp delete mode 100644 src/test/consensus/RCLCensorshipDetector_test.cpp delete mode 100644 src/test/consensus/ScaleFreeSim_test.cpp delete mode 100644 src/test/consensus/Validations_test.cpp delete mode 100644 src/test/csf/BasicNetwork_test.cpp delete mode 100644 src/test/csf/Digraph_test.cpp delete mode 100644 src/test/csf/Histogram_test.cpp delete mode 100644 src/test/csf/Scheduler_test.cpp rename src/tests/libxrpl/basics/{base_uint_test.cpp => base_uint.cpp} (99%) create mode 100644 src/tests/libxrpl/consensus/ByzantineFailureSim.cpp create mode 100644 src/tests/libxrpl/consensus/CensorshipDetector.cpp create mode 100644 src/tests/libxrpl/consensus/Consensus.cpp create mode 100644 src/tests/libxrpl/consensus/DistributedValidatorsSim.cpp create mode 100644 src/tests/libxrpl/consensus/LedgerTiming.cpp create mode 100644 src/tests/libxrpl/consensus/LedgerTrie.cpp create mode 100644 src/tests/libxrpl/consensus/ScaleFreeSim.cpp create mode 100644 src/tests/libxrpl/consensus/Validations.cpp create mode 100644 src/tests/libxrpl/csf/BasicNetwork.cpp rename src/{test => tests/libxrpl}/csf/BasicNetwork.h (99%) rename src/{test => tests/libxrpl}/csf/CollectorRef.h (97%) create mode 100644 src/tests/libxrpl/csf/Digraph.cpp rename src/{test => tests/libxrpl}/csf/Digraph.h (100%) create mode 100644 src/tests/libxrpl/csf/Histogram.cpp rename src/{test => tests/libxrpl}/csf/Histogram.h (100%) rename src/{test => tests/libxrpl}/csf/Peer.h (98%) rename src/{test => tests/libxrpl}/csf/PeerGroup.h (98%) rename src/{test => tests/libxrpl}/csf/Proposal.h (66%) rename src/{test => tests/libxrpl}/csf/README.md (100%) create mode 100644 src/tests/libxrpl/csf/Scheduler.cpp rename src/{test => tests/libxrpl}/csf/Scheduler.h (100%) rename src/{test => tests/libxrpl}/csf/Sim.h (94%) rename src/{test => tests/libxrpl}/csf/SimTime.h (100%) rename src/{test => tests/libxrpl}/csf/TrustGraph.h (99%) rename src/{test => tests/libxrpl}/csf/Tx.h (100%) rename src/{test => tests/libxrpl}/csf/Validation.h (99%) rename src/{test => tests/libxrpl}/csf/collectors.h (99%) rename src/{test => tests/libxrpl}/csf/csf_graph.png (100%) rename src/{test => tests/libxrpl}/csf/csf_overview.png (100%) rename src/{test => tests/libxrpl}/csf/events.h (96%) rename src/{test => tests/libxrpl}/csf/impl/Sim.cpp (93%) rename src/{test => tests/libxrpl}/csf/impl/ledgers.cpp (98%) rename src/{test => tests/libxrpl}/csf/ledgers.h (99%) rename src/{test => tests/libxrpl}/csf/random.h (98%) rename src/{test => tests/libxrpl}/csf/submitters.h (96%) rename src/{test => tests/libxrpl}/csf/timers.h (96%) diff --git a/.cspell.config.yaml b/.cspell.config.yaml index c862379f08..9cd8417362 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -30,7 +30,9 @@ ignoreRegExpList: - ABCDEFGHIJKLMNOPQRSTUVWXYZ - ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz overrides: - - filename: "**/*_test.cpp" # all test files + - filename: + - "**/*_test.cpp" # legacy boost.test files + - "src/tests/**/*.cpp" # gtest test files ignoreRegExpList: - /"[^"]*"/g # double-quoted strings - /'[^']*'/g # single-quoted strings diff --git a/.github/scripts/levelization/README.md b/.github/scripts/levelization/README.md index f657344827..93748c43e1 100644 --- a/.github/scripts/levelization/README.md +++ b/.github/scripts/levelization/README.md @@ -40,18 +40,18 @@ listed later. | 04 | xrpl/protocol | | 05 | xrpl/core xrpl/resource xrpl/server | | 06 | xrpl/ledger xrpl/nodestore xrpl/net | -| 07 | xrpl/shamap | +| 07 | xrpl/shamap xrpl/consensus | ## xrpld Modules (Application Implementation) -| Level / Tier | Module(s) | -| ------------ | -------------------------------- | -| 05 | xrpld/conditions xrpld/consensus | -| 06 | xrpld/core xrpld/peerfinder | -| 07 | xrpld/shamap xrpld/overlay | -| 08 | xrpld/app | -| 09 | xrpld/rpc | -| 10 | xrpld/perflog | +| Level / Tier | Module(s) | +| ------------ | --------------------------- | +| 05 | xrpld/conditions | +| 06 | xrpld/core xrpld/peerfinder | +| 07 | xrpld/shamap xrpld/overlay | +| 08 | xrpld/app | +| 09 | xrpld/rpc | +| 10 | xrpld/perflog | ## Test Modules diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index fdd134dc8a..709ba4d6d4 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -6,6 +6,8 @@ libxrpl.conditions > xrpl.basics libxrpl.conditions > xrpl.conditions libxrpl.config > xrpl.basics libxrpl.config > xrpl.config +libxrpl.consensus > xrpl.basics +libxrpl.consensus > xrpl.consensus libxrpl.core > xrpl.basics libxrpl.core > xrpl.core libxrpl.core > xrpl.json @@ -63,9 +65,9 @@ test.app > test.jtx test.app > test.unit_test test.app > xrpl.basics test.app > xrpl.config +test.app > xrpl.consensus test.app > xrpl.core test.app > xrpld.app -test.app > xrpld.consensus test.app > xrpld.core test.app > xrpld.overlay test.app > xrpld.rpc @@ -86,12 +88,9 @@ test.basics > xrpl.protocol test.beast > xrpl.basics test.conditions > xrpl.basics test.conditions > xrpl.conditions -test.consensus > test.csf test.consensus > test.jtx -test.consensus > test.unit_test test.consensus > xrpl.basics test.consensus > xrpld.app -test.consensus > xrpld.consensus test.consensus > xrpl.ledger test.consensus > xrpl.protocol test.consensus > xrpl.shamap @@ -106,10 +105,6 @@ test.core > xrpl.json test.core > xrpl.protocol test.core > xrpl.rdb test.core > xrpl.server -test.csf > xrpl.basics -test.csf > xrpld.consensus -test.csf > xrpl.json -test.csf > xrpl.ledger test.json > test.jtx test.json > xrpl.json test.jtx > test.unit_test @@ -189,6 +184,7 @@ test.unit_test > xrpl.basics test.unit_test > xrpl.protocol tests.libxrpl > xrpl.basics tests.libxrpl > xrpl.config +tests.libxrpl > xrpl.consensus tests.libxrpl > xrpl.core tests.libxrpl > xrpl.json tests.libxrpl > xrpl.ledger @@ -204,6 +200,10 @@ tests.libxrpl > xrpl.tx xrpl.conditions > xrpl.basics xrpl.conditions > xrpl.protocol xrpl.config > xrpl.basics +xrpl.consensus > xrpl.basics +xrpl.consensus > xrpl.json +xrpl.consensus > xrpl.ledger +xrpl.consensus > xrpl.protocol xrpl.core > xrpl.basics xrpl.core > xrpl.json xrpl.core > xrpl.protocol @@ -246,8 +246,8 @@ xrpl.tx > xrpl.protocol xrpld.app > test.unit_test xrpld.app > xrpl.basics xrpld.app > xrpl.config +xrpld.app > xrpl.consensus xrpld.app > xrpl.core -xrpld.app > xrpld.consensus xrpld.app > xrpld.core xrpld.app > xrpl.json xrpld.app > xrpl.ledger @@ -260,10 +260,6 @@ xrpld.app > xrpl.resource xrpld.app > xrpl.server xrpld.app > xrpl.shamap xrpld.app > xrpl.tx -xrpld.consensus > xrpl.basics -xrpld.consensus > xrpl.json -xrpld.consensus > xrpl.ledger -xrpld.consensus > xrpl.protocol xrpld.core > xrpl.basics xrpld.core > xrpl.config xrpld.core > xrpl.core @@ -272,8 +268,8 @@ xrpld.core > xrpl.protocol xrpld.core > xrpl.rdb xrpld.overlay > xrpl.basics xrpld.overlay > xrpl.config +xrpld.overlay > xrpl.consensus xrpld.overlay > xrpl.core -xrpld.overlay > xrpld.consensus xrpld.overlay > xrpld.core xrpld.overlay > xrpld.peerfinder xrpld.overlay > xrpl.json diff --git a/cmake/XrplCore.cmake b/cmake/XrplCore.cmake index 62a8fe143b..a3e08145d5 100644 --- a/cmake/XrplCore.cmake +++ b/cmake/XrplCore.cmake @@ -207,6 +207,16 @@ target_link_libraries( add_module(xrpl tx) target_link_libraries(xrpl.libxrpl.tx PUBLIC xrpl.libxrpl.ledger) +add_module(xrpl consensus) +target_link_libraries( + xrpl.libxrpl.consensus + PUBLIC + xrpl.libxrpl.basics + xrpl.libxrpl.json + xrpl.libxrpl.protocol + xrpl.libxrpl.ledger +) + add_library(xrpl.libxrpl) set_target_properties(xrpl.libxrpl PROPERTIES OUTPUT_NAME xrpl) @@ -226,6 +236,7 @@ target_link_modules( beast conditions config + consensus core crypto git diff --git a/src/xrpld/app/consensus/RCLCensorshipDetector.h b/include/xrpl/consensus/CensorshipDetector.h similarity index 98% rename from src/xrpld/app/consensus/RCLCensorshipDetector.h rename to include/xrpl/consensus/CensorshipDetector.h index 6d0e20031e..3d2708f68f 100644 --- a/src/xrpld/app/consensus/RCLCensorshipDetector.h +++ b/include/xrpl/consensus/CensorshipDetector.h @@ -10,7 +10,7 @@ namespace xrpl { template -class RCLCensorshipDetector +class CensorshipDetector { public: struct TxIDSeq @@ -49,7 +49,7 @@ private: TxIDSeqVec tracker_; public: - RCLCensorshipDetector() = default; + CensorshipDetector() = default; /** * Add transactions being proposed for the current consensus round. diff --git a/src/xrpld/consensus/Consensus.h b/include/xrpl/consensus/Consensus.h similarity index 99% rename from src/xrpld/consensus/Consensus.h rename to include/xrpl/consensus/Consensus.h index 440191939b..f9d5f7ef02 100644 --- a/src/xrpld/consensus/Consensus.h +++ b/include/xrpl/consensus/Consensus.h @@ -1,15 +1,14 @@ #pragma once -#include -#include -#include - #include #include #include #include #include #include +#include +#include +#include #include #include #include diff --git a/src/xrpld/consensus/ConsensusParms.h b/include/xrpl/consensus/ConsensusParms.h similarity index 100% rename from src/xrpld/consensus/ConsensusParms.h rename to include/xrpl/consensus/ConsensusParms.h diff --git a/src/xrpld/consensus/ConsensusProposal.h b/include/xrpl/consensus/ConsensusProposal.h similarity index 100% rename from src/xrpld/consensus/ConsensusProposal.h rename to include/xrpl/consensus/ConsensusProposal.h diff --git a/src/xrpld/consensus/ConsensusTypes.h b/include/xrpl/consensus/ConsensusTypes.h similarity index 98% rename from src/xrpld/consensus/ConsensusTypes.h rename to include/xrpl/consensus/ConsensusTypes.h index 4553e46f48..4dac2d9912 100644 --- a/src/xrpld/consensus/ConsensusTypes.h +++ b/include/xrpl/consensus/ConsensusTypes.h @@ -1,11 +1,10 @@ #pragma once -#include -#include - #include #include #include +#include +#include #include #include diff --git a/src/xrpld/consensus/DisputedTx.h b/include/xrpl/consensus/DisputedTx.h similarity index 99% rename from src/xrpld/consensus/DisputedTx.h rename to include/xrpl/consensus/DisputedTx.h index 12ed00d460..c194716d43 100644 --- a/src/xrpld/consensus/DisputedTx.h +++ b/include/xrpl/consensus/DisputedTx.h @@ -1,9 +1,8 @@ #pragma once -#include - #include #include +#include #include #include diff --git a/src/xrpld/consensus/LedgerTrie.h b/include/xrpl/consensus/LedgerTrie.h similarity index 100% rename from src/xrpld/consensus/LedgerTrie.h rename to include/xrpl/consensus/LedgerTrie.h diff --git a/src/xrpld/consensus/README.md b/include/xrpl/consensus/README.md similarity index 100% rename from src/xrpld/consensus/README.md rename to include/xrpl/consensus/README.md diff --git a/src/xrpld/consensus/Validations.h b/include/xrpl/consensus/Validations.h similarity index 99% rename from src/xrpld/consensus/Validations.h rename to include/xrpl/consensus/Validations.h index 2696804c86..ebb13c5e7c 100644 --- a/src/xrpld/consensus/Validations.h +++ b/include/xrpl/consensus/Validations.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include #include @@ -11,6 +9,7 @@ #include #include #include +#include #include #include diff --git a/src/xrpld/consensus/Consensus.cpp b/src/libxrpl/consensus/Consensus.cpp similarity index 98% rename from src/xrpld/consensus/Consensus.cpp rename to src/libxrpl/consensus/Consensus.cpp index d529ab2e44..6f398cf66c 100644 --- a/src/xrpld/consensus/Consensus.cpp +++ b/src/libxrpl/consensus/Consensus.cpp @@ -1,10 +1,9 @@ -#include - -#include -#include +#include #include #include +#include +#include #include #include diff --git a/src/test/app/RCLValidations_test.cpp b/src/test/app/RCLValidations_test.cpp index aaf84225a7..9ace2e21af 100644 --- a/src/test/app/RCLValidations_test.cpp +++ b/src/test/app/RCLValidations_test.cpp @@ -2,12 +2,12 @@ #include #include -#include #include #include #include #include +#include #include #include #include diff --git a/src/test/consensus/ByzantineFailureSim_test.cpp b/src/test/consensus/ByzantineFailureSim_test.cpp deleted file mode 100644 index c3c51125b5..0000000000 --- a/src/test/consensus/ByzantineFailureSim_test.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include - -namespace xrpl::test { - -class ByzantineFailureSim_test : public beast::unit_test::Suite -{ - void - run() override - { - using namespace csf; - using namespace std::chrono; - - // This test simulates a specific topology with nodes generating - // different ledgers due to a simulated byzantine failure (injecting - // an extra non-consensus transaction). - - Sim sim; - ConsensusParms const parms{}; - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - PeerGroup a = sim.createGroup(1); - PeerGroup b = sim.createGroup(1); - PeerGroup c = sim.createGroup(1); - PeerGroup d = sim.createGroup(1); - PeerGroup e = sim.createGroup(1); - PeerGroup f = sim.createGroup(1); - PeerGroup g = sim.createGroup(1); - - a.trustAndConnect(a + b + c + g, delay); - b.trustAndConnect(b + a + c + d + e, delay); - c.trustAndConnect(c + a + b + d + e, delay); - d.trustAndConnect(d + b + c + e + f, delay); - e.trustAndConnect(e + b + c + d + f, delay); - f.trustAndConnect(f + d + e + g, delay); - g.trustAndConnect(g + a + f, delay); - - PeerGroup const network = a + b + c + d + e + f + g; - - StreamCollector sc{std::cout}; - - sim.collectors.add(sc); - - for (TrustGraph::ForkInfo const& fi : sim.trustGraph.forkablePairs(0.8)) - { - std::cout << "Can fork " << PeerGroup{fi.unlA} << " " - << " " << PeerGroup{fi.unlB} << " overlap " << fi.overlap << " required " - << fi.required << "\n"; - }; - - // set prior state - sim.run(1); - - PeerGroup byzantineNodes = a + b + c + g; - // All peers see some TX 0 - for (Peer* peer : network) - { - peer->submit(Tx{0}); - // Peers 0,1,2,6 will close the next ledger differently by injecting - // a non-consensus approved transaction - if (byzantineNodes.contains(peer)) - { - peer->txInjections.emplace(peer->lastClosedLedger.seq(), Tx{42}); - } - } - sim.run(4); - std::cout << "Branches: " << sim.branches() << "\n"; - std::cout << "Fully synchronized: " << std::boolalpha << sim.synchronized() << "\n"; - // Not tessting anything currently. - pass(); - } -}; - -BEAST_DEFINE_TESTSUITE_MANUAL(ByzantineFailureSim, consensus, xrpl); - -} // namespace xrpl::test diff --git a/src/test/consensus/Consensus_test.cpp b/src/test/consensus/Consensus_test.cpp deleted file mode 100644 index 45f58d16ba..0000000000 --- a/src/test/consensus/Consensus_test.cpp +++ /dev/null @@ -1,1432 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace xrpl::test { - -class Consensus_test : public beast::unit_test::Suite -{ - SuiteJournal journal_; - -public: - Consensus_test() : journal_("Consensus_test", *this) - { - } - - void - testShouldCloseLedger() - { - using namespace std::chrono_literals; - testcase("should close ledger"); - - // Use default parameters - ConsensusParms const p{}; - - // Bizarre times forcibly close - BEAST_EXPECT(shouldCloseLedger(true, 10, 10, 10, -10s, 10s, 1s, 1s, p, journal_)); - BEAST_EXPECT(shouldCloseLedger(true, 10, 10, 10, 100h, 10s, 1s, 1s, p, journal_)); - BEAST_EXPECT(shouldCloseLedger(true, 10, 10, 10, 10s, 100h, 1s, 1s, p, journal_)); - - // Rest of network has closed - BEAST_EXPECT(shouldCloseLedger(true, 10, 3, 5, 10s, 10s, 10s, 10s, p, journal_)); - - // No transactions means wait until end of internval - BEAST_EXPECT(!shouldCloseLedger(false, 10, 0, 0, 1s, 1s, 1s, 10s, p, journal_)); - BEAST_EXPECT(shouldCloseLedger(false, 10, 0, 0, 1s, 10s, 1s, 10s, p, journal_)); - - // Enforce minimum ledger open time - BEAST_EXPECT(!shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 1s, 10s, p, journal_)); - - // Don't go too much faster than last time - BEAST_EXPECT(!shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 3s, 10s, p, journal_)); - - BEAST_EXPECT(shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 10s, 10s, p, journal_)); - } - - void - testCheckConsensus() - { - using namespace std::chrono_literals; - testcase("check consensus"); - - // Use default parameters - ConsensusParms const p{}; - - /////////////// - // Disputes still in doubt - // - // Not enough time has elapsed - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 2s, false, p, true, journal_)); - - // If not enough peers have proposed, ensure - // more time for proposals - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 4s, false, p, true, journal_)); - - // Enough time has elapsed and we all agree - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 2, 2, 0, 3s, 10s, false, p, true, journal_)); - - // Enough time has elapsed and we don't yet agree - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 1, 0, 3s, 10s, false, p, true, journal_)); - - // Our peers have moved on - // Enough time has elapsed and we all agree - BEAST_EXPECT( - ConsensusState::MovedOn == - checkConsensus(10, 2, 1, 8, 3s, 10s, false, p, true, journal_)); - - // If no peers, don't agree until time has passed. - BEAST_EXPECT( - ConsensusState::No == checkConsensus(0, 0, 0, 0, 3s, 10s, false, p, true, journal_)); - - // Agree if no peers and enough time has passed. - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(0, 0, 0, 0, 3s, 16s, false, p, true, journal_)); - - // Expire if too much time has passed without agreement - BEAST_EXPECT( - ConsensusState::Expired == - checkConsensus(10, 8, 1, 0, 1s, 19s, false, p, true, journal_)); - - /////////////// - // Stalled - // - // Not enough time has elapsed - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 2s, true, p, true, journal_)); - - // If not enough peers have proposed, ensure - // more time for proposals - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 4s, true, p, true, journal_)); - - // Enough time has elapsed and we all agree - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 2, 2, 0, 3s, 10s, true, p, true, journal_)); - - // Enough time has elapsed and we don't yet agree, but there's nothing - // left to dispute - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 2, 1, 0, 3s, 10s, true, p, true, journal_)); - - // Our peers have moved on - // Enough time has elapsed and we all agree, nothing left to dispute - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 2, 1, 8, 3s, 10s, true, p, true, journal_)); - - // If no peers, don't agree until time has passed. - BEAST_EXPECT( - ConsensusState::No == checkConsensus(0, 0, 0, 0, 3s, 10s, true, p, true, journal_)); - - // Agree if no peers and enough time has passed. - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(0, 0, 0, 0, 3s, 16s, true, p, true, journal_)); - - // We are done if there's nothing left to dispute, no matter how much - // time has passed - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 8, 1, 0, 1s, 19s, true, p, true, journal_)); - } - - void - testStandalone() - { - using namespace std::chrono_literals; - using namespace csf; - testcase("standalone"); - - Sim s; - PeerGroup const peers = s.createGroup(1); - Peer* peer = peers[0]; - peer->targetLedgers = 1; - peer->start(); - peer->submit(Tx{1}); - - s.scheduler.step(); - - // Inspect that the proper ledger was created - auto const& lcl = peer->lastClosedLedger; - BEAST_EXPECT(peer->prevLedgerID() == lcl.id()); - BEAST_EXPECT(lcl.seq() == Ledger::Seq{1}); - BEAST_EXPECT(lcl.txs().size() == 1); - BEAST_EXPECT(lcl.txs().contains(Tx{1})); - BEAST_EXPECT(peer->prevProposers == 0); - } - - void - testPeersAgree() - { - using namespace csf; - using namespace std::chrono; - testcase("peers agree"); - - ConsensusParms const parms{}; - Sim sim; - PeerGroup peers = sim.createGroup(5); - - // Connected trust and network graphs with single fixed delay - peers.trustAndConnect(peers, round(0.2 * parms.ledgerGRANULARITY)); - - // everyone submits their own ID as a TX - for (Peer* p : peers) - p->submit(Tx(static_cast(p->id))); - - sim.run(1); - - // All peers are in sync - if (BEAST_EXPECT(sim.synchronized())) - { - for (Peer const* peer : peers) - { - auto const& lcl = peer->lastClosedLedger; - BEAST_EXPECT(lcl.id() == peer->prevLedgerID()); - BEAST_EXPECT(lcl.seq() == Ledger::Seq{1}); - // All peers proposed - BEAST_EXPECT(peer->prevProposers == peers.size() - 1); - // All transactions were accepted - for (std::uint32_t i = 0; i < peers.size(); ++i) - BEAST_EXPECT(lcl.txs().contains(Tx{i})); - } - } - } - - void - testSlowPeers() - { - using namespace csf; - using namespace std::chrono; - testcase("slow peers"); - - // Several tests of a complete trust graph with a subset of peers - // that have significantly longer network delays to the rest of the - // network - - // Test when a slow peer doesn't delay a consensus quorum (4/5 agree) - { - ConsensusParms const parms{}; - Sim sim; - PeerGroup slow = sim.createGroup(1); - PeerGroup fast = sim.createGroup(4); - PeerGroup network = fast + slow; - - // Fully connected trust graph - network.trust(network); - - // Fast and slow network connections - fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); - - slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); - - // All peers submit their own ID as a transaction - for (Peer* peer : network) - peer->submit(Tx{static_cast(peer->id)}); - - sim.run(1); - - // Verify all peers have same LCL but are missing transaction 0 - // All peers are in sync even with a slower peer 0 - if (BEAST_EXPECT(sim.synchronized())) - { - for (Peer const* peer : network) - { - auto const& lcl = peer->lastClosedLedger; - BEAST_EXPECT(lcl.id() == peer->prevLedgerID()); - BEAST_EXPECT(lcl.seq() == Ledger::Seq{1}); - - BEAST_EXPECT(peer->prevProposers == network.size() - 1); - BEAST_EXPECT(peer->prevRoundTime == network[0]->prevRoundTime); - - BEAST_EXPECT(not lcl.txs().contains(Tx{0})); - for (std::uint32_t i = 2; i < network.size(); ++i) - BEAST_EXPECT(lcl.txs().contains(Tx{i})); - - // Tx 0 didn't make it - BEAST_EXPECT(peer->openTxs.contains(Tx{0})); - } - } - } - - // Test when the slow peers delay a consensus quorum (4/6 agree) - { - // Run two tests - // 1. The slow peers are participating in consensus - // 2. The slow peers are just observing - - for (auto isParticipant : {true, false}) - { - ConsensusParms const parms{}; - - Sim sim; - PeerGroup slow = sim.createGroup(2); - PeerGroup fast = sim.createGroup(4); - PeerGroup network = fast + slow; - - // Connected trust graph - network.trust(network); - - // Fast and slow network connections - fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); - - slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); - - for (Peer* peer : slow) - peer->runAsValidator = isParticipant; - - // All peers submit their own ID as a transaction and relay it - // to peers - for (Peer* peer : network) - peer->submit(Tx{static_cast(peer->id)}); - - sim.run(1); - - if (BEAST_EXPECT(sim.synchronized())) - { - // Verify all peers have same LCL but are missing - // transaction 0,1 which was not received by all peers - // before the ledger closed - for (Peer const* peer : network) - { - // Closed ledger has all but transaction 0,1 - auto const& lcl = peer->lastClosedLedger; - BEAST_EXPECT(lcl.seq() == Ledger::Seq{1}); - BEAST_EXPECT(not lcl.txs().contains(Tx{0})); - BEAST_EXPECT(not lcl.txs().contains(Tx{1})); - for (std::uint32_t i = slow.size(); i < network.size(); ++i) - BEAST_EXPECT(lcl.txs().contains(Tx{i})); - - // Tx 0-1 didn't make it - BEAST_EXPECT(peer->openTxs.contains(Tx{0})); - BEAST_EXPECT(peer->openTxs.contains(Tx{1})); - } - - Peer const* slowPeer = slow[0]; - if (isParticipant) - { - BEAST_EXPECT(slowPeer->prevProposers == network.size() - 1); - } - else - { - BEAST_EXPECT(slowPeer->prevProposers == fast.size()); - } - - for (Peer const* peer : fast) - { - // Due to the network link delay settings - // Peer 0 initially proposes {0} - // Peer 1 initially proposes {1} - // Peers 2-5 initially propose {2,3,4,5} - // Since peers 2-5 agree, 4/6 > the initial 50% needed - // to include a disputed transaction, so Peer 0/1 switch - // to agree with those peers. Peer 0/1 then closes with - // an 80% quorum of agreeing positions (5/6) match. - // - // Peers 2-5 do not change position, since tx 0 or tx 1 - // have less than the 50% initial threshold. They also - // cannot declare consensus, since 4/6 agreeing - // positions are < 80% threshold. They therefore need an - // additional timerEntry call to see the updated - // positions from Peer 0 & 1. - - if (isParticipant) - { - BEAST_EXPECT(peer->prevProposers == network.size() - 1); - BEAST_EXPECT(peer->prevRoundTime > slowPeer->prevRoundTime); - } - else - { - BEAST_EXPECT(peer->prevProposers == fast.size() - 1); - // so all peers should have closed together - BEAST_EXPECT(peer->prevRoundTime == slowPeer->prevRoundTime); - } - } - } - } - } - } - - void - testCloseTimeDisagree() - { - using namespace csf; - using namespace std::chrono; - testcase("close time disagree"); - - // This is a very specialized test to get ledgers to disagree on - // the close time. It unfortunately assumes knowledge about current - // timing constants. This is a necessary evil to get coverage up - // pending more extensive refactorings of timing constants. - - // In order to agree-to-disagree on the close time, there must be no - // clear majority of nodes agreeing on a close time. This test - // sets a relative offset to the peers internal clocks so that they - // send proposals with differing times. - - // However, agreement is on the effective close time, not the - // exact close time. The minimum closeTimeResolution is given by - // ledgerPossibleTimeResolutions[0], which is currently 10s. This means - // the skews need to be at least 10 seconds to have different effective - // close times. - - // Complicating this matter is that nodes will ignore proposals - // with times more than proposeFRESHNESS =20s in the past. So at - // the minimum granularity, we have at most 3 types of skews - // (0s,10s,20s). - - // This test therefore has 6 nodes, with 2 nodes having each type of - // skew. Then no majority (1/3 < 1/2) of nodes will agree on an - // actual close time. - - ConsensusParms const parms{}; - Sim sim; - - PeerGroup groupA = sim.createGroup(2); - PeerGroup const groupB = sim.createGroup(2); - PeerGroup const groupC = sim.createGroup(2); - PeerGroup network = groupA + groupB + groupC; - - network.trust(network); - network.connect(network, round(0.2 * parms.ledgerGRANULARITY)); - - // Run consensus without skew until we have a short close time - // resolution - Peer const* firstPeer = *groupA.begin(); - while (firstPeer->lastClosedLedger.closeTimeResolution() >= parms.proposeFRESHNESS) - sim.run(1); - - // Introduce a shift on the time of 2/3 of peers - for (Peer* peer : groupA) - peer->clockSkew = parms.proposeFRESHNESS / 2; - for (Peer* peer : groupB) - peer->clockSkew = parms.proposeFRESHNESS; - - sim.run(1); - - // All nodes agreed to disagree on the close time - if (BEAST_EXPECT(sim.synchronized())) - { - for (Peer const* peer : network) - BEAST_EXPECT(!peer->lastClosedLedger.closeAgree()); - } - } - - void - testWrongLCL() - { - using namespace csf; - using namespace std::chrono; - testcase("wrong LCL"); - - // Specialized test to exercise a temporary fork in which some peers - // are working on an incorrect prior ledger. - - ConsensusParms const parms{}; - - // Vary the time it takes to process validations to exercise detecting - // the wrong LCL at different phases of consensus - for (auto validationDelay : {0ms, parms.ledgerMinClose}) - { - // Consider 10 peers: - // 0 1 2 3 4 5 6 7 8 9 - // minority majorityA majorityB - // - // Nodes 0-1 trust nodes 0-4 - // Nodes 2-9 trust nodes 2-9 - // - // By submitting tx 0 to nodes 0-4 and tx 1 to nodes 5-9, - // nodes 0-1 will generate the wrong LCL (with tx 0). The remaining - // nodes will instead accept the ledger with tx 1. - - // Nodes 0-1 will detect this mismatch during a subsequent round - // since nodes 2-4 will validate a different ledger. - - // Nodes 0-1 will acquire the proper ledger from the network and - // resume consensus and eventually generate the dominant network - // ledger. - - // This topology can potentially fork with the above trust relations - // but that is intended for this test. - - Sim sim; - - PeerGroup minority = sim.createGroup(2); - PeerGroup const majorityA = sim.createGroup(3); - PeerGroup const majorityB = sim.createGroup(5); - - PeerGroup majority = majorityA + majorityB; - PeerGroup const network = minority + majority; - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - minority.trustAndConnect(minority + majorityA, delay); - majority.trustAndConnect(majority, delay); - - CollectByNode jumps; - sim.collectors.add(jumps); - - BEAST_EXPECT(sim.trustGraph.canFork(parms.minConsensusPct / 100.)); - - // initial round to set prior state - sim.run(1); - - // Nodes in smaller UNL have seen tx 0, nodes in other unl have seen - // tx 1 - for (Peer* peer : network) - peer->delays.recvValidation = validationDelay; - for (Peer* peer : (minority + majorityA)) - peer->openTxs.insert(Tx{0}); - for (Peer* peer : majorityB) - peer->openTxs.insert(Tx{1}); - - // Run for additional rounds - // With no validation delay, only 2 more rounds are needed. - // 1. Round to generate different ledgers - // 2. Round to detect different prior ledgers (but still generate - // wrong ones) and recover within that round since wrong LCL - // is detected before we close - // - // With a validation delay of ledgerMIN_CLOSE, we need 3 more - // rounds. - // 1. Round to generate different ledgers - // 2. Round to detect different prior ledgers (but still generate - // wrong ones) but end up declaring consensus on wrong LCL (but - // with the right transaction set!). This is because we detect - // the wrong LCL after we have closed the ledger, so we declare - // consensus based solely on our peer proposals. But we haven't - // had time to acquire the right ledger. - // 3. Round to correct - sim.run(3); - - // The network never actually forks, since node 0-1 never see a - // quorum of validations to fully validate the incorrect chain. - - // However, for a non zero-validation delay, the network is not - // synchronized because nodes 0 and 1 are running one ledger behind - if (BEAST_EXPECT(sim.branches() == 1)) - { - for (Peer const* peer : majority) - { - // No jumps for majority nodes - BEAST_EXPECT(jumps[peer->id].closeJumps.empty()); - BEAST_EXPECT(jumps[peer->id].fullyValidatedJumps.empty()); - } - for (Peer const* peer : minority) - { - auto& peerJumps = jumps[peer->id]; - // last closed ledger jump between chains - { - if (BEAST_EXPECT(peerJumps.closeJumps.size() == 1)) - { - JumpCollector::Jump const& jump = peerJumps.closeJumps.front(); - // Jump is to a different chain - BEAST_EXPECT(jump.from.seq() <= jump.to.seq()); - BEAST_EXPECT(!jump.to.isAncestor(jump.from)); - } - } - // fully validated jump forward in same chain - { - if (BEAST_EXPECT(peerJumps.fullyValidatedJumps.size() == 1)) - { - JumpCollector::Jump const& jump = peerJumps.fullyValidatedJumps.front(); - // Jump is to a different chain with same seq - BEAST_EXPECT(jump.from.seq() < jump.to.seq()); - BEAST_EXPECT(jump.to.isAncestor(jump.from)); - } - } - } - } - } - - { - // Additional test engineered to switch LCL during the establish - // phase. This was added to trigger a scenario that previously - // crashed, in which switchLCL switched from establish to open - // phase, but still processed the establish phase logic. - - // Loner node will accept an initial ledger A, but all other nodes - // accept ledger B a bit later. By delaying the time it takes - // to process a validation, loner node will detect the wrongLCL - // after it is already in the establish phase of the next round. - - Sim sim; - PeerGroup loner = sim.createGroup(1); - PeerGroup const friends = sim.createGroup(3); - loner.trust(loner + friends); - - PeerGroup const others = sim.createGroup(6); - PeerGroup clique = friends + others; - clique.trust(clique); - - PeerGroup network = loner + clique; - network.connect(network, round(0.2 * parms.ledgerGRANULARITY)); - - // initial round to set prior state - sim.run(1); - for (Peer* peer : (loner + friends)) - peer->openTxs.insert(Tx(0)); - for (Peer* peer : others) - peer->openTxs.insert(Tx(1)); - - // Delay validation processing - for (Peer* peer : network) - peer->delays.recvValidation = parms.ledgerGRANULARITY; - - // additional rounds to generate wrongLCL and recover - sim.run(2); - - // Check all peers recovered - for (Peer const* p : network) - BEAST_EXPECT(p->prevLedgerID() == network[0]->prevLedgerID()); - } - } - - void - testConsensusCloseTimeRounding() - { - using namespace csf; - using namespace std::chrono; - testcase("consensus close time rounding"); - - // This is a specialized test engineered to yield ledgers with different - // close times even though the peers believe they had close time - // consensus on the ledger. - ConsensusParms const parms; - - Sim sim; - - // This requires a group of 4 fast and 2 slow peers to create a - // situation in which a subset of peers requires seeing additional - // proposals to declare consensus. - PeerGroup slow = sim.createGroup(2); - PeerGroup fast = sim.createGroup(4); - PeerGroup network = fast + slow; - - // Connected trust graph - network.trust(network); - - // Fast and slow network connections - fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); - slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); - - // Run to the ledger *prior* to decreasing the resolution - sim.run(kIncreaseLedgerTimeResolutionEvery - 2); - - // In order to create the discrepancy, we want a case where if - // X = effCloseTime(closeTime, resolution, parentCloseTime) - // X != effCloseTime(X, resolution, parentCloseTime) - // - // That is, the effective close time is not a fixed point. This can - // happen if X = parentCloseTime + 1, but a subsequent rounding goes - // to the next highest multiple of resolution. - - // So we want to find an offset (now + offset) % 30s = 15 - // (now + offset) % 20s = 15 - // This way, the next ledger will close and round up Due to the - // network delay settings, the round of consensus will take 5s, so - // the next ledger's close time will - - NetClock::duration when = network[0]->now().time_since_epoch(); - - // Check we are before the 30s to 20s transition - NetClock::duration const resolution = network[0]->lastClosedLedger.closeTimeResolution(); - BEAST_EXPECT(resolution == NetClock::duration{30s}); - - while (((when % NetClock::duration{30s}) != NetClock::duration{15s}) || - ((when % NetClock::duration{20s}) != NetClock::duration{15s})) - when += 1s; - // Advance the clock without consensus running (IS THIS WHAT - // PREVENTS IT IN PRACTICE?) - sim.scheduler.stepFor(NetClock::time_point{when} - network[0]->now()); - - // Run one more ledger with 30s resolution - sim.run(1); - if (BEAST_EXPECT(sim.synchronized())) - { - // close time should be ahead of clock time since we engineered - // the close time to round up - for (Peer const* peer : network) - { - BEAST_EXPECT(peer->lastClosedLedger.closeTime() > peer->now()); - BEAST_EXPECT(peer->lastClosedLedger.closeAgree()); - } - } - - // All peers submit their own ID as a transaction - for (Peer* peer : network) - peer->submit(Tx{static_cast(peer->id)}); - - // Run 1 more round, this time it will have a decreased - // resolution of 20 seconds. - - // The network delays are engineered so that the slow peers - // initially have the wrong tx hash, but they see a majority - // of agreement from their peers and declare consensus - // - // The trick is that everyone starts with a raw close time of - // 84681s - // Which has - // effCloseTime(86481s, 20s, 86490s) = 86491s - // However, when the slow peers update their position, they change - // the close time to 86451s. The fast peers declare consensus with - // the 86481s as their position still. - // - // When accepted the ledger - // - fast peers use eff(86481s) -> 86491s as the close time - // - slow peers use eff(eff(86481s)) -> eff(86491s) -> 86500s! - - sim.run(1); - - BEAST_EXPECT(sim.synchronized()); - } - - void - testFork() - { - using namespace csf; - using namespace std::chrono; - testcase("fork"); - - std::uint32_t const numPeers = 10; - // Vary overlap between two UNLs - for (std::uint32_t overlap = 0; overlap <= numPeers; ++overlap) - { - ConsensusParms const parms{}; - Sim sim; - - std::uint32_t const numA = (numPeers - overlap) / 2; - std::uint32_t const numB = numPeers - numA - overlap; - - PeerGroup const aOnly = sim.createGroup(numA); - PeerGroup const bOnly = sim.createGroup(numB); - PeerGroup const commonOnly = sim.createGroup(overlap); - - PeerGroup a = aOnly + commonOnly; - PeerGroup b = bOnly + commonOnly; - - PeerGroup const network = a + b; - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - a.trustAndConnect(a, delay); - b.trustAndConnect(b, delay); - - // Initial round to set prior state - sim.run(1); - for (Peer* peer : network) - { - // Nodes have only seen transactions from their neighbors - peer->openTxs.insert(Tx{static_cast(peer->id)}); - for (Peer const* to : sim.trustGraph.trustedPeers(peer)) - peer->openTxs.insert(Tx{static_cast(to->id)}); - } - sim.run(1); - - // Fork should not happen for 40% or greater overlap - // Since the overlapped nodes have a UNL that is the union of the - // two cliques, the maximum sized UNL list is the number of peers - if (overlap > 0.4 * numPeers) - { - BEAST_EXPECT(sim.synchronized()); - } - else - { - // Even if we do fork, there shouldn't be more than 3 ledgers - // One for cliqueA, one for cliqueB and one for nodes in both - BEAST_EXPECT(sim.branches() <= 3); - } - } - } - - void - testHubNetwork() - { - using namespace csf; - using namespace std::chrono; - testcase("hub network"); - - // Simulate a set of 5 validators that aren't directly connected but - // rely on a single hub node for communication - - ConsensusParms const parms{}; - Sim sim; - PeerGroup validators = sim.createGroup(5); - PeerGroup center = sim.createGroup(1); - validators.trust(validators); - center.trust(validators); - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - validators.connect(center, delay); - - center[0]->runAsValidator = false; - - // prep round to set initial state. - sim.run(1); - - // everyone submits their own ID as a TX and relay it to peers - for (Peer* p : validators) - p->submit(Tx(static_cast(p->id))); - - sim.run(1); - - // All peers are in sync - BEAST_EXPECT(sim.synchronized()); - } - - // Helper collector for testPreferredByBranch - // Invasively disconnects network at bad times to cause splits - struct Disruptor - { - csf::PeerGroup& network; - csf::PeerGroup& groupCfast; - csf::PeerGroup& groupCsplit; - csf::SimDuration delay; - bool reconnected = false; - - Disruptor(csf::PeerGroup& net, csf::PeerGroup& c, csf::PeerGroup& split, csf::SimDuration d) - : network(net), groupCfast(c), groupCsplit(split), delay(d) - { - } - - template - void - on(csf::PeerID, csf::SimTime, E const&) - { - } - - void - on(csf::PeerID who, csf::SimTime, csf::FullyValidateLedger const& e) - { - using namespace std::chrono; - // As soon as the fastC node fully validates C, disconnect - // ALL c nodes from the network. The fast C node needs to disconnect - // as well to prevent it from relaying the validations it did see - if (who == groupCfast[0]->id && e.ledger.seq() == csf::Ledger::Seq{2}) - { - network.disconnect(groupCsplit); - network.disconnect(groupCfast); - } - } - - void - on(csf::PeerID who, csf::SimTime, csf::AcceptLedger const& e) - { - // As soon as anyone generates a child of B or C, reconnect the - // network so those validations make it through - if (!reconnected && e.ledger.seq() == csf::Ledger::Seq{3}) - { - reconnected = true; - network.connect(groupCsplit, delay); - } - } - }; - - void - testPreferredByBranch() - { - using namespace csf; - using namespace std::chrono; - testcase("preferred by branch"); - - // Simulate network splits that are prevented from forking when using - // preferred ledger by trie. This is a contrived example that involves - // excessive network splits, but demonstrates the safety improvement - // from the preferred ledger by trie approach. - - // Consider 10 validating nodes that comprise a single common UNL - // Ledger history: - // 1: A - // _/ \_ - // 2: B C - // _/ _/ \_ - // 3: D C' |||||||| (8 different ledgers) - - // - All nodes generate the common ledger A - // - 2 nodes generate B and 8 nodes generate C - // - Only 1 of the C nodes sees all the C validations and fully - // validates C. The rest of the C nodes split at just the right time - // such that they never see any C validations but their own. - // - The C nodes continue and generate 8 different child ledgers. - // - Meanwhile, the D nodes only saw 1 validation for C and 2 - // validations - // for B. - // - The network reconnects and the validations for generation 3 ledgers - // are observed (D and the 8 C's) - // - In the old approach, 2 votes for D outweighs 1 vote for each C' - // so the network would avalanche towards D and fully validate it - // EVEN though C was fully validated by one node - // - In the new approach, 2 votes for D are not enough to outweight the - // 8 implicit votes for C, so nodes will avalanche to C instead - - ConsensusParms const parms{}; - Sim sim; - - // Goes A->B->D - PeerGroup const groupABD = sim.createGroup(2); - // Single node that initially fully validates C before the split - PeerGroup groupCfast = sim.createGroup(1); - // Generates C, but fails to fully validate before the split - PeerGroup groupCsplit = sim.createGroup(7); - - PeerGroup groupNotFastC = groupABD + groupCsplit; - PeerGroup network = groupABD + groupCsplit + groupCfast; - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - SimDuration const fDelay = round(0.1 * parms.ledgerGRANULARITY); - - network.trust(network); - // C must have a shorter delay to see all the validations before the - // other nodes - network.connect(groupCfast, fDelay); - // The rest of the network is connected at the same speed - groupNotFastC.connect(groupNotFastC, delay); - - Disruptor dc(network, groupCfast, groupCsplit, delay); - sim.collectors.add(dc); - - // Consensus round to generate ledger A - sim.run(1); - BEAST_EXPECT(sim.synchronized()); - - // Next round generates B and C - // To force B, we inject an extra transaction in to those nodes - for (Peer* peer : groupABD) - { - peer->txInjections.emplace(peer->lastClosedLedger.seq(), Tx{42}); - } - // The Disruptor will ensure that nodes disconnect before the C - // validations make it to all but the fastC node - sim.run(1); - - // We are no longer in sync, but have not yet forked: - // 9 nodes consider A the last fully validated ledger and fastC sees C - BEAST_EXPECT(!sim.synchronized()); - BEAST_EXPECT(sim.branches() == 1); - - // Run another round to generate the 8 different C' ledgers - for (Peer* p : network) - p->submit(Tx(static_cast(p->id))); - sim.run(1); - - // Still not forked - BEAST_EXPECT(!sim.synchronized()); - BEAST_EXPECT(sim.branches() == 1); - - // Disruptor will reconnect all but the fastC node - sim.run(1); - - if (BEAST_EXPECT(sim.branches() == 1)) - { - BEAST_EXPECT(sim.synchronized()); - } - else // old approach caused a fork - { - BEAST_EXPECT(sim.branches(groupNotFastC) == 1); - BEAST_EXPECT(sim.synchronized(groupNotFastC) == 1); - } - } - - // Helper collector for testPauseForLaggards - // This will remove the ledgerAccept delay used to - // initially create the slow vs. fast validator groups. - struct UndoDelay - { - csf::PeerGroup& g; - - UndoDelay(csf::PeerGroup& a) : g(a) - { - } - - template - void - on(csf::PeerID, csf::SimTime, E const&) - { - } - - void - on(csf::PeerID who, csf::SimTime, csf::AcceptLedger const& e) - { - for (csf::Peer* p : g) - { - if (p->id == who) - p->delays.ledgerAccept = std::chrono::seconds{0}; - } - } - }; - - void - testPauseForLaggards() - { - using namespace csf; - using namespace std::chrono; - testcase("pause for laggards"); - - // Test that validators that jump ahead of the network slow - // down. - - // We engineer the following validated ledger history scenario: - // - // / --> B1 --> C1 --> ... -> G1 "ahead" - // A - // \ --> B2 --> C2 "behind" - // - // After validating a common ledger A, a set of "behind" validators - // briefly run slower and validate the lower chain of ledgers. - // The "ahead" validators run normal speed and run ahead validating the - // upper chain of ledgers. - // - // Due to the uncommitted support definition of the preferred branch - // protocol, even if the "behind" validators are a majority, the "ahead" - // validators cannot jump to the proper branch until the "behind" - // validators catch up to the same sequence number. For this test to - // succeed, the ahead validators need to briefly slow down consensus. - - ConsensusParms const parms{}; - Sim sim; - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - - PeerGroup behind = sim.createGroup(3); - PeerGroup const ahead = sim.createGroup(2); - PeerGroup network = ahead + behind; - - hash_set trustedKeys; - for (Peer const* p : network) - trustedKeys.insert(p->key); - for (Peer* p : network) - p->trustedKeys = trustedKeys; - - network.trustAndConnect(network, delay); - - // Initial seed round to set prior state - sim.run(1); - - // Have the "behind" group initially take a really long time to - // accept a ledger after ending deliberation - for (Peer* p : behind) - p->delays.ledgerAccept = 20s; - - // Use the collector to revert the delay after the single - // slow ledger is generated - UndoDelay undoDelay{behind}; - sim.collectors.add(undoDelay); - // Run the simulation for 100 seconds of simulation time with - std::chrono::nanoseconds const simDuration = 100s; - - // Simulate clients submitting 1 tx every 5 seconds to a random - // validator - Rate const rate{.count = 1, .duration = 5s}; - auto peerSelector = makeSelector( - network.begin(), network.end(), std::vector(network.size(), 1.), sim.rng); - auto txSubmitter = makeSubmitter( - ConstantDistribution{rate.inv()}, - sim.scheduler.now(), - sim.scheduler.now() + simDuration, - peerSelector, - sim.scheduler, - sim.rng); - - // Run simulation - sim.run(simDuration); - - // Verify that the network recovered - BEAST_EXPECT(sim.synchronized()); - } - - void - testDisputes() - { - testcase("disputes"); - - using namespace csf; - - // Test dispute objects directly - using Dispute = DisputedTx; - - Tx const txTrue{99}; - Tx const txFalse{98}; - Tx const txFollowingTrue{97}; - Tx const txFollowingFalse{96}; - int const numPeers = 100; - ConsensusParms const p; - std::size_t peersUnchanged = 0; - - auto logs = std::make_unique(beast::Severity::Error); - auto j = logs->journal("Test"); - auto clog = std::make_unique(); - - // Three cases: - // 1 proposing, initial vote yes - // 2 proposing, initial vote no - // 3 not proposing, initial vote doesn't matter after the first update, - // use yes - { - Dispute proposingTrue{txTrue.id(), true, numPeers, journal_}; - Dispute proposingFalse{txFalse.id(), false, numPeers, journal_}; - Dispute followingTrue{txFollowingTrue.id(), true, numPeers, journal_}; - Dispute followingFalse{txFollowingFalse.id(), false, numPeers, journal_}; - BEAST_EXPECT(proposingTrue.id() == 99); - BEAST_EXPECT(proposingFalse.id() == 98); - BEAST_EXPECT(followingTrue.id() == 97); - BEAST_EXPECT(followingFalse.id() == 96); - - // Create an even split in the peer votes - for (int i = 0; i < numPeers; ++i) - { - BEAST_EXPECT(proposingTrue.setVote(PeerID(i), i < 50)); - BEAST_EXPECT(proposingFalse.setVote(PeerID(i), i < 50)); - BEAST_EXPECT(followingTrue.setVote(PeerID(i), i < 50)); - BEAST_EXPECT(followingFalse.setVote(PeerID(i), i < 50)); - } - // Switch the middle vote to match mine - BEAST_EXPECT(proposingTrue.setVote(PeerID(50), true)); - BEAST_EXPECT(proposingFalse.setVote(PeerID(49), false)); - BEAST_EXPECT(followingTrue.setVote(PeerID(50), true)); - BEAST_EXPECT(followingFalse.setVote(PeerID(49), false)); - - // no changes yet - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(clog->str().empty()); - - // I'm in the majority, my vote should not change - BEAST_EXPECT(!proposingTrue.updateVote(5, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(5, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(5, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(5, false, p)); - - BEAST_EXPECT(!proposingTrue.updateVote(10, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(10, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(10, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(10, false, p)); - - peersUnchanged = 2; - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(clog->str().empty()); - - // Right now, the vote is 51%. The requirement is about to jump to - // 65% - BEAST_EXPECT(proposingTrue.updateVote(55, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(55, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(55, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(55, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == false); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - // 16 validators change their vote to match my original vote - for (int i = 0; i < 16; ++i) - { - auto pTrue = PeerID(numPeers - i - 1); - auto pFalse = PeerID(i); - BEAST_EXPECT(proposingTrue.setVote(pTrue, true)); - BEAST_EXPECT(proposingFalse.setVote(pFalse, false)); - BEAST_EXPECT(followingTrue.setVote(pTrue, true)); - BEAST_EXPECT(followingFalse.setVote(pFalse, false)); - } - // The vote should now be 66%, threshold is 65% - BEAST_EXPECT(proposingTrue.updateVote(60, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(60, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(60, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(60, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // Threshold jumps to 70% - BEAST_EXPECT(proposingTrue.updateVote(86, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(86, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(86, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(86, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == false); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // 5 more validators change their vote to match my original vote - for (int i = 16; i < 21; ++i) - { - auto pTrue = PeerID(numPeers - i - 1); - auto pFalse = PeerID(i); - BEAST_EXPECT(proposingTrue.setVote(pTrue, true)); - BEAST_EXPECT(proposingFalse.setVote(pFalse, false)); - BEAST_EXPECT(followingTrue.setVote(pTrue, true)); - BEAST_EXPECT(followingFalse.setVote(pFalse, false)); - } - - // The vote should now be 71%, threshold is 70% - BEAST_EXPECT(proposingTrue.updateVote(90, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(90, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(90, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(90, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // The vote should now be 71%, threshold is 70% - BEAST_EXPECT(!proposingTrue.updateVote(150, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(150, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(150, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(150, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // The vote should now be 71%, threshold is 70% - BEAST_EXPECT(!proposingTrue.updateVote(190, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(190, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(190, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(190, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - peersUnchanged = 3; - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(clog->str().empty()); - - // Threshold jumps to 95% - BEAST_EXPECT(proposingTrue.updateVote(220, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(220, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(220, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(220, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == false); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // 25 more validators change their vote to match my original vote - for (int i = 21; i < 46; ++i) - { - auto pTrue = PeerID(numPeers - i - 1); - auto pFalse = PeerID(i); - BEAST_EXPECT(proposingTrue.setVote(pTrue, true)); - BEAST_EXPECT(proposingFalse.setVote(pFalse, false)); - BEAST_EXPECT(followingTrue.setVote(pTrue, true)); - BEAST_EXPECT(followingFalse.setVote(pFalse, false)); - } - - // The vote should now be 96%, threshold is 95% - BEAST_EXPECT(proposingTrue.updateVote(250, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(250, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(250, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(250, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - for (peersUnchanged = 0; peersUnchanged < 6; ++peersUnchanged) - { - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(clog->str().empty()); - } - - auto expectStalled = [this, &clog]( - int txid, - bool ourVote, - int ourTime, - int peerTime, - int support, - std::uint32_t line) { - using namespace std::string_literals; - - auto const s = clog->str(); - expect(s.find("stalled"), s, __FILE__, line); - expect(s.starts_with("Transaction "s + std::to_string(txid)), s, __FILE__, line); - expect(s.contains("voting "s + (ourVote ? "YES" : "NO")), s, __FILE__, line); - expect( - s.contains("for "s + std::to_string(ourTime) + " rounds."s), s, __FILE__, line); - expect( - s.contains("votes in "s + std::to_string(peerTime) + " rounds."), - s, - __FILE__, - line); - expect( - s.ends_with("has "s + std::to_string(support) + "% support. "s), - s, - __FILE__, - line); - clog = std::make_unique(); - }; - - for (int i = 0; i < 1; ++i) - { - BEAST_EXPECT(!proposingTrue.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!followingTrue.updateVote(250 + (10 * i), false, p)); - BEAST_EXPECT(!followingFalse.updateVote(250 + (10 * i), false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // true vote has changed recently, so not stalled - BEAST_EXPECT(!proposingTrue.stalled(p, true, 0, j, clog)); - BEAST_EXPECT(clog->str().empty()); - // remaining votes have been unchanged in so long that we only - // need to hit the second round at 95% to be stalled, regardless - // of peers - BEAST_EXPECT(proposingFalse.stalled(p, true, 0, j, clog)); - expectStalled(98, false, 11, 0, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, 0, j, clog)); - expectStalled(97, true, 11, 0, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, 0, j, clog)); - expectStalled(96, false, 11, 0, 3, __LINE__); - - // true vote has changed recently, so not stalled - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECTS(clog->str().empty(), clog->str()); - // remaining votes have been unchanged in so long that we only - // need to hit the second round at 95% to be stalled, regardless - // of peers - BEAST_EXPECT(proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(98, false, 11, 6, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(97, true, 11, 6, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(96, false, 11, 6, 3, __LINE__); - } - for (int i = 1; i < 3; ++i) - { - BEAST_EXPECT(!proposingTrue.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!followingTrue.updateVote(250 + (10 * i), false, p)); - BEAST_EXPECT(!followingFalse.updateVote(250 + (10 * i), false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // true vote changed 2 rounds ago, and peers are changing, so - // not stalled - BEAST_EXPECT(!proposingTrue.stalled(p, true, 0, j, clog)); - BEAST_EXPECTS(clog->str().empty(), clog->str()); - // still stalled - BEAST_EXPECT(proposingFalse.stalled(p, true, 0, j, clog)); - expectStalled(98, false, 11 + i, 0, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, 0, j, clog)); - expectStalled(97, true, 11 + i, 0, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, 0, j, clog)); - expectStalled(96, false, 11 + i, 0, 3, __LINE__); - - // true vote changed 2 rounds ago, and peers are NOT changing, - // so stalled - BEAST_EXPECT(proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(99, true, 1 + i, 6, 97, __LINE__); - // still stalled - BEAST_EXPECT(proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(98, false, 11 + i, 6, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(97, true, 11 + i, 6, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(96, false, 11 + i, 6, 3, __LINE__); - } - for (int i = 3; i < 5; ++i) - { - BEAST_EXPECT(!proposingTrue.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!followingTrue.updateVote(250 + (10 * i), false, p)); - BEAST_EXPECT(!followingFalse.updateVote(250 + (10 * i), false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - BEAST_EXPECT(proposingTrue.stalled(p, true, 0, j, clog)); - expectStalled(99, true, 1 + i, 0, 97, __LINE__); - BEAST_EXPECT(proposingFalse.stalled(p, true, 0, j, clog)); - expectStalled(98, false, 11 + i, 0, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, 0, j, clog)); - expectStalled(97, true, 11 + i, 0, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, 0, j, clog)); - expectStalled(96, false, 11 + i, 0, 3, __LINE__); - - BEAST_EXPECT(proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(99, true, 1 + i, 6, 97, __LINE__); - BEAST_EXPECT(proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(98, false, 11 + i, 6, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(97, true, 11 + i, 6, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(96, false, 11 + i, 6, 3, __LINE__); - } - } - } - - void - run() override - { - testShouldCloseLedger(); - testCheckConsensus(); - - testStandalone(); - testPeersAgree(); - testSlowPeers(); - testCloseTimeDisagree(); - testWrongLCL(); - testConsensusCloseTimeRounding(); - testFork(); - testHubNetwork(); - testPreferredByBranch(); - testPauseForLaggards(); - testDisputes(); - } -}; - -BEAST_DEFINE_TESTSUITE(Consensus, consensus, xrpl); -} // namespace xrpl::test diff --git a/src/test/consensus/DistributedValidatorsSim_test.cpp b/src/test/consensus/DistributedValidatorsSim_test.cpp deleted file mode 100644 index 437ad81ee0..0000000000 --- a/src/test/consensus/DistributedValidatorsSim_test.cpp +++ /dev/null @@ -1,253 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace xrpl::test { - -/** - * In progress simulations for diversifying and distributing validators - */ -class DistributedValidators_test : public beast::unit_test::Suite -{ - void - completeTrustCompleteConnectFixedDelay( - std::size_t numPeers, - std::chrono::milliseconds delay = std::chrono::milliseconds(200), - bool printHeaders = false) - { - using namespace csf; - using namespace std::chrono; - - // Initialize persistent collector logs specific to this method - std::string const prefix = - "DistributedValidators_" - "completeTrustCompleteConnectFixedDelay"; - std::fstream txLog(prefix + "_tx.csv", std::ofstream::app), - ledgerLog(prefix + "_ledger.csv", std::ofstream::app); - - // title - log << prefix << "(" << numPeers << "," << delay.count() << ")" << std::endl; - - // number of peers, UNLs, connections - BEAST_EXPECT(numPeers >= 1); - - Sim sim; - PeerGroup peers = sim.createGroup(numPeers); - - // complete trust graph - peers.trust(peers); - - // complete connect graph with fixed delay - peers.connect(peers, delay); - - // Initialize collectors to track statistics to report - TxCollector txCollector; - LedgerCollector ledgerCollector; - auto colls = makeCollectors(txCollector, ledgerCollector); - sim.collectors.add(colls); - - // Initial round to set prior state - sim.run(1); - - // Run for 10 minutes, submitting 100 tx/second - std::chrono::nanoseconds const simDuration = 10min; - std::chrono::nanoseconds const quiet = 10s; - Rate const rate{.count = 100, .duration = 1000ms}; - - // Initialize timers - HeartbeatTimer heart(sim.scheduler); - - // txs, start/stop/step, target - auto peerSelector = - makeSelector(peers.begin(), peers.end(), std::vector(numPeers, 1.), sim.rng); - auto txSubmitter = makeSubmitter( - ConstantDistribution{rate.inv()}, - sim.scheduler.now() + quiet, - sim.scheduler.now() + simDuration - quiet, - peerSelector, - sim.scheduler, - sim.rng); - - // run simulation for given duration - heart.start(); - sim.run(simDuration); - - // BEAST_EXPECT(sim.branches() == 1); - // BEAST_EXPECT(sim.synchronized()); - - log << std::right; - log << "| Peers: " << std::setw(2) << peers.size(); - log << " | Duration: " << std::setw(6) << duration_cast(simDuration).count() - << " ms"; - log << " | Branches: " << std::setw(1) << sim.branches(); - log << " | Synchronized: " << std::setw(1) << (sim.synchronized() ? "Y" : "N"); - log << " |" << std::endl; - - txCollector.report(simDuration, log, true); - ledgerCollector.report(simDuration, log, false); - - std::string const tag = std::to_string(numPeers); - txCollector.csv(simDuration, txLog, tag, printHeaders); - ledgerCollector.csv(simDuration, ledgerLog, tag, printHeaders); - - log << std::endl; - } - - void - completeTrustScaleFreeConnectFixedDelay( - std::size_t numPeers, - std::chrono::milliseconds delay = std::chrono::milliseconds(200), - bool printHeaders = false) - { - using namespace csf; - using namespace std::chrono; - - // Initialize persistent collector logs specific to this method - std::string const prefix = - "DistributedValidators__" - "completeTrustScaleFreeConnectFixedDelay"; - std::fstream txLog(prefix + "_tx.csv", std::ofstream::app), - ledgerLog(prefix + "_ledger.csv", std::ofstream::app); - - // title - log << prefix << "(" << numPeers << "," << delay.count() << ")" << std::endl; - - // number of peers, UNLs, connections - int const numCNLs = std::max(int(1.00 * numPeers), 1); - int const minCNLSize = std::max(int(0.25 * numCNLs), 1); - int const maxCNLSize = std::max(int(0.50 * numCNLs), 1); - BEAST_EXPECT(numPeers >= 1); - BEAST_EXPECT(numCNLs >= 1); - BEAST_EXPECT(1 <= minCNLSize && minCNLSize <= maxCNLSize && maxCNLSize <= numPeers); - - Sim sim; - PeerGroup peers = sim.createGroup(numPeers); - - // complete trust graph - peers.trust(peers); - - // scale-free connect graph with fixed delay - std::vector const ranks = sample(peers.size(), PowerLawDistribution{1, 3}, sim.rng); - randomRankedConnect( - peers, - ranks, - numCNLs, - std::uniform_int_distribution<>{minCNLSize, maxCNLSize}, - sim.rng, - delay); - - // Initialize collectors to track statistics to report - TxCollector txCollector; - LedgerCollector ledgerCollector; - auto colls = makeCollectors(txCollector, ledgerCollector); - sim.collectors.add(colls); - - // Initial round to set prior state - sim.run(1); - - // Run for 10 minutes, submitting 100 tx/second - std::chrono::nanoseconds const simDuration = 10min; - std::chrono::nanoseconds const quiet = 10s; - Rate const rate{.count = 100, .duration = 1000ms}; - - // Initialize timers - HeartbeatTimer heart(sim.scheduler); - - // txs, start/stop/step, target - auto peerSelector = - makeSelector(peers.begin(), peers.end(), std::vector(numPeers, 1.), sim.rng); - auto txSubmitter = makeSubmitter( - ConstantDistribution{rate.inv()}, - sim.scheduler.now() + quiet, - sim.scheduler.now() + simDuration - quiet, - peerSelector, - sim.scheduler, - sim.rng); - - // run simulation for given duration - heart.start(); - sim.run(simDuration); - - // BEAST_EXPECT(sim.branches() == 1); - // BEAST_EXPECT(sim.synchronized()); - - log << std::right; - log << "| Peers: " << std::setw(2) << peers.size(); - log << " | Duration: " << std::setw(6) << duration_cast(simDuration).count() - << " ms"; - log << " | Branches: " << std::setw(1) << sim.branches(); - log << " | Synchronized: " << std::setw(1) << (sim.synchronized() ? "Y" : "N"); - log << " |" << std::endl; - - txCollector.report(simDuration, log, true); - ledgerCollector.report(simDuration, log, false); - - std::string const tag = std::to_string(numPeers); - txCollector.csv(simDuration, txLog, tag, printHeaders); - ledgerCollector.csv(simDuration, ledgerLog, tag, printHeaders); - - log << std::endl; - } - - void - run() override - { - std::string const defaultArgs = "5 200"; - std::string const args = arg().empty() ? defaultArgs : arg(); - std::stringstream argStream(args); - - int maxNumValidators = 0; - int delayCount(200); - argStream >> maxNumValidators; - argStream >> delayCount; - - std::chrono::milliseconds const delay(delayCount); - - log << "DistributedValidators: 1 to " << maxNumValidators << " Peers" << std::endl; - - /** - * Simulate with N = 1 to N - * - complete trust graph is complete - * - complete network connectivity - * - fixed delay for network links - */ - completeTrustCompleteConnectFixedDelay(1, delay, true); - for (int i = 2; i <= maxNumValidators; i++) - { - completeTrustCompleteConnectFixedDelay(i, delay); - } - - /** - * Simulate with N = 1 to N - * - complete trust graph is complete - * - scale-free network connectivity - * - fixed delay for network links - */ - completeTrustScaleFreeConnectFixedDelay(1, delay, true); - for (int i = 2; i <= maxNumValidators; i++) - { - completeTrustScaleFreeConnectFixedDelay(i, delay); - } - } -}; - -BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(DistributedValidators, consensus, xrpl, 2); - -} // namespace xrpl::test diff --git a/src/test/consensus/LedgerTiming_test.cpp b/src/test/consensus/LedgerTiming_test.cpp deleted file mode 100644 index 632361c799..0000000000 --- a/src/test/consensus/LedgerTiming_test.cpp +++ /dev/null @@ -1,118 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -namespace xrpl::test { - -class LedgerTiming_test : public beast::unit_test::Suite -{ - void - testGetNextLedgerTimeResolution() - { - // helper to iteratively call into getNextLedgerTimeResolution - struct TestRes - { - std::uint32_t decrease = 0; - std::uint32_t equal = 0; - std::uint32_t increase = 0; - - static TestRes - run(bool previousAgree, std::uint32_t rounds) - { - TestRes res; - auto closeResolution = kLedgerDefaultTimeResolution; - auto nextCloseResolution = closeResolution; - std::uint32_t round = 0; - do - { - nextCloseResolution = - getNextLedgerTimeResolution(closeResolution, previousAgree, ++round); - if (nextCloseResolution < closeResolution) - { - ++res.decrease; - } - else if (nextCloseResolution > closeResolution) - { - ++res.increase; - } - else - { - ++res.equal; - } - std::swap(nextCloseResolution, closeResolution); - } while (round < rounds); - return res; - } - }; - - // If we never agree on close time, only can increase resolution - // until hit the max - auto decreases = TestRes::run(false, 10); - BEAST_EXPECT(decreases.increase == 3); - BEAST_EXPECT(decreases.decrease == 0); - BEAST_EXPECT(decreases.equal == 7); - - // If we always agree on close time, only can decrease resolution - // until hit the min - auto increases = TestRes::run(false, 100); - BEAST_EXPECT(increases.increase == 3); - BEAST_EXPECT(increases.decrease == 0); - BEAST_EXPECT(increases.equal == 97); - } - - void - testRoundCloseTime() - { - using namespace std::chrono_literals; - // A closeTime equal to the epoch is not modified - using tp = NetClock::time_point; - tp const def; - BEAST_EXPECT(def == roundCloseTime(def, 30s)); - - // Otherwise, the closeTime is rounded to the nearest - // rounding up on ties - BEAST_EXPECT(tp{0s} == roundCloseTime(tp{29s}, 60s)); - BEAST_EXPECT(tp{30s} == roundCloseTime(tp{30s}, 1s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{31s}, 60s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{30s}, 60s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{59s}, 60s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{60s}, 60s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{61s}, 60s)); - } - - void - testEffCloseTime() - { - using namespace std::chrono_literals; - using tp = NetClock::time_point; - tp close = effCloseTime(tp{10s}, 30s, tp{0s}); - BEAST_EXPECT(close == tp{1s}); - - close = effCloseTime(tp{16s}, 30s, tp{0s}); - BEAST_EXPECT(close == tp{30s}); - - close = effCloseTime(tp{16s}, 30s, tp{30s}); - BEAST_EXPECT(close == tp{31s}); - - close = effCloseTime(tp{16s}, 30s, tp{60s}); - BEAST_EXPECT(close == tp{61s}); - - close = effCloseTime(tp{31s}, 30s, tp{0s}); - BEAST_EXPECT(close == tp{30s}); - } - - void - run() override - { - testGetNextLedgerTimeResolution(); - testRoundCloseTime(); - testEffCloseTime(); - } -}; - -BEAST_DEFINE_TESTSUITE(LedgerTiming, consensus, xrpl); -} // namespace xrpl::test diff --git a/src/test/consensus/LedgerTrie_test.cpp b/src/test/consensus/LedgerTrie_test.cpp deleted file mode 100644 index a4eb7bc087..0000000000 --- a/src/test/consensus/LedgerTrie_test.cpp +++ /dev/null @@ -1,716 +0,0 @@ -#include - -#include - -#include - -#include -#include -#include - -namespace xrpl::test { - -class LedgerTrie_test : public beast::unit_test::Suite -{ - void - testInsert() - { - using namespace csf; - // Single entry by itself - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - - t.insert(h["abc"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - } - // Suffix of existing (extending tree) - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - BEAST_EXPECT(t.checkInvariants()); - // extend with no siblings - t.insert(h["abcd"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - - // extend with existing sibling - t.insert(h["abce"]); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abce"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abce"]) == 1); - } - // uncommitted of existing node - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"]); - BEAST_EXPECT(t.checkInvariants()); - // uncommitted with no siblings - t.insert(h["abcdf"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcdf"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcdf"]) == 1); - - // uncommitted with existing child - t.insert(h["abc"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcdf"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcdf"]) == 1); - } - // Suffix + uncommitted of existing node - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"]); - BEAST_EXPECT(t.checkInvariants()); - t.insert(h["abce"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abce"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abce"]) == 1); - } - // Suffix + uncommitted with existing child - { - // abcd : abcde, abcf - - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"]); - BEAST_EXPECT(t.checkInvariants()); - t.insert(h["abcde"]); - BEAST_EXPECT(t.checkInvariants()); - t.insert(h["abcf"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcf"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcf"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abcde"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcde"]) == 1); - } - - // Multiple counts - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"], 4); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 4); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 4); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.branchSupport(h["a"]) == 4); - - t.insert(h["abc"], 2); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 4); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 6); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.branchSupport(h["a"]) == 6); - } - } - - void - testRemove() - { - using namespace csf; - // Not in trie - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - - BEAST_EXPECT(!t.remove(h["ab"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(!t.remove(h["a"])); - BEAST_EXPECT(t.checkInvariants()); - } - // In trie but with 0 tip support - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"]); - t.insert(h["abce"]); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(!t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - } - // In trie with > 1 tip support - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"], 2); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); - BEAST_EXPECT(t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - - t.insert(h["abc"], 1); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); - BEAST_EXPECT(t.remove(h["abc"], 2)); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - - t.insert(h["abc"], 3); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 3); - BEAST_EXPECT(t.remove(h["abc"], 300)); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - } - // In trie with = 1 tip support, no children - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["abc"]); - - BEAST_EXPECT(t.tipSupport(h["ab"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - - BEAST_EXPECT(t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 0); - } - // In trie with = 1 tip support, 1 child - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["abc"]); - t.insert(h["abcd"]); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - - BEAST_EXPECT(t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - } - // In trie with = 1 tip support, > 1 children - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["abc"]); - t.insert(h["abcd"]); - t.insert(h["abce"]); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); - - BEAST_EXPECT(t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - } - - // In trie with = 1 tip support, parent compaction - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["abc"]); - t.insert(h["abd"]); - BEAST_EXPECT(t.checkInvariants()); - t.remove(h["ab"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abd"]) == 1); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 2); - - t.remove(h["abd"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); - } - } - - void - testEmpty() - { - using namespace csf; - LedgerTrie t; - LedgerHistoryHelper h; - BEAST_EXPECT(t.empty()); - - Ledger const genesis = h[""]; - t.insert(genesis); - BEAST_EXPECT(!t.empty()); - t.remove(genesis); - BEAST_EXPECT(t.empty()); - - t.insert(h["abc"]); - BEAST_EXPECT(!t.empty()); - t.remove(h["abc"]); - BEAST_EXPECT(t.empty()); - } - - void - testSupport() - { - using namespace csf; - - LedgerTrie t; - LedgerHistoryHelper h; - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.tipSupport(h["axy"]) == 0); - - BEAST_EXPECT(t.branchSupport(h["a"]) == 0); - BEAST_EXPECT(t.branchSupport(h["axy"]) == 0); - - t.insert(h["abc"]); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 0); - - BEAST_EXPECT(t.branchSupport(h["a"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 0); - - t.insert(h["abe"]); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abe"]) == 1); - - BEAST_EXPECT(t.branchSupport(h["a"]) == 2); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 2); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abe"]) == 1); - - t.remove(h["abc"]); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.tipSupport(h["abe"]) == 1); - - BEAST_EXPECT(t.branchSupport(h["a"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abe"]) == 1); - } - - void - testGetPreferred() - { - using namespace csf; - using Seq = Ledger::Seq; - // Empty - { - LedgerTrie const t; - BEAST_EXPECT(t.getPreferred(Seq{0}) == std::nullopt); - BEAST_EXPECT(t.getPreferred(Seq{2}) == std::nullopt); - } - // Genesis support is NOT empty - { - LedgerTrie t; - LedgerHistoryHelper h; - Ledger const genesis = h[""]; - t.insert(genesis); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{0})->id == genesis.id()); - BEAST_EXPECT(t.remove(genesis)); - BEAST_EXPECT(t.getPreferred(Seq{0}) == std::nullopt); - BEAST_EXPECT(!t.remove(genesis)); - } - // Single node no children - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - } - // Single node smaller child support - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"]); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - } - // Single node larger child - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"], 2); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abcd"].id()); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcd"].id()); - } - // Single node smaller children support - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"]); - t.insert(h["abce"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - - t.insert(h["abc"]); - - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - // Single node larger children - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"], 2); - t.insert(h["abce"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - - t.insert(h["abcd"]); - - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abcd"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcd"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - // Tie-breaker by id - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"], 2); - t.insert(h["abce"], 2); - - BEAST_EXPECT(h["abce"].id() > h["abcd"].id()); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abce"].id()); - - t.insert(h["abcd"]); - BEAST_EXPECT(h["abce"].id() > h["abcd"].id()); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcd"].id()); - } - - // Tie-breaker not needed - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"]); - t.insert(h["abce"], 2); - // abce only has a margin of 1, but it owns the tie-breaker - BEAST_EXPECT(h["abce"].id() > h["abcd"].id()); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abce"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abce"].id()); - - // Switch support from abce to abcd, tie-breaker now needed - t.remove(h["abce"]); - t.insert(h["abcd"]); - - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - - // Single node larger grand child - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"], 2); - t.insert(h["abcde"], 4); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abcde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["abcde"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - - // Too much uncommitted support from competing branches - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcde"], 2); - t.insert(h["abcfg"], 2); - // 'de' and 'fg' are tied without 'abc' vote - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["abc"].id()); - - t.remove(h["abc"]); - t.insert(h["abcd"]); - - // 'de' branch has 3 votes to 2, so earlier sequences see it as preferred - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abcde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcde"].id()); - - // However, if you validated a ledger with Seq 5, potentially on - // a different branch, you do not yet know if they chose abcd - // or abcf because of you, so abc remains preferred - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["abc"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - - // Changing largestSeq perspective changes preferred branch - { - /** - * Build the tree below with initial tip support annotated - * A - * / \ - * B(1) C(1) - * / | | - * H D F(1) - * | - * E(2) - * | - * G - */ - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["ac"]); - t.insert(h["acf"]); - t.insert(h["abde"], 2); - - // B has more branch support - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{1})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{2})->id == h["ab"].id()); - - // But if you last validated D,F or E, you do not yet know - // if someone used that validation to commit to B or C - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["a"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["a"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - - /** - * One of E advancing to G doesn't change anything - * A - * / \ - * B(1) C(1) - * / | | - * H D F(1) - * | - * E(1) - * | - * G(1) - */ - t.remove(h["abde"]); - t.insert(h["abdeg"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{1})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{2})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["a"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["a"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["a"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - - /** - * C advancing to H does advance the seq 3 preferred ledger - * A - * / \ - * B(1) C - * / | | - * H(1)D F(1) - * | - * E(1) - * | - * G(1) - */ - t.remove(h["ac"]); - t.insert(h["abh"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{1})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{2})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["a"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["a"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - - /** - * F advancing to E also moves the preferred ledger forward - * A - * / \ - * B(1) C - * / | | - * H(1)D F - * | - * E(2) - * | - * G(1) - */ - t.remove(h["acf"]); - t.insert(h["abde"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{1})->id == h["abde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{2})->id == h["abde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["ab"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - } - - void - testRootRelated() - { - using namespace csf; - // Since the root is a special node that breaks the no-single child - // invariant, do some tests that exercise it. - - LedgerTrie t; - LedgerHistoryHelper h; - BEAST_EXPECT(!t.remove(h[""])); - BEAST_EXPECT(t.branchSupport(h[""]) == 0); - BEAST_EXPECT(t.tipSupport(h[""]) == 0); - - t.insert(h["a"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.branchSupport(h[""]) == 1); - BEAST_EXPECT(t.tipSupport(h[""]) == 0); - - t.insert(h["e"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.branchSupport(h[""]) == 2); - BEAST_EXPECT(t.tipSupport(h[""]) == 0); - - BEAST_EXPECT(t.remove(h["e"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.branchSupport(h[""]) == 1); - BEAST_EXPECT(t.tipSupport(h[""]) == 0); - } - - void - testStress() - { - using namespace csf; - LedgerTrie t; - LedgerHistoryHelper h; - - // Test quasi-randomly add/remove supporting for different ledgers - // from a branching history. - - // Ledgers have sequence 1,2,3,4 - std::uint32_t const depthConst = 4; - // Each ledger has 4 possible children - std::uint32_t const width = 4; - - std::uint32_t const iterations = 10000; - - // Use explicit seed to have same results for CI - // NOLINTNEXTLINE(bugprone-random-generator-seed): fixed seed for reproducible test - std::mt19937 gen{42}; - std::uniform_int_distribution<> depthDist(0, depthConst - 1); - std::uniform_int_distribution<> widthDist(0, width - 1); - std::uniform_int_distribution<> flip(0, 1); - for (std::uint32_t i = 0; i < iterations; ++i) - { - // pick a random ledger history - std::string curr; - char const depth = depthDist(gen); - char offset = 0; - for (char d = 0; d < depth; ++d) - { - char const a = offset + widthDist(gen); - curr += a; - offset = (a + 1) * width; - } - - // 50-50 to add remove - if (flip(gen) == 0) - { - t.insert(h[curr]); - } - else - { - t.remove(h[curr]); - } - if (!BEAST_EXPECT(t.checkInvariants())) - return; - } - } - - void - run() override - { - testInsert(); - testRemove(); - testEmpty(); - testSupport(); - testGetPreferred(); - testRootRelated(); - testStress(); - } -}; - -BEAST_DEFINE_TESTSUITE(LedgerTrie, consensus, xrpl); -} // namespace xrpl::test diff --git a/src/test/consensus/RCLCensorshipDetector_test.cpp b/src/test/consensus/RCLCensorshipDetector_test.cpp deleted file mode 100644 index 722a34f937..0000000000 --- a/src/test/consensus/RCLCensorshipDetector_test.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include - -#include - -#include -#include -#include - -namespace xrpl::test { - -class RCLCensorshipDetector_test : public beast::unit_test::Suite -{ - void - test( - RCLCensorshipDetector& cdet, - int round, - std::vector proposed, - std::vector accepted, - std::vector remain, - std::vector remove) - { - // Begin tracking what we're proposing this round - RCLCensorshipDetector::TxIDSeqVec proposal; - for (auto const& i : proposed) - proposal.emplace_back(i, round); - cdet.propose(std::move(proposal)); - - // Finalize the round, by processing what we accepted; then - // remove anything that needs to be removed and ensure that - // what remains is correct. - cdet.check(std::move(accepted), [&remove, &remain](auto id, auto seq) { - // If the item is supposed to be removed from the censorship - // detector internal tracker manually, do it now: - if (std::ranges::find(remove, id) != remove.end()) - return true; - - // If the item is supposed to still remain in the censorship - // detector internal tracker; remove it from the vector. - auto it = std::ranges::find(remain, id); - if (it != remain.end()) - remain.erase(it); - return false; - }); - - // On entry, this set contained all the elements that should be tracked - // by the detector after we process this round. We removed all the items - // that actually were in the tracker, so this should now be empty: - BEAST_EXPECT(remain.empty()); - } - -public: - void - run() override - { - testcase("Censorship Detector"); - - RCLCensorshipDetector cdet; - int round = 0; - // proposed accepted remain remove - test(cdet, ++round, {}, {}, {}, {}); - test(cdet, ++round, {10, 11, 12, 13}, {11, 2}, {10, 13}, {}); - test(cdet, ++round, {10, 13, 14, 15}, {14}, {10, 13, 15}, {}); - test(cdet, ++round, {10, 13, 15, 16}, {15, 16}, {10, 13}, {}); - test(cdet, ++round, {10, 13}, {17, 18}, {10, 13}, {}); - test(cdet, ++round, {10, 19}, {}, {10, 19}, {}); - test(cdet, ++round, {10, 19, 20}, {20}, {10}, {19}); - test(cdet, ++round, {21}, {21}, {}, {}); - test(cdet, ++round, {}, {22}, {}, {}); - test(cdet, ++round, {23, 24, 25, 26}, {25, 27}, {23, 26}, {24}); - test(cdet, ++round, {23, 26, 28}, {26, 28}, {23}, {}); - - for (int i = 0; i != 10; ++i) - test(cdet, ++round, {23}, {}, {23}, {}); - - test(cdet, ++round, {23, 29}, {29}, {23}, {}); - test(cdet, ++round, {30, 31}, {31}, {30}, {}); - test(cdet, ++round, {30}, {30}, {}, {}); - test(cdet, ++round, {}, {}, {}, {}); - } -}; - -BEAST_DEFINE_TESTSUITE(RCLCensorshipDetector, consensus, xrpl); -} // namespace xrpl::test diff --git a/src/test/consensus/ScaleFreeSim_test.cpp b/src/test/consensus/ScaleFreeSim_test.cpp deleted file mode 100644 index e533e09eb0..0000000000 --- a/src/test/consensus/ScaleFreeSim_test.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include - -namespace xrpl::test { - -class ScaleFreeSim_test : public beast::unit_test::Suite -{ - void - run() override - { - using namespace std::chrono; - using namespace csf; - - // Generate a quasi-random scale free network and simulate consensus - // as we vary transaction submission rates - - int const n = 100; // Peers - - int const numUNLs = 15; // UNL lists - int const minUNLSize = n / 4, maxUNLSize = n / 2; - - ConsensusParms const parms{}; - Sim sim; - PeerGroup network = sim.createGroup(n); - - // generate trust ranks - std::vector const ranks = - sample(network.size(), PowerLawDistribution{1, 3}, sim.rng); - - // generate scale-free trust graph - randomRankedTrust( - network, - ranks, - numUNLs, - std::uniform_int_distribution<>{minUNLSize, maxUNLSize}, - sim.rng); - - // nodes with a trust line in either direction are network-connected - network.connectFromTrust(round(0.2 * parms.ledgerGRANULARITY)); - - // Initialize collectors to track statistics to report - TxCollector txCollector; - LedgerCollector ledgerCollector; - auto colls = makeCollectors(txCollector, ledgerCollector); - sim.collectors.add(colls); - - // Initial round to set prior state - sim.run(1); - - // Initialize timers - HeartbeatTimer heart(sim.scheduler, seconds(10s)); - - // Run for 10 minutes, submitting 100 tx/second - std::chrono::nanoseconds const simDuration = 10min; - std::chrono::nanoseconds const quiet = 10s; - Rate const rate{.count = 100, .duration = 1000ms}; - - // txs, start/stop/step, target - auto peerSelector = makeSelector(network.begin(), network.end(), ranks, sim.rng); - auto txSubmitter = makeSubmitter( - ConstantDistribution{rate.inv()}, - sim.scheduler.now() + quiet, - sim.scheduler.now() + (simDuration - quiet), - peerSelector, - sim.scheduler, - sim.rng); - - // run simulation for given duration - heart.start(); - sim.run(simDuration); - - BEAST_EXPECT(sim.branches() == 1); - BEAST_EXPECT(sim.synchronized()); - - // TODO: Clean up this formatting mess!! - - log << "Peers: " << network.size() << std::endl; - log << "Simulated Duration: " << duration_cast(simDuration).count() << " ms" - << std::endl; - log << "Branches: " << sim.branches() << std::endl; - log << "Synchronized: " << (sim.synchronized() ? "Y" : "N") << std::endl; - log << std::endl; - - txCollector.report(simDuration, log); - ledgerCollector.report(simDuration, log); - // Print summary? - // # forks? # of LCLs? - // # peers - // # tx submitted - // # ledgers/sec etc.? - } -}; - -BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(ScaleFreeSim, consensus, xrpl, 80); - -} // namespace xrpl::test diff --git a/src/test/consensus/Validations_test.cpp b/src/test/consensus/Validations_test.cpp deleted file mode 100644 index 606c6f2824..0000000000 --- a/src/test/consensus/Validations_test.cpp +++ /dev/null @@ -1,1059 +0,0 @@ -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace xrpl::test::csf { -class Validations_test : public beast::unit_test::Suite -{ - using clock_type = beast::AbstractClock const; - - // Helper to convert steady_clock to a reasonable NetClock - // This allows a single manual clock in the unit tests - static NetClock::time_point - toNetClock(clock_type const& c) - { - // We don't care about the actual epochs, but do want the - // generated NetClock time to be well past its epoch to ensure - // any subtractions are positive - using namespace std::chrono; - return NetClock::time_point( - duration_cast(c.now().time_since_epoch() + 86400s)); - } - - // Represents a node that can issue validations - class Node - { - clock_type const& c_; - PeerID nodeID_; - bool trusted_ = true; - std::size_t signIdx_{1}; - std::optional loadFee_; - - public: - Node(PeerID nodeID, clock_type const& c) : c_(c), nodeID_(nodeID) - { - } - - void - untrust() - { - trusted_ = false; - } - - void - trust() - { - trusted_ = true; - } - - void - setLoadFee(std::uint32_t fee) - { - loadFee_ = fee; - } - - [[nodiscard]] PeerID - nodeID() const - { - return nodeID_; - } - - void - advanceKey() - { - signIdx_++; - } - - [[nodiscard]] PeerKey - currKey() const - { - return std::make_pair(nodeID_, signIdx_); - } - - [[nodiscard]] PeerKey - masterKey() const - { - return std::make_pair(nodeID_, 0); - } - [[nodiscard]] NetClock::time_point - now() const - { - return toNetClock(c_); - } - - // Issue a new validation with given sequence number and id and - // with signing and seen times offset from the common clock - [[nodiscard]] Validation - validate( - Ledger::ID id, - Ledger::Seq seq, - NetClock::duration signOffset, - NetClock::duration seenOffset, - bool full) const - { - Validation v{ - id, - seq, - now() + signOffset, - now() + seenOffset, - currKey(), - nodeID_, - full, - loadFee_}; - if (trusted_) - v.setTrusted(); - return v; - } - - [[nodiscard]] Validation - validate(Ledger ledger, NetClock::duration signOffset, NetClock::duration seenOffset) const - { - return validate(ledger.id(), ledger.seq(), signOffset, seenOffset, true); - } - - [[nodiscard]] Validation - validate(Ledger ledger) const - { - return validate( - ledger.id(), ledger.seq(), NetClock::duration{0}, NetClock::duration{0}, true); - } - - [[nodiscard]] Validation - partial(Ledger ledger) const - { - return validate( - ledger.id(), ledger.seq(), NetClock::duration{0}, NetClock::duration{0}, false); - } - }; - - // Generic Validations adaptor - class Adaptor - { - clock_type& c_; - LedgerOracle& oracle_; - - public: - // Non-locking mutex to avoid locks in generic Validations - struct Mutex - { - void - lock() - { - } - - void - unlock() - { - } - }; - - using Validation = csf::Validation; - using Ledger = csf::Ledger; - - Adaptor(clock_type& c, LedgerOracle& o) : c_{c}, oracle_{o} - { - } - - [[nodiscard]] NetClock::time_point - now() const - { - return toNetClock(c_); - } - - std::optional - acquire(Ledger::ID const& id) - { - return oracle_.lookup(id); - } - }; - - // Specialize generic Validations using the above types - using TestValidations = Validations; - - // Gather the dependencies of TestValidations in a single class and provide - // accessors for simplifying test logic - class TestHarness - { - ValidationParms p_; - beast::ManualClock clock_; - TestValidations tv_; - PeerID nextNodeId_{0}; - - public: - explicit TestHarness(LedgerOracle& o) : tv_(p_, clock_, clock_, o) - { - } - - ValStatus - add(Validation const& v) - { - return tv_.add(v.nodeID(), v); - } - - TestValidations& - vals() - { - return tv_; - } - - Node - makeNode() - { - return Node(nextNodeId_++, clock_); - } - - ValidationParms - parms() const - { - return p_; - } - - auto& - clock() - { - return clock_; - } - }; - - Ledger const genesisLedger_{Ledger::MakeGenesis{}}; - - void - testAddValidation() - { - using namespace std::chrono_literals; - - testcase("Add validation"); - LedgerHistoryHelper h; - Ledger const ledgerA = h["a"]; - Ledger ledgerAB = h["ab"]; - Ledger ledgerAZ = h["az"]; - Ledger ledgerABC = h["abc"]; - Ledger const ledgerABCD = h["abcd"]; - Ledger const ledgerABCDE = h["abcde"]; - - { - TestHarness harness(h.oracle); - Node n = harness.makeNode(); - - auto const v = n.validate(ledgerA); - - // Add a current validation - BEAST_EXPECT(ValStatus::Current == harness.add(v)); - - // Re-adding violates the increasing seq requirement for full - // validations - BEAST_EXPECT(ValStatus::BadSeq == harness.add(v)); - - harness.clock().advance(1s); - - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerAB))); - - // Test the node changing signing key - - // Confirm old ledger on hand, but not new ledger - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerAB.id()) == 1); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerABC.id()) == 0); - - // Rotate signing keys - n.advanceKey(); - - harness.clock().advance(1s); - - // Cannot re-do the same full validation sequence - BEAST_EXPECT(ValStatus::Conflicting == harness.add(n.validate(ledgerAB))); - // Cannot send the same partial validation sequence - BEAST_EXPECT(ValStatus::Conflicting == harness.add(n.partial(ledgerAB))); - - // Now trusts the newest ledger too - harness.clock().advance(1s); - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerABC))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerAB.id()) == 1); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerABC.id()) == 1); - - // Processing validations out of order should ignore the older - // validation - harness.clock().advance(2s); - auto const valABCDE = n.validate(ledgerABCDE); - - harness.clock().advance(4s); - auto const valABCD = n.validate(ledgerABCD); - - BEAST_EXPECT(ValStatus::Current == harness.add(valABCD)); - - BEAST_EXPECT(ValStatus::Stale == harness.add(valABCDE)); - } - - { - // Process validations out of order with shifted times - - TestHarness harness(h.oracle); - Node const n = harness.makeNode(); - - // Establish a new current validation - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerA))); - - // Process a validation that has "later" seq but early sign time - BEAST_EXPECT(ValStatus::Stale == harness.add(n.validate(ledgerAB, -1s, -1s))); - - // Process a validation that has a later seq and later sign - // time - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerABC, 1s, 1s))); - } - - { - // Test stale on arrival validations - TestHarness harness(h.oracle); - Node const n = harness.makeNode(); - - BEAST_EXPECT( - ValStatus::Stale == - harness.add(n.validate(ledgerA, -harness.parms().validationCurrentEarly, 0s))); - - BEAST_EXPECT( - ValStatus::Stale == - harness.add(n.validate(ledgerA, harness.parms().validationCurrentWall, 0s))); - - BEAST_EXPECT( - ValStatus::Stale == - harness.add(n.validate(ledgerA, 0s, harness.parms().validationCurrentLocal))); - } - - { - // Test that full or partials cannot be sent for older sequence - // numbers, unless time-out has happened - for (bool doFull : {true, false}) - { - TestHarness harness(h.oracle); - Node n = harness.makeNode(); - - auto process = [&](Ledger& lgr) { - if (doFull) - return harness.add(n.validate(lgr)); - return harness.add(n.partial(lgr)); - }; - - BEAST_EXPECT(ValStatus::Current == process(ledgerABC)); - harness.clock().advance(1s); - BEAST_EXPECT(ledgerAB.seq() < ledgerABC.seq()); - BEAST_EXPECT(ValStatus::BadSeq == process(ledgerAB)); - - // If we advance far enough for AB to expire, we can fully - // validate or partially validate that sequence number again - BEAST_EXPECT(ValStatus::Conflicting == process(ledgerAZ)); - harness.clock().advance(harness.parms().validationSetExpires + 1ms); - BEAST_EXPECT(ValStatus::Current == process(ledgerAZ)); - } - } - } - - void - testOnStale() - { - testcase("Stale validation"); - // Verify validation becomes stale based solely on time passing, but - // use different functions to trigger the check for staleness - - LedgerHistoryHelper h; - Ledger ledgerA = h["a"]; - Ledger const ledgerAB = h["ab"]; - - using Trigger = std::function; - - std::vector const triggers = { - [&](TestValidations& vals) { vals.currentTrusted(); }, - [&](TestValidations& vals) { vals.getCurrentNodeIDs(); }, - [&](TestValidations& vals) { vals.getPreferred(genesisLedger_); }, - [&](TestValidations& vals) { vals.getNodesAfter(ledgerA, ledgerA.id()); }}; - for (Trigger const& trigger : triggers) - { - TestHarness harness(h.oracle); - Node const n = harness.makeNode(); - - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerAB))); - trigger(harness.vals()); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 1); - BEAST_EXPECT( - harness.vals().getPreferred(genesisLedger_) == - std::make_pair(ledgerAB.seq(), ledgerAB.id())); - harness.clock().advance(harness.parms().validationCurrentLocal); - - // trigger check for stale - trigger(harness.vals()); - - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 0); - BEAST_EXPECT(harness.vals().getPreferred(genesisLedger_) == std::nullopt); - } - } - - void - testGetNodesAfter() - { - // Test getting number of nodes working on a validation descending - // a prescribed one. This count should only be for trusted nodes, but - // includes partial and full validations - - using namespace std::chrono_literals; - testcase("Get nodes after"); - - LedgerHistoryHelper h; - Ledger const ledgerA = h["a"]; - Ledger const ledgerAB = h["ab"]; - Ledger const ledgerABC = h["abc"]; - Ledger const ledgerAD = h["ad"]; - - TestHarness harness(h.oracle); - Node const trustedNode1 = harness.makeNode(); - Node const trustedNode2 = harness.makeNode(); - Node const trustedNode3 = harness.makeNode(); - - Node notTrustedNode = harness.makeNode(); - notTrustedNode.untrust(); - - // first round a,b,c agree, d has is partial - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode1.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode2.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode3.partial(ledgerA))); - - for (Ledger const& ledger : {ledgerA, ledgerAB, ledgerABC, ledgerAD}) - BEAST_EXPECT(harness.vals().getNodesAfter(ledger, ledger.id()) == 0); - - harness.clock().advance(5s); - - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode1.validate(ledgerAB))); - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode2.validate(ledgerABC))); - BEAST_EXPECT(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerAB))); - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode3.partial(ledgerABC))); - - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 3); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerAB, ledgerAB.id()) == 2); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerABC, ledgerABC.id()) == 0); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerAD, ledgerAD.id()) == 0); - - // If given a ledger inconsistent with the id, is still able to check using slower method - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerAD, ledgerA.id()) == 1); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerAD, ledgerAB.id()) == 2); - } - - void - testCurrentTrusted() - { - using namespace std::chrono_literals; - testcase("Current trusted validations"); - - LedgerHistoryHelper h; - Ledger const ledgerA = h["a"]; - Ledger const ledgerB = h["b"]; - Ledger const ledgerAC = h["ac"]; - - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Node b = harness.makeNode(); - b.untrust(); - - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(b.validate(ledgerB))); - - // Only a is trusted - BEAST_EXPECT(harness.vals().currentTrusted().size() == 1); - BEAST_EXPECT(harness.vals().currentTrusted()[0].ledgerID() == ledgerA.id()); - BEAST_EXPECT(harness.vals().currentTrusted()[0].seq() == ledgerA.seq()); - - harness.clock().advance(3s); - - for (auto const& node : {a, b}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.validate(ledgerAC))); - - // New validation for a - BEAST_EXPECT(harness.vals().currentTrusted().size() == 1); - BEAST_EXPECT(harness.vals().currentTrusted()[0].ledgerID() == ledgerAC.id()); - BEAST_EXPECT(harness.vals().currentTrusted()[0].seq() == ledgerAC.seq()); - - // Pass enough time for it to go stale - harness.clock().advance(harness.parms().validationCurrentLocal); - BEAST_EXPECT(harness.vals().currentTrusted().empty()); - } - - void - testGetCurrentPublicKeys() - { - using namespace std::chrono_literals; - testcase("Current public keys"); - - LedgerHistoryHelper h; - Ledger const ledgerA = h["a"]; - Ledger const ledgerAC = h["ac"]; - - TestHarness harness(h.oracle); - Node a = harness.makeNode(), b = harness.makeNode(); - b.untrust(); - - for (auto const& node : {a, b}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.validate(ledgerA))); - - { - hash_set const expectedKeys = {a.nodeID(), b.nodeID()}; - BEAST_EXPECT(harness.vals().getCurrentNodeIDs() == expectedKeys); - } - - harness.clock().advance(3s); - - // Change keys and issue partials - a.advanceKey(); - b.advanceKey(); - - for (auto const& node : {a, b}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.partial(ledgerAC))); - - { - hash_set const expectedKeys = {a.nodeID(), b.nodeID()}; - BEAST_EXPECT(harness.vals().getCurrentNodeIDs() == expectedKeys); - } - - // Pass enough time for them to go stale - harness.clock().advance(harness.parms().validationCurrentLocal); - BEAST_EXPECT(harness.vals().getCurrentNodeIDs().empty()); - } - - void - testTrustedByLedgerFunctions() - { - // Test the Validations functions that calculate a value by ledger ID - using namespace std::chrono_literals; - testcase("By ledger functions"); - - // Several Validations functions return a set of values associated - // with trusted ledgers sharing the same ledger ID. The tests below - // exercise this logic by saving the set of trusted Validations, and - // verifying that the Validations member functions all calculate the - // proper transformation of the available ledgers. - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - - Node a = harness.makeNode(), b = harness.makeNode(), c = harness.makeNode(), - d = harness.makeNode(), e = harness.makeNode(); - - c.untrust(); - // Mix of load fees - a.setLoadFee(12); - b.setLoadFee(1); - c.setLoadFee(12); - e.setLoadFee(12); - - hash_map, std::vector> trustedValidations; - - //---------------------------------------------------------------------- - // checkers - auto sorted = [](auto vec) { - std::sort(vec.begin(), vec.end()); - return vec; - }; - auto compare = [&]() { - for (auto& it : trustedValidations) - { - auto const& id = it.first.first; - auto const& seq = it.first.second; - auto const& expectedValidations = it.second; - - BEAST_EXPECT(harness.vals().numTrustedForLedger(id) == expectedValidations.size()); - BEAST_EXPECT( - sorted(harness.vals().getTrustedForLedger(id, seq)) == - sorted(expectedValidations)); - - std::uint32_t const baseFee = 0; - std::vector expectedFees; - expectedFees.reserve(expectedValidations.size()); - for (auto const& val : expectedValidations) - { - expectedFees.push_back(val.loadFee().value_or(baseFee)); - } - - BEAST_EXPECT(sorted(harness.vals().fees(id, baseFee)) == sorted(expectedFees)); - } - }; - - //---------------------------------------------------------------------- - Ledger const ledgerA = h["a"]; - Ledger const ledgerB = h["b"]; - Ledger const ledgerAC = h["ac"]; - - // Add a dummy ID to cover unknown ledger identifiers - trustedValidations[{Ledger::ID{100}, Ledger::Seq{100}}] = {}; - - // first round a,b,c agree - for (auto const& node : {a, b, c}) - { - auto const val = node.validate(ledgerA); - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - if (val.trusted()) - trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); - } - // d disagrees - { - auto const val = d.validate(ledgerB); - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); - } - // e only issues partials - { - BEAST_EXPECT(ValStatus::Current == harness.add(e.partial(ledgerA))); - } - - harness.clock().advance(5s); - // second round, a,b,c move to ledger 2 - for (auto const& node : {a, b, c}) - { - auto const val = node.validate(ledgerAC); - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - if (val.trusted()) - trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); - } - // d now thinks ledger 1, but cannot re-issue a previously used seq - // and attempting it should generate a conflict. - { - BEAST_EXPECT(ValStatus::Conflicting == harness.add(d.partial(ledgerA))); - } - // e only issues partials - { - BEAST_EXPECT(ValStatus::Current == harness.add(e.partial(ledgerAC))); - } - - compare(); - } - - void - testExpire() - { - // Verify expiring clears out validations stored by ledger - testcase("Expire validations"); - SuiteJournal j("Validations_test", *this); - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - constexpr Ledger::Seq kOne(1); - constexpr Ledger::Seq kTwo(2); - - // simple cases - Ledger const ledgerA = h["a"]; - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerA))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); - harness.vals().expire(j); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); - harness.clock().advance(harness.parms().validationSetExpires); - harness.vals().expire(j); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 0); - - // use setSeqToKeep to keep the validation from expire - Ledger const ledgerB = h["ab"]; - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerB))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); - harness.vals().setSeqToKeep(ledgerB.seq(), ledgerB.seq() + kOne); - harness.clock().advance(harness.parms().validationSetExpires); - harness.vals().expire(j); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); - // change toKeep - harness.vals().setSeqToKeep(ledgerB.seq() + kOne, ledgerB.seq() + kTwo); - // advance clock slowly - int const loops = - harness.parms().validationSetExpires / harness.parms().validationFRESHNESS + 1; - for (int i = 0; i < loops; ++i) - { - harness.clock().advance(harness.parms().validationFRESHNESS); - harness.vals().expire(j); - } - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerB.id()) == 0); - - // Allow the validation with high seq to expire - Ledger const ledgerC = h["abc"]; - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerC))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerC.id()) == 1); - harness.vals().setSeqToKeep(ledgerC.seq() - kOne, ledgerC.seq()); - harness.clock().advance(harness.parms().validationSetExpires); - harness.vals().expire(j); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerC.id()) == 0); - } - - void - testFlush() - { - // Test final flush of validations - using namespace std::chrono_literals; - testcase("Flush validations"); - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const trustedNode1 = harness.makeNode(); - Node const trustedNode2 = harness.makeNode(); - Node notTrustedNode = harness.makeNode(); - notTrustedNode.untrust(); - - Ledger const ledgerA = h["a"]; - Ledger const ledgerAB = h["ab"]; - - hash_map expected; - for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode}) - { - auto const val = node.validate(ledgerA); - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - expected.emplace(node.nodeID(), val); - } - - // Send in a new validation for a, saving the new one into the expected - // map after setting the proper prior ledger ID it replaced - harness.clock().advance(1s); - auto newVal = trustedNode1.validate(ledgerAB); - BEAST_EXPECT(ValStatus::Current == harness.add(newVal)); - expected.find(trustedNode1.nodeID())->second = newVal; - } - - void - testGetPreferredLedger() - { - using namespace std::chrono_literals; - testcase("Preferred Ledger"); - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const trustedNode1 = harness.makeNode(); - Node const trustedNode2 = harness.makeNode(); - Node const trustedNode3 = harness.makeNode(); - - Node notTrustedNode = harness.makeNode(); - notTrustedNode.untrust(); - - Ledger const ledgerA = h["a"]; - Ledger const ledgerB = h["b"]; - Ledger const ledgerAC = h["ac"]; - Ledger const ledgerACD = h["acd"]; - - using Seq = Ledger::Seq; - - auto pref = [](Ledger ledger) { return std::make_pair(ledger.seq(), ledger.id()); }; - - // Empty (no ledgers) - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == std::nullopt); - - // Single ledger - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode1.validate(ledgerB))); - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == pref(ledgerB)); - BEAST_EXPECT(harness.vals().getPreferred(ledgerB) == pref(ledgerB)); - - // Minimum valid sequence - BEAST_EXPECT(harness.vals().getPreferred(ledgerA, Seq{10}) == ledgerA.id()); - - // Untrusted doesn't impact preferred ledger - // (ledgerB has tie-break over ledgerA) - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode2.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerA))); - BEAST_EXPECT(ledgerB.id() > ledgerA.id()); - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == pref(ledgerB)); - BEAST_EXPECT(harness.vals().getPreferred(ledgerB) == pref(ledgerB)); - - // Partial does break ties - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode3.partial(ledgerA))); - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == pref(ledgerA)); - BEAST_EXPECT(harness.vals().getPreferred(ledgerB) == pref(ledgerA)); - - harness.clock().advance(5s); - - // Parent of preferred-> stick with ledger - for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode, trustedNode3}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.validate(ledgerAC))); - // Parent of preferred stays put - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == pref(ledgerA)); - // Earlier different chain, switch - BEAST_EXPECT(harness.vals().getPreferred(ledgerB) == pref(ledgerAC)); - // Later on chain, stays where it is - BEAST_EXPECT(harness.vals().getPreferred(ledgerACD) == pref(ledgerACD)); - - // Any later grandchild or different chain is preferred - harness.clock().advance(5s); - for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode, trustedNode3}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.validate(ledgerACD))); - for (auto const& ledger : {ledgerA, ledgerB, ledgerACD}) - BEAST_EXPECT(harness.vals().getPreferred(ledger) == pref(ledgerACD)); - } - - void - testGetPreferredLCL() - { - using namespace std::chrono_literals; - testcase("Get preferred LCL"); - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - - Ledger const ledgerA = h["a"]; - Ledger const ledgerB = h["b"]; - Ledger const ledgerC = h["c"]; - - using ID = Ledger::ID; - using Seq = Ledger::Seq; - - hash_map peerCounts; - - // No trusted validations or counts sticks with current ledger - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerA.id()); - - ++peerCounts[ledgerB.id()]; - - // No trusted validations, rely on peer counts - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerB.id()); - - ++peerCounts[ledgerC.id()]; - // No trusted validations, tied peers goes with larger ID - BEAST_EXPECT(ledgerC.id() > ledgerB.id()); - - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerC.id()); - - peerCounts[ledgerC.id()] += 1000; - - // Single trusted always wins over peer counts - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerA))); - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerA.id()); - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerB, Seq{0}, peerCounts) == ledgerA.id()); - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerC, Seq{0}, peerCounts) == ledgerA.id()); - - // Stick with current ledger if trusted validation ledger has too old - // of a sequence - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerB, Seq{2}, peerCounts) == ledgerB.id()); - } - - void - testAcquireValidatedLedger() - { - using namespace std::chrono_literals; - testcase("Acquire validated ledger"); - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Node const b = harness.makeNode(); - - using ID = Ledger::ID; - using Seq = Ledger::Seq; - - // Validate the ledger before it is actually available - Validation const val = a.validate(ID{2}, Seq{2}, 0s, 0s, true); - - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - // Validation is available - BEAST_EXPECT(harness.vals().numTrustedForLedger(ID{2}) == 1); - // but ledger based data is not - BEAST_EXPECT(harness.vals().getNodesAfter(genesisLedger_, ID{0}) == 0); - // Initial preferred branch falls back to the ledger we are trying to - // acquire - BEAST_EXPECT(harness.vals().getPreferred(genesisLedger_) == std::make_pair(Seq{2}, ID{2})); - - // After adding another unavailable validation, the preferred ledger - // breaks ties via higher ID - BEAST_EXPECT(ValStatus::Current == harness.add(b.validate(ID{3}, Seq{2}, 0s, 0s, true))); - BEAST_EXPECT(harness.vals().getPreferred(genesisLedger_) == std::make_pair(Seq{2}, ID{3})); - - // Create the ledger - Ledger const ledgerAB = h["ab"]; - // Now it should be available - BEAST_EXPECT(harness.vals().getNodesAfter(genesisLedger_, ID{0}) == 1); - - // Create a validation that is not available - harness.clock().advance(5s); - Validation const val2 = a.validate(ID{4}, Seq{4}, 0s, 0s, true); - BEAST_EXPECT(ValStatus::Current == harness.add(val2)); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ID{4}) == 1); - BEAST_EXPECT( - harness.vals().getPreferred(genesisLedger_) == - std::make_pair(ledgerAB.seq(), ledgerAB.id())); - - // Another node requesting that ledger still doesn't change things - Validation const val3 = b.validate(ID{4}, Seq{4}, 0s, 0s, true); - BEAST_EXPECT(ValStatus::Current == harness.add(val3)); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ID{4}) == 2); - BEAST_EXPECT( - harness.vals().getPreferred(genesisLedger_) == - std::make_pair(ledgerAB.seq(), ledgerAB.id())); - - // Switch to validation that is available - harness.clock().advance(5s); - Ledger const ledgerABCDE = h["abcde"]; - BEAST_EXPECT(ValStatus::Current == harness.add(a.partial(ledgerABCDE))); - BEAST_EXPECT(ValStatus::Current == harness.add(b.partial(ledgerABCDE))); - BEAST_EXPECT( - harness.vals().getPreferred(genesisLedger_) == - std::make_pair(ledgerABCDE.seq(), ledgerABCDE.id())); - } - - void - testNumTrustedForLedger() - { - testcase("NumTrustedForLedger"); - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Node const b = harness.makeNode(); - Ledger const ledgerA = h["a"]; - - BEAST_EXPECT(ValStatus::Current == harness.add(a.partial(ledgerA))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 0); - - BEAST_EXPECT(ValStatus::Current == harness.add(b.validate(ledgerA))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); - } - - void - testSeqEnforcer() - { - testcase("SeqEnforcer"); - using Seq = Ledger::Seq; - using namespace std::chrono; - - beast::ManualClock clock; - SeqEnforcer enforcer; - - ValidationParms const p; - - BEAST_EXPECT(enforcer(clock.now(), Seq{1}, p)); - BEAST_EXPECT(enforcer(clock.now(), Seq{10}, p)); - BEAST_EXPECT(!enforcer(clock.now(), Seq{5}, p)); - BEAST_EXPECT(!enforcer(clock.now(), Seq{9}, p)); - clock.advance(p.validationSetExpires - 1ms); - BEAST_EXPECT(!enforcer(clock.now(), Seq{1}, p)); - clock.advance(2ms); - BEAST_EXPECT(enforcer(clock.now(), Seq{1}, p)); - } - - void - testTrustChanged() - { - testcase("TrustChanged"); - using namespace std::chrono; - - auto checker = [this]( - TestValidations& vals, - hash_set const& listed, - std::vector const& trustedVals) { - Ledger::ID const testID = - trustedVals.empty() ? this->genesisLedger_.id() : trustedVals[0].ledgerID(); - Ledger::Seq const testSeq = - trustedVals.empty() ? this->genesisLedger_.seq() : trustedVals[0].seq(); - BEAST_EXPECT(vals.currentTrusted() == trustedVals); - BEAST_EXPECT(vals.getCurrentNodeIDs() == listed); - BEAST_EXPECT( - vals.getNodesAfter(this->genesisLedger_, genesisLedger_.id()) == - trustedVals.size()); - if (trustedVals.empty()) - { - BEAST_EXPECT(vals.getPreferred(this->genesisLedger_) == std::nullopt); - } - else - { - BEAST_EXPECT(vals.getPreferred(this->genesisLedger_)->second == testID); - } - BEAST_EXPECT(vals.getTrustedForLedger(testID, testSeq) == trustedVals); - BEAST_EXPECT(vals.numTrustedForLedger(testID) == trustedVals.size()); - }; - - { - // Trusted to untrusted - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Ledger const ledgerAB = h["ab"]; - Validation const v = a.validate(ledgerAB); - BEAST_EXPECT(ValStatus::Current == harness.add(v)); - - hash_set const listed({a.nodeID()}); - std::vector trustedVals({v}); - checker(harness.vals(), listed, trustedVals); - - trustedVals.clear(); - harness.vals().trustChanged({}, {a.nodeID()}); - checker(harness.vals(), listed, trustedVals); - } - - { - // Untrusted to trusted - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node a = harness.makeNode(); - a.untrust(); - Ledger const ledgerAB = h["ab"]; - Validation const v = a.validate(ledgerAB); - BEAST_EXPECT(ValStatus::Current == harness.add(v)); - - hash_set const listed({a.nodeID()}); - std::vector trustedVals; - checker(harness.vals(), listed, trustedVals); - - trustedVals.push_back(v); - harness.vals().trustChanged({a.nodeID()}, {}); - checker(harness.vals(), listed, trustedVals); - } - - { - // Trusted but not acquired -> untrusted - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Validation const v = a.validate(Ledger::ID{2}, Ledger::Seq{2}, 0s, 0s, true); - BEAST_EXPECT(ValStatus::Current == harness.add(v)); - - hash_set const listed({a.nodeID()}); - std::vector trustedVals({v}); - auto& vals = harness.vals(); - BEAST_EXPECT(vals.currentTrusted() == trustedVals); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(vals.getPreferred(genesisLedger_)->second == v.ledgerID()); - BEAST_EXPECT(vals.getNodesAfter(genesisLedger_, genesisLedger_.id()) == 0); - - trustedVals.clear(); - harness.vals().trustChanged({}, {a.nodeID()}); - // make acquiring ledger available - h["ab"]; - BEAST_EXPECT(vals.currentTrusted() == trustedVals); - BEAST_EXPECT(vals.getPreferred(genesisLedger_) == std::nullopt); - BEAST_EXPECT(vals.getNodesAfter(genesisLedger_, genesisLedger_.id()) == 0); - } - } - - void - run() override - { - testAddValidation(); - testOnStale(); - testGetNodesAfter(); - testCurrentTrusted(); - testGetCurrentPublicKeys(); - testTrustedByLedgerFunctions(); - testExpire(); - testFlush(); - testGetPreferredLedger(); - testGetPreferredLCL(); - testAcquireValidatedLedger(); - testNumTrustedForLedger(); - testSeqEnforcer(); - testTrustChanged(); - } -}; - -BEAST_DEFINE_TESTSUITE(Validations, consensus, xrpl); -} // namespace xrpl::test::csf diff --git a/src/test/csf/BasicNetwork_test.cpp b/src/test/csf/BasicNetwork_test.cpp deleted file mode 100644 index 22f52a1b63..0000000000 --- a/src/test/csf/BasicNetwork_test.cpp +++ /dev/null @@ -1,134 +0,0 @@ -#include -#include - -#include - -#include -#include - -namespace xrpl::test { - -class BasicNetwork_test : public beast::unit_test::Suite -{ -public: - struct Peer - { - int id; - std::set set; - - Peer(Peer const&) = default; - Peer(Peer&&) = default; - - explicit Peer(int id) : id(id) - { - } - - template - void - start(csf::Scheduler& scheduler, Net& net) - { - using namespace std::chrono_literals; - auto t = scheduler.in(1s, [&] { set.insert(0); }); - if (id == 0) - { - for (auto const link : net.links(this)) - { - net.send( - this, link.target, [&, to = link.target] { to->receive(net, this, 1); }); - } - } - else - { - scheduler.cancel(t); - } - } - - template - void - receive(Net& net, Peer* from, int m) - { - set.insert(m); - ++m; - if (m < 5) - { - for (auto const link : net.links(this)) - { - net.send(this, link.target, [&, mm = m, to = link.target] { - to->receive(net, this, mm); - }); - } - } - } - }; - - void - testNetwork() - { - using namespace std::chrono_literals; - std::vector pv; - pv.emplace_back(0); - pv.emplace_back(1); - pv.emplace_back(2); - csf::Scheduler scheduler; - csf::BasicNetwork net(scheduler); - BEAST_EXPECT(!net.connect(&pv[0], &pv[0])); - BEAST_EXPECT(net.connect(&pv[0], &pv[1], 1s)); - BEAST_EXPECT(net.connect(&pv[1], &pv[2], 1s)); - BEAST_EXPECT(!net.connect(&pv[0], &pv[1])); - for (auto& peer : pv) - peer.start(scheduler, net); - BEAST_EXPECT(scheduler.stepFor(0s)); - BEAST_EXPECT(scheduler.stepFor(1s)); - BEAST_EXPECT(scheduler.step()); - BEAST_EXPECT(!scheduler.step()); - BEAST_EXPECT(!scheduler.stepFor(1s)); - net.send(&pv[0], &pv[1], [] {}); - net.send(&pv[1], &pv[0], [] {}); - BEAST_EXPECT(net.disconnect(&pv[0], &pv[1])); - BEAST_EXPECT(!net.disconnect(&pv[0], &pv[1])); - for (;;) - { - auto const links = net.links(&pv[1]); - if (links.empty()) - break; - BEAST_EXPECT(net.disconnect(&pv[1], links[0].target)); - } - BEAST_EXPECT(pv[0].set == std::set({0, 2, 4})); - BEAST_EXPECT(pv[1].set == std::set({1, 3})); - BEAST_EXPECT(pv[2].set == std::set({2, 4})); - } - - void - testDisconnect() - { - using namespace std::chrono_literals; - csf::Scheduler scheduler; - csf::BasicNetwork net(scheduler); - BEAST_EXPECT(net.connect(0, 1, 1s)); - BEAST_EXPECT(net.connect(0, 2, 2s)); - - std::set delivered; - net.send(0, 1, [&]() { delivered.insert(1); }); - net.send(0, 2, [&]() { delivered.insert(2); }); - - scheduler.in(1000ms, [&]() { BEAST_EXPECT(net.disconnect(0, 2)); }); - scheduler.in(1100ms, [&]() { BEAST_EXPECT(net.connect(0, 2)); }); - - scheduler.step(); - - // only the first message is delivered because the disconnect at 1 s - // purges all pending messages from 0 to 2 - BEAST_EXPECT(delivered == std::set({1})); - } - - void - run() override - { - testNetwork(); - testDisconnect(); - } -}; - -BEAST_DEFINE_TESTSUITE(BasicNetwork, csf, xrpl); - -} // namespace xrpl::test diff --git a/src/test/csf/Digraph_test.cpp b/src/test/csf/Digraph_test.cpp deleted file mode 100644 index 40bfafde9c..0000000000 --- a/src/test/csf/Digraph_test.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#include - -#include - -#include -#include -#include -#include - -namespace xrpl::test { - -class Digraph_test : public beast::unit_test::Suite -{ -public: - void - run() override - { - using namespace csf; - using Graph = Digraph; - Graph graph; - - BEAST_EXPECT(!graph.connected('a', 'b')); - BEAST_EXPECT(!graph.edge('a', 'b')); - BEAST_EXPECT(!graph.disconnect('a', 'b')); - - BEAST_EXPECT(graph.connect('a', 'b', "foobar")); - BEAST_EXPECT(graph.connected('a', 'b')); - BEAST_EXPECT( - *graph.edge('a', 'b') == "foobar"); // NOLINT(bugprone-unchecked-optional-access) - - BEAST_EXPECT(!graph.connect('a', 'b', "repeat")); - BEAST_EXPECT(graph.disconnect('a', 'b')); - BEAST_EXPECT(graph.connect('a', 'b', "repeat")); - BEAST_EXPECT(graph.connected('a', 'b')); - BEAST_EXPECT( - *graph.edge('a', 'b') == "repeat"); // NOLINT(bugprone-unchecked-optional-access) - - BEAST_EXPECT(graph.connect('a', 'c', "tree")); - - { - std::vector> edges; - - for (auto const& edge : graph.outEdges('a')) - { - edges.emplace_back(edge.source, edge.target, edge.data); - } - - std::vector> expected; - expected.emplace_back('a', 'b', "repeat"); - expected.emplace_back('a', 'c', "tree"); - BEAST_EXPECT(edges == expected); - BEAST_EXPECT(graph.outDegree('a') == expected.size()); - } - - BEAST_EXPECT(graph.outEdges('r').size() == 0); - BEAST_EXPECT(graph.outDegree('r') == 0); - BEAST_EXPECT(graph.outDegree('c') == 0); - - // only 'a' has out edges - BEAST_EXPECT(graph.outVertices().size() == 1); - std::vector const expected = {'b', 'c'}; - - BEAST_EXPECT((graph.outVertices('a') == expected)); - BEAST_EXPECT(graph.outVertices('b').size() == 0); - BEAST_EXPECT(graph.outVertices('c').size() == 0); - BEAST_EXPECT(graph.outVertices('r').size() == 0); - - std::stringstream ss; - graph.saveDot(ss, [](char v) { return v; }); - std::string const expectedDot = - "digraph {\n" - "a -> b;\n" - "a -> c;\n" - "}\n"; - BEAST_EXPECT(ss.str() == expectedDot); - } -}; - -BEAST_DEFINE_TESTSUITE(Digraph, csf, xrpl); - -} // namespace xrpl::test diff --git a/src/test/csf/Histogram_test.cpp b/src/test/csf/Histogram_test.cpp deleted file mode 100644 index 65edb14e5d..0000000000 --- a/src/test/csf/Histogram_test.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include - -#include - -namespace xrpl::test { - -class Histogram_test : public beast::unit_test::Suite -{ -public: - void - run() override - { - using namespace csf; - Histogram hist; - - BEAST_EXPECT(hist.size() == 0); - BEAST_EXPECT(hist.numBins() == 0); - BEAST_EXPECT(hist.minValue() == 0); - BEAST_EXPECT(hist.maxValue() == 0); - BEAST_EXPECT(hist.avg() == 0); - BEAST_EXPECT(hist.percentile(0.0f) == hist.minValue()); - BEAST_EXPECT(hist.percentile(0.5f) == 0); - BEAST_EXPECT(hist.percentile(0.9f) == 0); - BEAST_EXPECT(hist.percentile(1.0f) == hist.maxValue()); - - hist.insert(1); - - BEAST_EXPECT(hist.size() == 1); - BEAST_EXPECT(hist.numBins() == 1); - BEAST_EXPECT(hist.minValue() == 1); - BEAST_EXPECT(hist.maxValue() == 1); - BEAST_EXPECT(hist.avg() == 1); - BEAST_EXPECT(hist.percentile(0.0f) == hist.minValue()); - BEAST_EXPECT(hist.percentile(0.5f) == 1); - BEAST_EXPECT(hist.percentile(0.9f) == 1); - BEAST_EXPECT(hist.percentile(1.0f) == hist.maxValue()); - - hist.insert(9); - - BEAST_EXPECT(hist.size() == 2); - BEAST_EXPECT(hist.numBins() == 2); - BEAST_EXPECT(hist.minValue() == 1); - BEAST_EXPECT(hist.maxValue() == 9); - BEAST_EXPECT(hist.avg() == 5); - BEAST_EXPECT(hist.percentile(0.0f) == hist.minValue()); - BEAST_EXPECT(hist.percentile(0.5f) == 1); - BEAST_EXPECT(hist.percentile(0.9f) == 9); - BEAST_EXPECT(hist.percentile(1.0f) == hist.maxValue()); - - hist.insert(1); - - BEAST_EXPECT(hist.size() == 3); - BEAST_EXPECT(hist.numBins() == 2); - BEAST_EXPECT(hist.minValue() == 1); - BEAST_EXPECT(hist.maxValue() == 9); - BEAST_EXPECT(hist.avg() == 11 / 3); - BEAST_EXPECT(hist.percentile(0.0f) == hist.minValue()); - BEAST_EXPECT(hist.percentile(0.5f) == 1); - BEAST_EXPECT(hist.percentile(0.9f) == 9); - BEAST_EXPECT(hist.percentile(1.0f) == hist.maxValue()); - } -}; - -BEAST_DEFINE_TESTSUITE(Histogram, csf, xrpl); - -} // namespace xrpl::test diff --git a/src/test/csf/Scheduler_test.cpp b/src/test/csf/Scheduler_test.cpp deleted file mode 100644 index 6f4ac051c4..0000000000 --- a/src/test/csf/Scheduler_test.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include - -#include - -#include - -namespace xrpl::test { - -class Scheduler_test : public beast::unit_test::Suite -{ -public: - void - run() override - { - using namespace std::chrono_literals; - csf::Scheduler scheduler; - std::set seen; - - scheduler.in(1s, [&] { seen.insert(1); }); - scheduler.in(2s, [&] { seen.insert(2); }); - auto token = scheduler.in(3s, [&] { seen.insert(3); }); - scheduler.at(scheduler.now() + 4s, [&] { seen.insert(4); }); - scheduler.at(scheduler.now() + 8s, [&] { seen.insert(8); }); - - auto start = scheduler.now(); - - // Process first event - BEAST_EXPECT(seen.empty()); - BEAST_EXPECT(scheduler.stepOne()); - BEAST_EXPECT(seen == std::set({1})); - BEAST_EXPECT(scheduler.now() == (start + 1s)); - - // No processing if stepping until current time - BEAST_EXPECT(scheduler.stepUntil(scheduler.now())); - BEAST_EXPECT(seen == std::set({1})); - BEAST_EXPECT(scheduler.now() == (start + 1s)); - - // Process next event - BEAST_EXPECT(scheduler.stepFor(1s)); - BEAST_EXPECT(seen == std::set({1, 2})); - BEAST_EXPECT(scheduler.now() == (start + 2s)); - - // Don't process cancelled event, but advance clock - scheduler.cancel(token); - BEAST_EXPECT(scheduler.stepFor(1s)); - BEAST_EXPECT(seen == std::set({1, 2})); - BEAST_EXPECT(scheduler.now() == (start + 3s)); - - // Process until 3 seen ints - BEAST_EXPECT(scheduler.stepWhile([&]() { return seen.size() < 3; })); - BEAST_EXPECT(seen == std::set({1, 2, 4})); - BEAST_EXPECT(scheduler.now() == (start + 4s)); - - // Process the rest - BEAST_EXPECT(scheduler.step()); - BEAST_EXPECT(seen == std::set({1, 2, 4, 8})); - BEAST_EXPECT(scheduler.now() == (start + 8s)); - - // Process the rest again doesn't advance - BEAST_EXPECT(!scheduler.step()); - BEAST_EXPECT(seen == std::set({1, 2, 4, 8})); - BEAST_EXPECT(scheduler.now() == (start + 8s)); - } -}; - -BEAST_DEFINE_TESTSUITE(Scheduler, csf, xrpl); - -} // namespace xrpl::test diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index 2e131b895e..4828e03815 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -27,6 +27,7 @@ target_link_libraries(xrpl_tests PRIVATE GTest::gtest GTest::gmock xrpl.libxrpl) # supported on Windows. set(test_modules basics + consensus crypto json peerfinder @@ -58,6 +59,16 @@ foreach(module IN LISTS test_modules) ) endforeach() +# The consensus tests use the CSF (Consensus Simulation Framework) helpers, so +# compile the CSF sources into the test binary. The consensus engine itself now +# lives in libxrpl, so no xrpld sources or include paths are needed here. +file( + GLOB_RECURSE csf_sources + CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/csf/*.cpp" +) +target_sources(xrpl_tests PRIVATE ${csf_sources}) + # The test helpers and per-module test headers are not built with add_module, # so verify them against the test binary's own compile environment. if(verify_headers) diff --git a/src/tests/libxrpl/basics/base_uint_test.cpp b/src/tests/libxrpl/basics/base_uint.cpp similarity index 99% rename from src/tests/libxrpl/basics/base_uint_test.cpp rename to src/tests/libxrpl/basics/base_uint.cpp index c9bfc35c94..174cc33aa0 100644 --- a/src/tests/libxrpl/basics/base_uint_test.cpp +++ b/src/tests/libxrpl/basics/base_uint.cpp @@ -1,5 +1,6 @@ -#include #include + +#include #include #include diff --git a/src/tests/libxrpl/consensus/ByzantineFailureSim.cpp b/src/tests/libxrpl/consensus/ByzantineFailureSim.cpp new file mode 100644 index 0000000000..e712817121 --- /dev/null +++ b/src/tests/libxrpl/consensus/ByzantineFailureSim.cpp @@ -0,0 +1,81 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace xrpl::test { + +TEST(ByzantineFailureSimTest, DISABLED_byzantine_failure_sim) +{ + using namespace csf; + using namespace std::chrono; + + // This test simulates a specific topology with nodes generating + // different ledgers due to a simulated byzantine failure (injecting + // an extra non-consensus transaction). + + Sim sim; + ConsensusParms const parms{}; + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + PeerGroup a = sim.createGroup(1); + PeerGroup b = sim.createGroup(1); + PeerGroup c = sim.createGroup(1); + PeerGroup d = sim.createGroup(1); + PeerGroup e = sim.createGroup(1); + PeerGroup f = sim.createGroup(1); + PeerGroup g = sim.createGroup(1); + + a.trustAndConnect(a + b + c + g, delay); + b.trustAndConnect(b + a + c + d + e, delay); + c.trustAndConnect(c + a + b + d + e, delay); + d.trustAndConnect(d + b + c + e + f, delay); + e.trustAndConnect(e + b + c + d + f, delay); + f.trustAndConnect(f + d + e + g, delay); + g.trustAndConnect(g + a + f, delay); + + PeerGroup const network = a + b + c + d + e + f + g; + + StreamCollector sc{std::cout}; + + sim.collectors.add(sc); + + for (TrustGraph::ForkInfo const& fi : sim.trustGraph.forkablePairs(0.8)) + { + std::cout << "Can fork " << PeerGroup{fi.unlA} << " " + << " " << PeerGroup{fi.unlB} << " overlap " << fi.overlap << " required " + << fi.required << "\n"; + }; + + // set prior state + sim.run(1); + + PeerGroup byzantineNodes = a + b + c + g; + // All peers see some TX 0 + for (Peer* peer : network) + { + peer->submit(Tx{0}); + // Peers 0,1,2,6 will close the next ledger differently by injecting + // a non-consensus approved transaction + if (byzantineNodes.contains(peer)) + { + peer->txInjections.emplace(peer->lastClosedLedger.seq(), Tx{42}); + } + } + sim.run(4); + std::cout << "Branches: " << sim.branches() << "\n"; + std::cout << "Fully synchronized: " << std::boolalpha << sim.synchronized() << "\n"; + // Not tessting anything currently. + SUCCEED(); +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/CensorshipDetector.cpp b/src/tests/libxrpl/consensus/CensorshipDetector.cpp new file mode 100644 index 0000000000..aa6b2d086b --- /dev/null +++ b/src/tests/libxrpl/consensus/CensorshipDetector.cpp @@ -0,0 +1,81 @@ +#include + +#include + +#include +#include +#include + +namespace xrpl::test { + +namespace { + +void +runRound( + CensorshipDetector& cdet, + int round, + std::vector proposed, + std::vector accepted, + std::vector remain, + std::vector remove) +{ + // Begin tracking what we're proposing this round + CensorshipDetector::TxIDSeqVec proposal; + for (auto const& i : proposed) + proposal.emplace_back(i, round); + cdet.propose(std::move(proposal)); + + // Finalize the round, by processing what we accepted; then + // remove anything that needs to be removed and ensure that + // what remains is correct. + cdet.check(std::move(accepted), [&remove, &remain](auto id, auto seq) { + // If the item is supposed to be removed from the censorship + // detector internal tracker manually, do it now: + if (std::ranges::find(remove, id) != remove.end()) + return true; + + // If the item is supposed to still remain in the censorship + // detector internal tracker; remove it from the vector. + auto it = std::ranges::find(remain, id); + if (it != remain.end()) + remain.erase(it); + return false; + }); + + // On entry, this set contained all the elements that should be tracked + // by the detector after we process this round. We removed all the items + // that actually were in the tracker, so this should now be empty: + EXPECT_TRUE(remain.empty()); +} + +} // namespace + +TEST(CensorshipDetectorTest, censorship_detector) +{ + SCOPED_TRACE("Censorship Detector"); + + CensorshipDetector cdet; + int round = 0; + // proposed accepted remain remove + runRound(cdet, ++round, {}, {}, {}, {}); + runRound(cdet, ++round, {10, 11, 12, 13}, {11, 2}, {10, 13}, {}); + runRound(cdet, ++round, {10, 13, 14, 15}, {14}, {10, 13, 15}, {}); + runRound(cdet, ++round, {10, 13, 15, 16}, {15, 16}, {10, 13}, {}); + runRound(cdet, ++round, {10, 13}, {17, 18}, {10, 13}, {}); + runRound(cdet, ++round, {10, 19}, {}, {10, 19}, {}); + runRound(cdet, ++round, {10, 19, 20}, {20}, {10}, {19}); + runRound(cdet, ++round, {21}, {21}, {}, {}); + runRound(cdet, ++round, {}, {22}, {}, {}); + runRound(cdet, ++round, {23, 24, 25, 26}, {25, 27}, {23, 26}, {24}); + runRound(cdet, ++round, {23, 26, 28}, {26, 28}, {23}, {}); + + for (int i = 0; i != 10; ++i) + runRound(cdet, ++round, {23}, {}, {23}, {}); + + runRound(cdet, ++round, {23, 29}, {29}, {23}, {}); + runRound(cdet, ++round, {30, 31}, {31}, {30}, {}); + runRound(cdet, ++round, {30}, {30}, {}, {}); + runRound(cdet, ++round, {}, {}, {}, {}); +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/Consensus.cpp b/src/tests/libxrpl/consensus/Consensus.cpp new file mode 100644 index 0000000000..d303d28e89 --- /dev/null +++ b/src/tests/libxrpl/consensus/Consensus.cpp @@ -0,0 +1,1455 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::test { + +namespace { + +beast::Journal +journal() +{ + return beast::Journal{TestSink::instance()}; +} + +bool +shouldCloseLedger( + bool anyTransactions, + std::size_t prevProposers, + std::size_t proposersClosed, + std::size_t proposersValidated, + std::chrono::milliseconds prevRoundTime, + std::chrono::milliseconds timeSincePrevClose, + std::chrono::milliseconds openTime, + std::chrono::milliseconds idleInterval, + ConsensusParms const& parms, + std::unique_ptr const& clog = {}) +{ + return xrpl::shouldCloseLedger( + anyTransactions, + prevProposers, + proposersClosed, + proposersValidated, + prevRoundTime, + timeSincePrevClose, + openTime, + idleInterval, + parms, + journal(), + clog); +} + +ConsensusState +checkConsensus( + std::size_t prevProposers, + std::size_t currentProposers, + std::size_t currentAgree, + std::size_t currentFinished, + std::chrono::milliseconds previousAgreeTime, + std::chrono::milliseconds currentAgreeTime, + bool stalled, + ConsensusParms const& parms, + bool proposing, + std::unique_ptr const& clog = {}) +{ + return xrpl::checkConsensus( + prevProposers, + currentProposers, + currentAgree, + currentFinished, + previousAgreeTime, + currentAgreeTime, + stalled, + parms, + proposing, + journal(), + clog); +} + +using CsfDisputedTx = DisputedTx; + +CsfDisputedTx +makeDisputedTx(csf::Tx tx, bool ourVote, std::size_t numPeers) +{ + return CsfDisputedTx{tx, ourVote, numPeers, journal()}; +} + +bool +isStalled( + CsfDisputedTx const& dispute, + ConsensusParms const& parms, + bool proposing, + int peersUnchanged, + std::unique_ptr const& clog) +{ + return dispute.stalled(parms, proposing, peersUnchanged, journal(), clog); +} + +// Helper collector for testPreferredByBranch +// Invasively disconnects network at bad times to cause splits +struct Disruptor +{ + csf::PeerGroup& network; + csf::PeerGroup& groupCfast; + csf::PeerGroup& groupCsplit; + csf::SimDuration delay; + bool reconnected = false; + + Disruptor(csf::PeerGroup& net, csf::PeerGroup& c, csf::PeerGroup& split, csf::SimDuration d) + : network(net), groupCfast(c), groupCsplit(split), delay(d) + { + } + + template + void + on(csf::PeerID, csf::SimTime, E const&) + { + } + + void + on(csf::PeerID who, csf::SimTime, csf::FullyValidateLedger const& e) + { + using namespace std::chrono; + // As soon as the fastC node fully validates C, disconnect + // ALL c nodes from the network. The fast C node needs to disconnect + // as well to prevent it from relaying the validations it did see + if (who == groupCfast[0]->id && e.ledger.seq() == csf::Ledger::Seq{2}) + { + network.disconnect(groupCsplit); + network.disconnect(groupCfast); + } + } + + void + on(csf::PeerID who, csf::SimTime, csf::AcceptLedger const& e) + { + // As soon as anyone generates a child of B or C, reconnect the + // network so those validations make it through + if (!reconnected && e.ledger.seq() == csf::Ledger::Seq{3}) + { + reconnected = true; + network.connect(groupCsplit, delay); + } + } +}; + +// Helper collector for testPauseForLaggards +// This will remove the ledgerAccept delay used to +// initially create the slow vs. fast validator groups. +struct UndoDelay +{ + csf::PeerGroup& g; + + UndoDelay(csf::PeerGroup& a) : g(a) + { + } + + template + void + on(csf::PeerID, csf::SimTime, E const&) + { + } + + void + on(csf::PeerID who, csf::SimTime, csf::AcceptLedger const& e) + { + for (csf::Peer* p : g) + { + if (p->id == who) + p->delays.ledgerAccept = std::chrono::seconds{0}; + } + } +}; + +} // namespace + +TEST(ConsensusTest, should_close_ledger) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("should close ledger"); + + // Use default parameters + ConsensusParms const p{}; + + // Bizarre times forcibly close + EXPECT_TRUE(shouldCloseLedger(true, 10, 10, 10, -10s, 10s, 1s, 1s, p)); + EXPECT_TRUE(shouldCloseLedger(true, 10, 10, 10, 100h, 10s, 1s, 1s, p)); + EXPECT_TRUE(shouldCloseLedger(true, 10, 10, 10, 10s, 100h, 1s, 1s, p)); + + // Rest of network has closed + EXPECT_TRUE(shouldCloseLedger(true, 10, 3, 5, 10s, 10s, 10s, 10s, p)); + + // No transactions means wait until end of internval + EXPECT_TRUE(!shouldCloseLedger(false, 10, 0, 0, 1s, 1s, 1s, 10s, p)); + EXPECT_TRUE(shouldCloseLedger(false, 10, 0, 0, 1s, 10s, 1s, 10s, p)); + + // Enforce minimum ledger open time + EXPECT_TRUE(!shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 1s, 10s, p)); + + // Don't go too much faster than last time + EXPECT_TRUE(!shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 3s, 10s, p)); + + EXPECT_TRUE(shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 10s, 10s, p)); +} + +TEST(ConsensusTest, check_consensus) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("check consensus"); + + // Use default parameters + ConsensusParms const p{}; + + /////////////// + // Disputes still in doubt + // + // Not enough time has elapsed + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 2s, false, p, true)); + + // If not enough peers have proposed, ensure + // more time for proposals + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 4s, false, p, true)); + + // Enough time has elapsed and we all agree + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 2, 2, 0, 3s, 10s, false, p, true)); + + // Enough time has elapsed and we don't yet agree + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 1, 0, 3s, 10s, false, p, true)); + + // Our peers have moved on + // Enough time has elapsed and we all agree + EXPECT_TRUE(ConsensusState::MovedOn == checkConsensus(10, 2, 1, 8, 3s, 10s, false, p, true)); + + // If no peers, don't agree until time has passed. + EXPECT_TRUE(ConsensusState::No == checkConsensus(0, 0, 0, 0, 3s, 10s, false, p, true)); + + // Agree if no peers and enough time has passed. + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(0, 0, 0, 0, 3s, 16s, false, p, true)); + + // Expire if too much time has passed without agreement + EXPECT_TRUE(ConsensusState::Expired == checkConsensus(10, 8, 1, 0, 1s, 19s, false, p, true)); + + /////////////// + // Stalled + // + // Not enough time has elapsed + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 2s, true, p, true)); + + // If not enough peers have proposed, ensure + // more time for proposals + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 4s, true, p, true)); + + // Enough time has elapsed and we all agree + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 2, 2, 0, 3s, 10s, true, p, true)); + + // Enough time has elapsed and we don't yet agree, but there's nothing + // left to dispute + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 2, 1, 0, 3s, 10s, true, p, true)); + + // Our peers have moved on + // Enough time has elapsed and we all agree, nothing left to dispute + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 2, 1, 8, 3s, 10s, true, p, true)); + + // If no peers, don't agree until time has passed. + EXPECT_TRUE(ConsensusState::No == checkConsensus(0, 0, 0, 0, 3s, 10s, true, p, true)); + + // Agree if no peers and enough time has passed. + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(0, 0, 0, 0, 3s, 16s, true, p, true)); + + // We are done if there's nothing left to dispute, no matter how much + // time has passed + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 8, 1, 0, 1s, 19s, true, p, true)); +} + +TEST(ConsensusTest, standalone) +{ + using namespace std::chrono_literals; + using namespace csf; + SCOPED_TRACE("standalone"); + + Sim s; + PeerGroup const peers = s.createGroup(1); + Peer* peer = peers[0]; + peer->targetLedgers = 1; + peer->start(); + peer->submit(Tx{1}); + + s.scheduler.step(); + + // Inspect that the proper ledger was created + auto const& lcl = peer->lastClosedLedger; + EXPECT_TRUE(peer->prevLedgerID() == lcl.id()); + EXPECT_TRUE(lcl.seq() == Ledger::Seq{1}); + EXPECT_TRUE(lcl.txs().size() == 1); + EXPECT_TRUE(lcl.txs().contains(Tx{1})); + EXPECT_TRUE(peer->prevProposers == 0); +} + +TEST(ConsensusTest, peers_agree) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("peers agree"); + + ConsensusParms const parms{}; + Sim sim; + PeerGroup peers = sim.createGroup(5); + + // Connected trust and network graphs with single fixed delay + peers.trustAndConnect(peers, round(0.2 * parms.ledgerGRANULARITY)); + + // everyone submits their own ID as a TX + for (Peer* p : peers) + p->submit(Tx(static_cast(p->id))); + + sim.run(1); + + // All peers are in sync + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + for (Peer const* peer : peers) + { + auto const& lcl = peer->lastClosedLedger; + EXPECT_TRUE(lcl.id() == peer->prevLedgerID()); + EXPECT_TRUE(lcl.seq() == Ledger::Seq{1}); + // All peers proposed + EXPECT_TRUE(peer->prevProposers == peers.size() - 1); + // All transactions were accepted + for (std::uint32_t i = 0; i < peers.size(); ++i) + EXPECT_TRUE(lcl.txs().contains(Tx{i})); + } + } +} + +TEST(ConsensusTest, slow_peers) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("slow peers"); + + // Several tests of a complete trust graph with a subset of peers + // that have significantly longer network delays to the rest of the + // network + + // Test when a slow peer doesn't delay a consensus quorum (4/5 agree) + { + ConsensusParms const parms{}; + Sim sim; + PeerGroup slow = sim.createGroup(1); + PeerGroup fast = sim.createGroup(4); + PeerGroup network = fast + slow; + + // Fully connected trust graph + network.trust(network); + + // Fast and slow network connections + fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); + + slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); + + // All peers submit their own ID as a transaction + for (Peer* peer : network) + peer->submit(Tx{static_cast(peer->id)}); + + sim.run(1); + + // Verify all peers have same LCL but are missing transaction 0 + // All peers are in sync even with a slower peer 0 + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + for (Peer const* peer : network) + { + auto const& lcl = peer->lastClosedLedger; + EXPECT_TRUE(lcl.id() == peer->prevLedgerID()); + EXPECT_TRUE(lcl.seq() == Ledger::Seq{1}); + + EXPECT_TRUE(peer->prevProposers == network.size() - 1); + EXPECT_TRUE(peer->prevRoundTime == network[0]->prevRoundTime); + + EXPECT_TRUE(not lcl.txs().contains(Tx{0})); + for (std::uint32_t i = 2; i < network.size(); ++i) + EXPECT_TRUE(lcl.txs().contains(Tx{i})); + + // Tx 0 didn't make it + EXPECT_TRUE(peer->openTxs.contains(Tx{0})); + } + } + } + + // Test when the slow peers delay a consensus quorum (4/6 agree) + { + // Run two tests + // 1. The slow peers are participating in consensus + // 2. The slow peers are just observing + + for (auto isParticipant : {true, false}) + { + ConsensusParms const parms{}; + + Sim sim; + PeerGroup slow = sim.createGroup(2); + PeerGroup fast = sim.createGroup(4); + PeerGroup network = fast + slow; + + // Connected trust graph + network.trust(network); + + // Fast and slow network connections + fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); + + slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); + + for (Peer* peer : slow) + peer->runAsValidator = isParticipant; + + // All peers submit their own ID as a transaction and relay it + // to peers + for (Peer* peer : network) + peer->submit(Tx{static_cast(peer->id)}); + + sim.run(1); + + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + // Verify all peers have same LCL but are missing + // transaction 0,1 which was not received by all peers + // before the ledger closed + for (Peer const* peer : network) + { + // Closed ledger has all but transaction 0,1 + auto const& lcl = peer->lastClosedLedger; + EXPECT_TRUE(lcl.seq() == Ledger::Seq{1}); + EXPECT_TRUE(not lcl.txs().contains(Tx{0})); + EXPECT_TRUE(not lcl.txs().contains(Tx{1})); + for (std::uint32_t i = slow.size(); i < network.size(); ++i) + EXPECT_TRUE(lcl.txs().contains(Tx{i})); + + // Tx 0-1 didn't make it + EXPECT_TRUE(peer->openTxs.contains(Tx{0})); + EXPECT_TRUE(peer->openTxs.contains(Tx{1})); + } + + Peer const* slowPeer = slow[0]; + if (isParticipant) + { + EXPECT_TRUE(slowPeer->prevProposers == network.size() - 1); + } + else + { + EXPECT_TRUE(slowPeer->prevProposers == fast.size()); + } + + for (Peer const* peer : fast) + { + // Due to the network link delay settings + // Peer 0 initially proposes {0} + // Peer 1 initially proposes {1} + // Peers 2-5 initially propose {2,3,4,5} + // Since peers 2-5 agree, 4/6 > the initial 50% needed + // to include a disputed transaction, so Peer 0/1 switch + // to agree with those peers. Peer 0/1 then closes with + // an 80% quorum of agreeing positions (5/6) match. + // + // Peers 2-5 do not change position, since tx 0 or tx 1 + // have less than the 50% initial threshold. They also + // cannot declare consensus, since 4/6 agreeing + // positions are < 80% threshold. They therefore need an + // additional timerEntry call to see the updated + // positions from Peer 0 & 1. + + if (isParticipant) + { + EXPECT_TRUE(peer->prevProposers == network.size() - 1); + EXPECT_TRUE(peer->prevRoundTime > slowPeer->prevRoundTime); + } + else + { + EXPECT_TRUE(peer->prevProposers == fast.size() - 1); + // so all peers should have closed together + EXPECT_TRUE(peer->prevRoundTime == slowPeer->prevRoundTime); + } + } + } + } + } +} + +TEST(ConsensusTest, close_time_disagree) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("close time disagree"); + + // This is a very specialized test to get ledgers to disagree on + // the close time. It unfortunately assumes knowledge about current + // timing constants. This is a necessary evil to get coverage up + // pending more extensive refactorings of timing constants. + + // In order to agree-to-disagree on the close time, there must be no + // clear majority of nodes agreeing on a close time. This test + // sets a relative offset to the peers internal clocks so that they + // send proposals with differing times. + + // However, agreement is on the effective close time, not the + // exact close time. The minimum closeTimeResolution is given by + // ledgerPossibleTimeResolutions[0], which is currently 10s. This means + // the skews need to be at least 10 seconds to have different effective + // close times. + + // Complicating this matter is that nodes will ignore proposals + // with times more than proposeFRESHNESS =20s in the past. So at + // the minimum granularity, we have at most 3 types of skews + // (0s,10s,20s). + + // This test therefore has 6 nodes, with 2 nodes having each type of + // skew. Then no majority (1/3 < 1/2) of nodes will agree on an + // actual close time. + + ConsensusParms const parms{}; + Sim sim; + + PeerGroup groupA = sim.createGroup(2); + PeerGroup const groupB = sim.createGroup(2); + PeerGroup const groupC = sim.createGroup(2); + PeerGroup network = groupA + groupB + groupC; + + network.trust(network); + network.connect(network, round(0.2 * parms.ledgerGRANULARITY)); + + // Run consensus without skew until we have a short close time + // resolution + Peer const* firstPeer = *groupA.begin(); + while (firstPeer->lastClosedLedger.closeTimeResolution() >= parms.proposeFRESHNESS) + sim.run(1); + + // Introduce a shift on the time of 2/3 of peers + for (Peer* peer : groupA) + peer->clockSkew = parms.proposeFRESHNESS / 2; + for (Peer* peer : groupB) + peer->clockSkew = parms.proposeFRESHNESS; + + sim.run(1); + + // All nodes agreed to disagree on the close time + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + for (Peer const* peer : network) + EXPECT_TRUE(!peer->lastClosedLedger.closeAgree()); + } +} + +TEST(ConsensusTest, wrong_lcl) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("wrong LCL"); + + // Specialized test to exercise a temporary fork in which some peers + // are working on an incorrect prior ledger. + + ConsensusParms const parms{}; + + // Vary the time it takes to process validations to exercise detecting + // the wrong LCL at different phases of consensus + for (auto validationDelay : {0ms, parms.ledgerMinClose}) + { + // Consider 10 peers: + // 0 1 2 3 4 5 6 7 8 9 + // minority majorityA majorityB + // + // Nodes 0-1 trust nodes 0-4 + // Nodes 2-9 trust nodes 2-9 + // + // By submitting tx 0 to nodes 0-4 and tx 1 to nodes 5-9, + // nodes 0-1 will generate the wrong LCL (with tx 0). The remaining + // nodes will instead accept the ledger with tx 1. + + // Nodes 0-1 will detect this mismatch during a subsequent round + // since nodes 2-4 will validate a different ledger. + + // Nodes 0-1 will acquire the proper ledger from the network and + // resume consensus and eventually generate the dominant network + // ledger. + + // This topology can potentially fork with the above trust relations + // but that is intended for this test. + + Sim sim; + + PeerGroup minority = sim.createGroup(2); + PeerGroup const majorityA = sim.createGroup(3); + PeerGroup const majorityB = sim.createGroup(5); + + PeerGroup majority = majorityA + majorityB; + PeerGroup const network = minority + majority; + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + minority.trustAndConnect(minority + majorityA, delay); + majority.trustAndConnect(majority, delay); + + CollectByNode jumps; + sim.collectors.add(jumps); + + EXPECT_TRUE(sim.trustGraph.canFork(parms.minConsensusPct / 100.)); + + // initial round to set prior state + sim.run(1); + + // Nodes in smaller UNL have seen tx 0, nodes in other unl have seen + // tx 1 + for (Peer* peer : network) + peer->delays.recvValidation = validationDelay; + for (Peer* peer : (minority + majorityA)) + peer->openTxs.insert(Tx{0}); + for (Peer* peer : majorityB) + peer->openTxs.insert(Tx{1}); + + // Run for additional rounds + // With no validation delay, only 2 more rounds are needed. + // 1. Round to generate different ledgers + // 2. Round to detect different prior ledgers (but still generate + // wrong ones) and recover within that round since wrong LCL + // is detected before we close + // + // With a validation delay of ledgerMinClose, we need 3 more + // rounds. + // 1. Round to generate different ledgers + // 2. Round to detect different prior ledgers (but still generate + // wrong ones) but end up declaring consensus on wrong LCL (but + // with the right transaction set!). This is because we detect + // the wrong LCL after we have closed the ledger, so we declare + // consensus based solely on our peer proposals. But we haven't + // had time to acquire the right ledger. + // 3. Round to correct + sim.run(3); + + // The network never actually forks, since node 0-1 never see a + // quorum of validations to fully validate the incorrect chain. + + // However, for a non zero-validation delay, the network is not + // synchronized because nodes 0 and 1 are running one ledger behind + EXPECT_TRUE(sim.branches() == 1); + if (sim.branches() == 1) + { + for (Peer const* peer : majority) + { + // No jumps for majority nodes + EXPECT_TRUE(jumps[peer->id].closeJumps.empty()); + EXPECT_TRUE(jumps[peer->id].fullyValidatedJumps.empty()); + } + for (Peer const* peer : minority) + { + auto& peerJumps = jumps[peer->id]; + // last closed ledger jump between chains + { + EXPECT_TRUE(peerJumps.closeJumps.size() == 1); + if (peerJumps.closeJumps.size() == 1) + { + JumpCollector::Jump const& jump = peerJumps.closeJumps.front(); + // Jump is to a different chain + EXPECT_TRUE(jump.from.seq() <= jump.to.seq()); + EXPECT_TRUE(!jump.to.isAncestor(jump.from)); + } + } + // fully validated jump forward in same chain + { + EXPECT_TRUE(peerJumps.fullyValidatedJumps.size() == 1); + if (peerJumps.fullyValidatedJumps.size() == 1) + { + JumpCollector::Jump const& jump = peerJumps.fullyValidatedJumps.front(); + // Jump is to a different chain with same seq + EXPECT_TRUE(jump.from.seq() < jump.to.seq()); + EXPECT_TRUE(jump.to.isAncestor(jump.from)); + } + } + } + } + } + + { + // Additional test engineered to switch LCL during the establish + // phase. This was added to trigger a scenario that previously + // crashed, in which switchLCL switched from establish to open + // phase, but still processed the establish phase logic. + + // Loner node will accept an initial ledger A, but all other nodes + // accept ledger B a bit later. By delaying the time it takes + // to process a validation, loner node will detect the wrongLCL + // after it is already in the establish phase of the next round. + + Sim sim; + PeerGroup loner = sim.createGroup(1); + PeerGroup const friends = sim.createGroup(3); + loner.trust(loner + friends); + + PeerGroup const others = sim.createGroup(6); + PeerGroup clique = friends + others; + clique.trust(clique); + + PeerGroup network = loner + clique; + network.connect(network, round(0.2 * parms.ledgerGRANULARITY)); + + // initial round to set prior state + sim.run(1); + for (Peer* peer : (loner + friends)) + peer->openTxs.insert(Tx(0)); + for (Peer* peer : others) + peer->openTxs.insert(Tx(1)); + + // Delay validation processing + for (Peer* peer : network) + peer->delays.recvValidation = parms.ledgerGRANULARITY; + + // additional rounds to generate wrongLCL and recover + sim.run(2); + + // Check all peers recovered + for (Peer const* p : network) + EXPECT_TRUE(p->prevLedgerID() == network[0]->prevLedgerID()); + } +} + +TEST(ConsensusTest, consensus_close_time_rounding) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("consensus close time rounding"); + + // This is a specialized test engineered to yield ledgers with different + // close times even though the peers believe they had close time + // consensus on the ledger. + ConsensusParms const parms; + + Sim sim; + + // This requires a group of 4 fast and 2 slow peers to create a + // situation in which a subset of peers requires seeing additional + // proposals to declare consensus. + PeerGroup slow = sim.createGroup(2); + PeerGroup fast = sim.createGroup(4); + PeerGroup network = fast + slow; + + // Connected trust graph + network.trust(network); + + // Fast and slow network connections + fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); + slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); + + // Run to the ledger *prior* to decreasing the resolution + sim.run(kIncreaseLedgerTimeResolutionEvery - 2); + + // In order to create the discrepancy, we want a case where if + // X = effCloseTime(closeTime, resolution, parentCloseTime) + // X != effCloseTime(X, resolution, parentCloseTime) + // + // That is, the effective close time is not a fixed point. This can + // happen if X = parentCloseTime + 1, but a subsequent rounding goes + // to the next highest multiple of resolution. + + // So we want to find an offset (now + offset) % 30s = 15 + // (now + offset) % 20s = 15 + // This way, the next ledger will close and round up Due to the + // network delay settings, the round of consensus will take 5s, so + // the next ledger's close time will + + NetClock::duration when = network[0]->now().time_since_epoch(); + + // Check we are before the 30s to 20s transition + NetClock::duration const resolution = network[0]->lastClosedLedger.closeTimeResolution(); + EXPECT_TRUE(resolution == NetClock::duration{30s}); + + while (((when % NetClock::duration{30s}) != NetClock::duration{15s}) || + ((when % NetClock::duration{20s}) != NetClock::duration{15s})) + when += 1s; + // Advance the clock without consensus running (IS THIS WHAT + // PREVENTS IT IN PRACTICE?) + sim.scheduler.stepFor(NetClock::time_point{when} - network[0]->now()); + + // Run one more ledger with 30s resolution + sim.run(1); + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + // close time should be ahead of clock time since we engineered + // the close time to round up + for (Peer const* peer : network) + { + EXPECT_TRUE(peer->lastClosedLedger.closeTime() > peer->now()); + EXPECT_TRUE(peer->lastClosedLedger.closeAgree()); + } + } + + // All peers submit their own ID as a transaction + for (Peer* peer : network) + peer->submit(Tx{static_cast(peer->id)}); + + // Run 1 more round, this time it will have a decreased + // resolution of 20 seconds. + + // The network delays are engineered so that the slow peers + // initially have the wrong tx hash, but they see a majority + // of agreement from their peers and declare consensus + // + // The trick is that everyone starts with a raw close time of + // 84681s + // Which has + // effCloseTime(86481s, 20s, 86490s) = 86491s + // However, when the slow peers update their position, they change + // the close time to 86451s. The fast peers declare consensus with + // the 86481s as their position still. + // + // When accepted the ledger + // - fast peers use eff(86481s) -> 86491s as the close time + // - slow peers use eff(eff(86481s)) -> eff(86491s) -> 86500s! + + sim.run(1); + + EXPECT_TRUE(sim.synchronized()); +} + +TEST(ConsensusTest, fork) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("fork"); + + std::uint32_t const numPeers = 10; + // Vary overlap between two UNLs + for (std::uint32_t overlap = 0; overlap <= numPeers; ++overlap) + { + ConsensusParms const parms{}; + Sim sim; + + std::uint32_t const numA = (numPeers - overlap) / 2; + std::uint32_t const numB = numPeers - numA - overlap; + + PeerGroup const aOnly = sim.createGroup(numA); + PeerGroup const bOnly = sim.createGroup(numB); + PeerGroup const commonOnly = sim.createGroup(overlap); + + PeerGroup a = aOnly + commonOnly; + PeerGroup b = bOnly + commonOnly; + + PeerGroup const network = a + b; + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + a.trustAndConnect(a, delay); + b.trustAndConnect(b, delay); + + // Initial round to set prior state + sim.run(1); + for (Peer* peer : network) + { + // Nodes have only seen transactions from their neighbors + peer->openTxs.insert(Tx{static_cast(peer->id)}); + for (Peer const* to : sim.trustGraph.trustedPeers(peer)) + peer->openTxs.insert(Tx{static_cast(to->id)}); + } + sim.run(1); + + // Fork should not happen for 40% or greater overlap + // Since the overlapped nodes have a UNL that is the union of the + // two cliques, the maximum sized UNL list is the number of peers + if (overlap > 0.4 * numPeers) + { + EXPECT_TRUE(sim.synchronized()); + } + else + { + // Even if we do fork, there shouldn't be more than 3 ledgers + // One for cliqueA, one for cliqueB and one for nodes in both + EXPECT_TRUE(sim.branches() <= 3); + } + } +} + +TEST(ConsensusTest, hub_network) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("hub network"); + + // Simulate a set of 5 validators that aren't directly connected but + // rely on a single hub node for communication + + ConsensusParms const parms{}; + Sim sim; + PeerGroup validators = sim.createGroup(5); + PeerGroup center = sim.createGroup(1); + validators.trust(validators); + center.trust(validators); + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + validators.connect(center, delay); + + center[0]->runAsValidator = false; + + // prep round to set initial state. + sim.run(1); + + // everyone submits their own ID as a TX and relay it to peers + for (Peer* p : validators) + p->submit(Tx(static_cast(p->id))); + + sim.run(1); + + // All peers are in sync + EXPECT_TRUE(sim.synchronized()); +} + +TEST(ConsensusTest, preferred_by_branch) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("preferred by branch"); + + // Simulate network splits that are prevented from forking when using + // preferred ledger by trie. This is a contrived example that involves + // excessive network splits, but demonstrates the safety improvement + // from the preferred ledger by trie approach. + + // Consider 10 validating nodes that comprise a single common UNL + // Ledger history: + // 1: A + // _/ \_ + // 2: B C + // _/ _/ \_ + // 3: D C' |||||||| (8 different ledgers) + + // - All nodes generate the common ledger A + // - 2 nodes generate B and 8 nodes generate C + // - Only 1 of the C nodes sees all the C validations and fully + // validates C. The rest of the C nodes split at just the right time + // such that they never see any C validations but their own. + // - The C nodes continue and generate 8 different child ledgers. + // - Meanwhile, the D nodes only saw 1 validation for C and 2 + // validations + // for B. + // - The network reconnects and the validations for generation 3 ledgers + // are observed (D and the 8 C's) + // - In the old approach, 2 votes for D outweighs 1 vote for each C' + // so the network would avalanche towards D and fully validate it + // EVEN though C was fully validated by one node + // - In the new approach, 2 votes for D are not enough to outweight the + // 8 implicit votes for C, so nodes will avalanche to C instead + + ConsensusParms const parms{}; + Sim sim; + + // Goes A->B->D + PeerGroup const groupABD = sim.createGroup(2); + // Single node that initially fully validates C before the split + PeerGroup groupCfast = sim.createGroup(1); + // Generates C, but fails to fully validate before the split + PeerGroup groupCsplit = sim.createGroup(7); + + PeerGroup groupNotFastC = groupABD + groupCsplit; + PeerGroup network = groupABD + groupCsplit + groupCfast; + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + SimDuration const fDelay = round(0.1 * parms.ledgerGRANULARITY); + + network.trust(network); + // C must have a shorter delay to see all the validations before the + // other nodes + network.connect(groupCfast, fDelay); + // The rest of the network is connected at the same speed + groupNotFastC.connect(groupNotFastC, delay); + + Disruptor dc(network, groupCfast, groupCsplit, delay); + sim.collectors.add(dc); + + // Consensus round to generate ledger A + sim.run(1); + EXPECT_TRUE(sim.synchronized()); + + // Next round generates B and C + // To force B, we inject an extra transaction in to those nodes + for (Peer* peer : groupABD) + { + peer->txInjections.emplace(peer->lastClosedLedger.seq(), Tx{42}); + } + // The Disruptor will ensure that nodes disconnect before the C + // validations make it to all but the fastC node + sim.run(1); + + // We are no longer in sync, but have not yet forked: + // 9 nodes consider A the last fully validated ledger and fastC sees C + EXPECT_TRUE(!sim.synchronized()); + EXPECT_TRUE(sim.branches() == 1); + + // Run another round to generate the 8 different C' ledgers + for (Peer* p : network) + p->submit(Tx(static_cast(p->id))); + sim.run(1); + + // Still not forked + EXPECT_TRUE(!sim.synchronized()); + EXPECT_TRUE(sim.branches() == 1); + + // Disruptor will reconnect all but the fastC node + sim.run(1); + + EXPECT_TRUE(sim.branches() == 1); + if (sim.branches() == 1) + { + EXPECT_TRUE(sim.synchronized()); + } + else // old approach caused a fork + { + EXPECT_TRUE(sim.branches(groupNotFastC) == 1); + EXPECT_TRUE(sim.synchronized(groupNotFastC) == 1); + } +} + +TEST(ConsensusTest, pause_for_laggards) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("pause for laggards"); + + // Test that validators that jump ahead of the network slow + // down. + + // We engineer the following validated ledger history scenario: + // + // / --> B1 --> C1 --> ... -> G1 "ahead" + // A + // \ --> B2 --> C2 "behind" + // + // After validating a common ledger A, a set of "behind" validators + // briefly run slower and validate the lower chain of ledgers. + // The "ahead" validators run normal speed and run ahead validating the + // upper chain of ledgers. + // + // Due to the uncommitted support definition of the preferred branch + // protocol, even if the "behind" validators are a majority, the "ahead" + // validators cannot jump to the proper branch until the "behind" + // validators catch up to the same sequence number. For this test to + // succeed, the ahead validators need to briefly slow down consensus. + + ConsensusParms const parms{}; + Sim sim; + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + + PeerGroup behind = sim.createGroup(3); + PeerGroup const ahead = sim.createGroup(2); + PeerGroup network = ahead + behind; + + hash_set trustedKeys; + for (Peer const* p : network) + trustedKeys.insert(p->key); + for (Peer* p : network) + p->trustedKeys = trustedKeys; + + network.trustAndConnect(network, delay); + + // Initial seed round to set prior state + sim.run(1); + + // Have the "behind" group initially take a really long time to + // accept a ledger after ending deliberation + for (Peer* p : behind) + p->delays.ledgerAccept = 20s; + + // Use the collector to revert the delay after the single + // slow ledger is generated + UndoDelay undoDelay{behind}; + sim.collectors.add(undoDelay); + + // Run the simulation for 100 seconds of simulation time with + std::chrono::nanoseconds const simDuration = 100s; + + // Simulate clients submitting 1 tx every 5 seconds to a random + // validator + Rate const rate{.count = 1, .duration = 5s}; + auto peerSelector = makeSelector( + network.begin(), network.end(), std::vector(network.size(), 1.), sim.rng); + auto txSubmitter = makeSubmitter( + ConstantDistribution{rate.inv()}, + sim.scheduler.now(), + sim.scheduler.now() + simDuration, + peerSelector, + sim.scheduler, + sim.rng); + + // Run simulation + sim.run(simDuration); + + // Verify that the network recovered + EXPECT_TRUE(sim.synchronized()); +} + +TEST(ConsensusTest, disputes) +{ + SCOPED_TRACE("disputes"); + + using namespace csf; + + // Test dispute objects directly + using Dispute = CsfDisputedTx; + + Tx const txTrue{99}; + Tx const txFalse{98}; + Tx const txFollowingTrue{97}; + Tx const txFollowingFalse{96}; + int const numPeers = 100; + ConsensusParms const p; + std::size_t peersUnchanged = 0; + + auto clog = std::make_unique(); + + // Three cases: + // 1 proposing, initial vote yes + // 2 proposing, initial vote no + // 3 not proposing, initial vote doesn't matter after the first update, + // use yes + { + Dispute proposingTrue = makeDisputedTx(txTrue, true, numPeers); + Dispute proposingFalse = makeDisputedTx(txFalse, false, numPeers); + Dispute followingTrue = makeDisputedTx(txFollowingTrue, true, numPeers); + Dispute followingFalse = makeDisputedTx(txFollowingFalse, false, numPeers); + EXPECT_TRUE(proposingTrue.id() == 99); + EXPECT_TRUE(proposingFalse.id() == 98); + EXPECT_TRUE(followingTrue.id() == 97); + EXPECT_TRUE(followingFalse.id() == 96); + + // Create an even split in the peer votes + for (int i = 0; i < numPeers; ++i) + { + EXPECT_TRUE(proposingTrue.setVote(PeerID(i), i < 50)); + EXPECT_TRUE(proposingFalse.setVote(PeerID(i), i < 50)); + EXPECT_TRUE(followingTrue.setVote(PeerID(i), i < 50)); + EXPECT_TRUE(followingFalse.setVote(PeerID(i), i < 50)); + } + // Switch the middle vote to match mine + EXPECT_TRUE(proposingTrue.setVote(PeerID(50), true)); + EXPECT_TRUE(proposingFalse.setVote(PeerID(49), false)); + EXPECT_TRUE(followingTrue.setVote(PeerID(50), true)); + EXPECT_TRUE(followingFalse.setVote(PeerID(49), false)); + + // no changes yet + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(proposingFalse, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingTrue, p, false, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingFalse, p, false, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()); + + // I'm in the majority, my vote should not change + EXPECT_TRUE(!proposingTrue.updateVote(5, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(5, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(5, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(5, false, p)); + + EXPECT_TRUE(!proposingTrue.updateVote(10, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(10, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(10, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(10, false, p)); + + peersUnchanged = 2; + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(proposingFalse, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingTrue, p, false, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingFalse, p, false, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()); + + // Right now, the vote is 51%. The requirement is about to jump to + // 65% + EXPECT_TRUE(proposingTrue.updateVote(55, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(55, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(55, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(55, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == false); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + // 16 validators change their vote to match my original vote + for (int i = 0; i < 16; ++i) + { + auto pTrue = PeerID(numPeers - i - 1); + auto pFalse = PeerID(i); + EXPECT_TRUE(proposingTrue.setVote(pTrue, true)); + EXPECT_TRUE(proposingFalse.setVote(pFalse, false)); + EXPECT_TRUE(followingTrue.setVote(pTrue, true)); + EXPECT_TRUE(followingFalse.setVote(pFalse, false)); + } + // The vote should now be 66%, threshold is 65% + EXPECT_TRUE(proposingTrue.updateVote(60, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(60, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(60, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(60, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // Threshold jumps to 70% + EXPECT_TRUE(proposingTrue.updateVote(86, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(86, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(86, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(86, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == false); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // 5 more validators change their vote to match my original vote + for (int i = 16; i < 21; ++i) + { + auto pTrue = PeerID(numPeers - i - 1); + auto pFalse = PeerID(i); + EXPECT_TRUE(proposingTrue.setVote(pTrue, true)); + EXPECT_TRUE(proposingFalse.setVote(pFalse, false)); + EXPECT_TRUE(followingTrue.setVote(pTrue, true)); + EXPECT_TRUE(followingFalse.setVote(pFalse, false)); + } + + // The vote should now be 71%, threshold is 70% + EXPECT_TRUE(proposingTrue.updateVote(90, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(90, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(90, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(90, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // The vote should now be 71%, threshold is 70% + EXPECT_TRUE(!proposingTrue.updateVote(150, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(150, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(150, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(150, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // The vote should now be 71%, threshold is 70% + EXPECT_TRUE(!proposingTrue.updateVote(190, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(190, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(190, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(190, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + peersUnchanged = 3; + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(proposingFalse, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingTrue, p, false, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingFalse, p, false, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()); + + // Threshold jumps to 95% + EXPECT_TRUE(proposingTrue.updateVote(220, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(220, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(220, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(220, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == false); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // 25 more validators change their vote to match my original vote + for (int i = 21; i < 46; ++i) + { + auto pTrue = PeerID(numPeers - i - 1); + auto pFalse = PeerID(i); + EXPECT_TRUE(proposingTrue.setVote(pTrue, true)); + EXPECT_TRUE(proposingFalse.setVote(pFalse, false)); + EXPECT_TRUE(followingTrue.setVote(pTrue, true)); + EXPECT_TRUE(followingFalse.setVote(pFalse, false)); + } + + // The vote should now be 96%, threshold is 95% + EXPECT_TRUE(proposingTrue.updateVote(250, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(250, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(250, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(250, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + for (peersUnchanged = 0; peersUnchanged < 6; ++peersUnchanged) + { + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(proposingFalse, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingTrue, p, false, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingFalse, p, false, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()); + } + + auto expectStalled = [&clog]( + int txid, + bool ourVote, + int ourTime, + int peerTime, + int support, + std::uint32_t line) { + using namespace std::string_literals; + + auto const s = clog->str(); + SCOPED_TRACE(::testing::Message() << __FILE__ << ":" << line); + EXPECT_NE(s.find("stalled"), s.npos) << s; + EXPECT_TRUE(s.starts_with("Transaction "s + std::to_string(txid))) << s; + EXPECT_NE(s.find("voting "s + (ourVote ? "YES" : "NO")), s.npos) << s; + EXPECT_NE(s.find("for "s + std::to_string(ourTime) + " rounds."s), s.npos) << s; + EXPECT_NE(s.find("votes in "s + std::to_string(peerTime) + " rounds."), s.npos) << s; + EXPECT_TRUE(s.ends_with("has "s + std::to_string(support) + "% support. "s)) << s; + clog = std::make_unique(); + }; + + for (int i = 0; i < 1; ++i) + { + EXPECT_TRUE(!proposingTrue.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!followingTrue.updateVote(250 + (10 * i), false, p)); + EXPECT_TRUE(!followingFalse.updateVote(250 + (10 * i), false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // true vote has changed recently, so not stalled + EXPECT_TRUE(!isStalled(proposingTrue, p, true, 0, clog)); + EXPECT_TRUE(clog->str().empty()); + // remaining votes have been unchanged in so long that we only + // need to hit the second round at 95% to be stalled, regardless + // of peers + EXPECT_TRUE(isStalled(proposingFalse, p, true, 0, clog)); + expectStalled(98, false, 11, 0, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, 0, clog)); + expectStalled(97, true, 11, 0, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, 0, clog)); + expectStalled(96, false, 11, 0, 3, __LINE__); + + // true vote has changed recently, so not stalled + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()) << clog->str(); + // remaining votes have been unchanged in so long that we only + // need to hit the second round at 95% to be stalled, regardless + // of peers + EXPECT_TRUE(isStalled(proposingFalse, p, true, peersUnchanged, clog)); + expectStalled(98, false, 11, 6, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, peersUnchanged, clog)); + expectStalled(97, true, 11, 6, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, peersUnchanged, clog)); + expectStalled(96, false, 11, 6, 3, __LINE__); + } + for (int i = 1; i < 3; ++i) + { + EXPECT_TRUE(!proposingTrue.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!followingTrue.updateVote(250 + (10 * i), false, p)); + EXPECT_TRUE(!followingFalse.updateVote(250 + (10 * i), false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // true vote changed 2 rounds ago, and peers are changing, so + // not stalled + EXPECT_TRUE(!isStalled(proposingTrue, p, true, 0, clog)); + EXPECT_TRUE(clog->str().empty()) << clog->str(); + // still stalled + EXPECT_TRUE(isStalled(proposingFalse, p, true, 0, clog)); + expectStalled(98, false, 11 + i, 0, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, 0, clog)); + expectStalled(97, true, 11 + i, 0, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, 0, clog)); + expectStalled(96, false, 11 + i, 0, 3, __LINE__); + + // true vote changed 2 rounds ago, and peers are NOT changing, + // so stalled + EXPECT_TRUE(isStalled(proposingTrue, p, true, peersUnchanged, clog)); + expectStalled(99, true, 1 + i, 6, 97, __LINE__); + // still stalled + EXPECT_TRUE(isStalled(proposingFalse, p, true, peersUnchanged, clog)); + expectStalled(98, false, 11 + i, 6, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, peersUnchanged, clog)); + expectStalled(97, true, 11 + i, 6, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, peersUnchanged, clog)); + expectStalled(96, false, 11 + i, 6, 3, __LINE__); + } + for (int i = 3; i < 5; ++i) + { + EXPECT_TRUE(!proposingTrue.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!followingTrue.updateVote(250 + (10 * i), false, p)); + EXPECT_TRUE(!followingFalse.updateVote(250 + (10 * i), false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + EXPECT_TRUE(isStalled(proposingTrue, p, true, 0, clog)); + expectStalled(99, true, 1 + i, 0, 97, __LINE__); + EXPECT_TRUE(isStalled(proposingFalse, p, true, 0, clog)); + expectStalled(98, false, 11 + i, 0, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, 0, clog)); + expectStalled(97, true, 11 + i, 0, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, 0, clog)); + expectStalled(96, false, 11 + i, 0, 3, __LINE__); + + EXPECT_TRUE(isStalled(proposingTrue, p, true, peersUnchanged, clog)); + expectStalled(99, true, 1 + i, 6, 97, __LINE__); + EXPECT_TRUE(isStalled(proposingFalse, p, true, peersUnchanged, clog)); + expectStalled(98, false, 11 + i, 6, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, peersUnchanged, clog)); + expectStalled(97, true, 11 + i, 6, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, peersUnchanged, clog)); + expectStalled(96, false, 11 + i, 6, 3, __LINE__); + } + } +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/DistributedValidatorsSim.cpp b/src/tests/libxrpl/consensus/DistributedValidatorsSim.cpp new file mode 100644 index 0000000000..f5ba508cbc --- /dev/null +++ b/src/tests/libxrpl/consensus/DistributedValidatorsSim.cpp @@ -0,0 +1,252 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::test { + +namespace { + +[[nodiscard]] std::string const& +arg() +{ + static std::string const kEMPTY; + return kEMPTY; +} + +void +completeTrustCompleteConnectFixedDelay( + std::size_t numPeers, + std::chrono::milliseconds delay = std::chrono::milliseconds(200), + bool printHeaders = false) +{ + using namespace csf; + using namespace std::chrono; + + // Initialize persistent collector logs specific to this method + std::string const prefix = + "DistributedValidators_" + "completeTrustCompleteConnectFixedDelay"; + std::fstream txLog(prefix + "_tx.csv", std::ofstream::app), + ledgerLog(prefix + "_ledger.csv", std::ofstream::app); + + // title + std::cout << prefix << "(" << numPeers << "," << delay.count() << ")" << std::endl; + + // number of peers, UNLs, connections + EXPECT_TRUE(numPeers >= 1); + + Sim sim; + PeerGroup peers = sim.createGroup(numPeers); + + // complete trust graph + peers.trust(peers); + + // complete connect graph with fixed delay + peers.connect(peers, delay); + + // Initialize collectors to track statistics to report + TxCollector txCollector; + LedgerCollector ledgerCollector; + auto colls = makeCollectors(txCollector, ledgerCollector); + sim.collectors.add(colls); + + // Initial round to set prior state + sim.run(1); + + // Run for 10 minutes, submitting 100 tx/second + std::chrono::nanoseconds const simDuration = 10min; + std::chrono::nanoseconds const quiet = 10s; + Rate const rate{.count = 100, .duration = 1000ms}; + + // Initialize timers + HeartbeatTimer heart(sim.scheduler); + + // txs, start/stop/step, target + auto peerSelector = + makeSelector(peers.begin(), peers.end(), std::vector(numPeers, 1.), sim.rng); + auto txSubmitter = makeSubmitter( + ConstantDistribution{rate.inv()}, + sim.scheduler.now() + quiet, + sim.scheduler.now() + simDuration - quiet, + peerSelector, + sim.scheduler, + sim.rng); + + // run simulation for given duration + heart.start(); + sim.run(simDuration); + + // EXPECT_TRUE(sim.branches() == 1); + // EXPECT_TRUE(sim.synchronized()); + + std::cout << std::right; + std::cout << "| Peers: " << std::setw(2) << peers.size(); + std::cout << " | Duration: " << std::setw(6) << duration_cast(simDuration).count() + << " ms"; + std::cout << " | Branches: " << std::setw(1) << sim.branches(); + std::cout << " | Synchronized: " << std::setw(1) << (sim.synchronized() ? "Y" : "N"); + std::cout << " |" << std::endl; + + txCollector.report(simDuration, std::cout, true); + ledgerCollector.report(simDuration, std::cout, false); + + std::string const tag = std::to_string(numPeers); + txCollector.csv(simDuration, txLog, tag, printHeaders); + ledgerCollector.csv(simDuration, ledgerLog, tag, printHeaders); + + std::cout << std::endl; +} + +void +completeTrustScaleFreeConnectFixedDelay( + std::size_t numPeers, + std::chrono::milliseconds delay = std::chrono::milliseconds(200), + bool printHeaders = false) +{ + using namespace csf; + using namespace std::chrono; + + // Initialize persistent collector logs specific to this method + std::string const prefix = + "DistributedValidators__" + "completeTrustScaleFreeConnectFixedDelay"; + std::fstream txLog(prefix + "_tx.csv", std::ofstream::app), + ledgerLog(prefix + "_ledger.csv", std::ofstream::app); + + // title + std::cout << prefix << "(" << numPeers << "," << delay.count() << ")" << std::endl; + + // number of peers, UNLs, connections + int const numCNLs = std::max(int(1.00 * numPeers), 1); + int const minCNLSize = std::max(int(0.25 * numCNLs), 1); + int const maxCNLSize = std::max(int(0.50 * numCNLs), 1); + EXPECT_TRUE(numPeers >= 1); + EXPECT_TRUE(numCNLs >= 1); + EXPECT_TRUE(1 <= minCNLSize && minCNLSize <= maxCNLSize && maxCNLSize <= numPeers); + + Sim sim; + PeerGroup peers = sim.createGroup(numPeers); + + // complete trust graph + peers.trust(peers); + + // scale-free connect graph with fixed delay + std::vector const ranks = sample(peers.size(), PowerLawDistribution{1, 3}, sim.rng); + randomRankedConnect( + peers, + ranks, + numCNLs, + std::uniform_int_distribution<>{minCNLSize, maxCNLSize}, + sim.rng, + delay); + + // Initialize collectors to track statistics to report + TxCollector txCollector; + LedgerCollector ledgerCollector; + auto colls = makeCollectors(txCollector, ledgerCollector); + sim.collectors.add(colls); + + // Initial round to set prior state + sim.run(1); + + // Run for 10 minutes, submitting 100 tx/second + std::chrono::nanoseconds const simDuration = 10min; + std::chrono::nanoseconds const quiet = 10s; + Rate const rate{.count = 100, .duration = 1000ms}; + + // Initialize timers + HeartbeatTimer heart(sim.scheduler); + + // txs, start/stop/step, target + auto peerSelector = + makeSelector(peers.begin(), peers.end(), std::vector(numPeers, 1.), sim.rng); + auto txSubmitter = makeSubmitter( + ConstantDistribution{rate.inv()}, + sim.scheduler.now() + quiet, + sim.scheduler.now() + simDuration - quiet, + peerSelector, + sim.scheduler, + sim.rng); + + // run simulation for given duration + heart.start(); + sim.run(simDuration); + + // EXPECT_TRUE(sim.branches() == 1); + // EXPECT_TRUE(sim.synchronized()); + + std::cout << std::right; + std::cout << "| Peers: " << std::setw(2) << peers.size(); + std::cout << " | Duration: " << std::setw(6) << duration_cast(simDuration).count() + << " ms"; + std::cout << " | Branches: " << std::setw(1) << sim.branches(); + std::cout << " | Synchronized: " << std::setw(1) << (sim.synchronized() ? "Y" : "N"); + std::cout << " |" << std::endl; + + txCollector.report(simDuration, std::cout, true); + ledgerCollector.report(simDuration, std::cout, false); + + std::string const tag = std::to_string(numPeers); + txCollector.csv(simDuration, txLog, tag, printHeaders); + ledgerCollector.csv(simDuration, ledgerLog, tag, printHeaders); + + std::cout << std::endl; +} + +} // namespace + +// In progress simulations for diversifying and distributing validators +TEST(DistributedValidatorsTest, DISABLED_distributed_validators) +{ + std::string const defaultArgs = "5 200"; + std::string const args = arg().empty() ? defaultArgs : arg(); + std::stringstream argStream(args); + + int maxNumValidators = 0; + int delayCount(200); + argStream >> maxNumValidators; + argStream >> delayCount; + + std::chrono::milliseconds const delay(delayCount); + + std::cout << "DistributedValidators: 1 to " << maxNumValidators << " Peers" << std::endl; + + // Simulate with N = 1 to N + // - complete trust graph is complete + // - complete network connectivity + // - fixed delay for network links + completeTrustCompleteConnectFixedDelay(1, delay, true); + for (int i = 2; i <= maxNumValidators; i++) + { + completeTrustCompleteConnectFixedDelay(i, delay); + } + + // Simulate with N = 1 to N + // - complete trust graph is complete + // - scale-free network connectivity + // - fixed delay for network links + completeTrustScaleFreeConnectFixedDelay(1, delay, true); + for (int i = 2; i <= maxNumValidators; i++) + { + completeTrustScaleFreeConnectFixedDelay(i, delay); + } +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/LedgerTiming.cpp b/src/tests/libxrpl/consensus/LedgerTiming.cpp new file mode 100644 index 0000000000..0cab895fda --- /dev/null +++ b/src/tests/libxrpl/consensus/LedgerTiming.cpp @@ -0,0 +1,105 @@ +#include + +#include + +#include + +#include +#include +#include + +namespace xrpl::test { + +TEST(LedgerTimingTest, get_next_ledger_time_resolution) +{ + // helper to iteratively call into getNextLedgerTimeResolution + struct TestRes + { + std::uint32_t decrease = 0; + std::uint32_t equal = 0; + std::uint32_t increase = 0; + + static TestRes + run(bool previousAgree, std::uint32_t rounds) + { + TestRes res; + auto closeResolution = kLedgerDefaultTimeResolution; + auto nextCloseResolution = closeResolution; + std::uint32_t round = 0; + do + { + nextCloseResolution = + getNextLedgerTimeResolution(closeResolution, previousAgree, ++round); + if (nextCloseResolution < closeResolution) + { + ++res.decrease; + } + else if (nextCloseResolution > closeResolution) + { + ++res.increase; + } + else + { + ++res.equal; + } + std::swap(nextCloseResolution, closeResolution); + } while (round < rounds); + return res; + } + }; + + // If we never agree on close time, only can increase resolution + // until hit the max + auto decreases = TestRes::run(false, 10); + EXPECT_TRUE(decreases.increase == 3); + EXPECT_TRUE(decreases.decrease == 0); + EXPECT_TRUE(decreases.equal == 7); + + // If we always agree on close time, only can decrease resolution + // until hit the min + auto increases = TestRes::run(false, 100); + EXPECT_TRUE(increases.increase == 3); + EXPECT_TRUE(increases.decrease == 0); + EXPECT_TRUE(increases.equal == 97); +} + +TEST(LedgerTimingTest, round_close_time) +{ + using namespace std::chrono_literals; + // A closeTime equal to the epoch is not modified + using tp = NetClock::time_point; + tp const def; + EXPECT_TRUE(def == roundCloseTime(def, 30s)); + + // Otherwise, the closeTime is rounded to the nearest + // rounding up on ties + EXPECT_TRUE(tp{0s} == roundCloseTime(tp{29s}, 60s)); + EXPECT_TRUE(tp{30s} == roundCloseTime(tp{30s}, 1s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{31s}, 60s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{30s}, 60s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{59s}, 60s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{60s}, 60s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{61s}, 60s)); +} + +TEST(LedgerTimingTest, eff_close_time) +{ + using namespace std::chrono_literals; + using tp = NetClock::time_point; + tp close = effCloseTime(tp{10s}, 30s, tp{0s}); + EXPECT_TRUE(close == tp{1s}); + + close = effCloseTime(tp{16s}, 30s, tp{0s}); + EXPECT_TRUE(close == tp{30s}); + + close = effCloseTime(tp{16s}, 30s, tp{30s}); + EXPECT_TRUE(close == tp{31s}); + + close = effCloseTime(tp{16s}, 30s, tp{60s}); + EXPECT_TRUE(close == tp{61s}); + + close = effCloseTime(tp{31s}, 30s, tp{0s}); + EXPECT_TRUE(close == tp{30s}); +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/LedgerTrie.cpp b/src/tests/libxrpl/consensus/LedgerTrie.cpp new file mode 100644 index 0000000000..1259a5049a --- /dev/null +++ b/src/tests/libxrpl/consensus/LedgerTrie.cpp @@ -0,0 +1,693 @@ +#include + +#include +#include + +#include +#include +#include + +namespace xrpl::test { + +TEST(LedgerTrieTest, insert) +{ + using namespace csf; + // Single entry by itself + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + + t.insert(h["abc"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 2); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + } + // Suffix of existing (extending tree) + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + EXPECT_TRUE(t.checkInvariants()); + // extend with no siblings + t.insert(h["abcd"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + + // extend with existing sibling + t.insert(h["abce"]); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 3); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abce"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abce"]) == 1); + } + // uncommitted of existing node + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + EXPECT_TRUE(t.checkInvariants()); + // uncommitted with no siblings + t.insert(h["abcdf"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcdf"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcdf"]) == 1); + + // uncommitted with existing child + t.insert(h["abc"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 3); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcdf"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcdf"]) == 1); + } + // Suffix + uncommitted of existing node + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + EXPECT_TRUE(t.checkInvariants()); + t.insert(h["abce"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abce"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abce"]) == 1); + } + // Suffix + uncommitted with existing child + { + // abcd : abcde, abcf + + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + EXPECT_TRUE(t.checkInvariants()); + t.insert(h["abcde"]); + EXPECT_TRUE(t.checkInvariants()); + t.insert(h["abcf"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 3); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcf"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcf"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abcde"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcde"]) == 1); + } + + // Multiple counts + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"], 4); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 4); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 4); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.branchSupport(h["a"]) == 4); + + t.insert(h["abc"], 2); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 2); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 4); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 6); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.branchSupport(h["a"]) == 6); + } +} + +TEST(LedgerTrieTest, remove) +{ + using namespace csf; + // Not in trie + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + + EXPECT_TRUE(!t.remove(h["ab"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(!t.remove(h["a"])); + EXPECT_TRUE(t.checkInvariants()); + } + // In trie but with 0 tip support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + t.insert(h["abce"]); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(!t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + } + // In trie with > 1 tip support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"], 2); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 2); + EXPECT_TRUE(t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + + t.insert(h["abc"], 1); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 2); + EXPECT_TRUE(t.remove(h["abc"], 2)); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + + t.insert(h["abc"], 3); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 3); + EXPECT_TRUE(t.remove(h["abc"], 300)); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + } + // In trie with = 1 tip support, no children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + + EXPECT_TRUE(t.tipSupport(h["ab"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + + EXPECT_TRUE(t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 0); + } + // In trie with = 1 tip support, 1 child + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + t.insert(h["abcd"]); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + + EXPECT_TRUE(t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + } + // In trie with = 1 tip support, > 1 children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + t.insert(h["abcd"]); + t.insert(h["abce"]); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 3); + + EXPECT_TRUE(t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + } + + // In trie with = 1 tip support, parent compaction + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + t.insert(h["abd"]); + EXPECT_TRUE(t.checkInvariants()); + t.remove(h["ab"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abd"]) == 1); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 0); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 2); + + t.remove(h["abd"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 1); + } +} + +TEST(LedgerTrieTest, empty) +{ + using namespace csf; + LedgerTrie t; + LedgerHistoryHelper h; + EXPECT_TRUE(t.empty()); + + Ledger const genesis = h[""]; + t.insert(genesis); + EXPECT_TRUE(!t.empty()); + t.remove(genesis); + EXPECT_TRUE(t.empty()); + + t.insert(h["abc"]); + EXPECT_TRUE(!t.empty()); + t.remove(h["abc"]); + EXPECT_TRUE(t.empty()); +} + +TEST(LedgerTrieTest, support) +{ + using namespace csf; + + LedgerTrie t; + LedgerHistoryHelper h; + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.tipSupport(h["axy"]) == 0); + + EXPECT_TRUE(t.branchSupport(h["a"]) == 0); + EXPECT_TRUE(t.branchSupport(h["axy"]) == 0); + + t.insert(h["abc"]); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 0); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 0); + + EXPECT_TRUE(t.branchSupport(h["a"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 0); + + t.insert(h["abe"]); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 0); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abe"]) == 1); + + EXPECT_TRUE(t.branchSupport(h["a"]) == 2); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 2); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abe"]) == 1); + + t.remove(h["abc"]); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 0); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.tipSupport(h["abe"]) == 1); + + EXPECT_TRUE(t.branchSupport(h["a"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abe"]) == 1); +} + +TEST(LedgerTrieTest, get_preferred) +{ + using namespace csf; + using Seq = Ledger::Seq; + // Empty + { + LedgerTrie const t; + EXPECT_TRUE(t.getPreferred(Seq{0}) == std::nullopt); + EXPECT_TRUE(t.getPreferred(Seq{2}) == std::nullopt); + } + // Genesis support is NOT empty + { + LedgerTrie t; + LedgerHistoryHelper h; + Ledger const genesis = h[""]; + t.insert(genesis); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{0})->id == genesis.id()); + EXPECT_TRUE(t.remove(genesis)); + EXPECT_TRUE(t.getPreferred(Seq{0}) == std::nullopt); + EXPECT_TRUE(!t.remove(genesis)); + } + // Single node no children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + } + // Single node smaller child support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"]); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + } + // Single node larger child + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"], 2); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abcd"].id()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcd"].id()); + } + // Single node smaller children support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"]); + t.insert(h["abce"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + + t.insert(h["abc"]); + + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + // Single node larger children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"], 2); + t.insert(h["abce"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + + t.insert(h["abcd"]); + + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abcd"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcd"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + // Tie-breaker by id + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"], 2); + t.insert(h["abce"], 2); + + EXPECT_TRUE(h["abce"].id() > h["abcd"].id()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abce"].id()); + + t.insert(h["abcd"]); + EXPECT_TRUE(h["abce"].id() > h["abcd"].id()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcd"].id()); + } + + // Tie-breaker not needed + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"]); + t.insert(h["abce"], 2); + // abce only has a margin of 1, but it owns the tie-breaker + EXPECT_TRUE(h["abce"].id() > h["abcd"].id()); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abce"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abce"].id()); + + // Switch support from abce to abcd, tie-breaker now needed + t.remove(h["abce"]); + t.insert(h["abcd"]); + + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + + // Single node larger grand child + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"], 2); + t.insert(h["abcde"], 4); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abcde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["abcde"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + + // Too much uncommitted support from competing branches + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcde"], 2); + t.insert(h["abcfg"], 2); + // 'de' and 'fg' are tied without 'abc' vote + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["abc"].id()); + + t.remove(h["abc"]); + t.insert(h["abcd"]); + + // 'de' branch has 3 votes to 2, so earlier sequences see it as preferred + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abcde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcde"].id()); + + // However, if you validated a ledger with Seq 5, potentially on + // a different branch, you do not yet know if they chose abcd + // or abcf because of you, so abc remains preferred + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["abc"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + + // Changing largestSeq perspective changes preferred branch + { + /** + * Build the tree below with initial tip support annotated + * A + * / \ + * B(1) C(1) + * / | | + * H D F(1) + * | + * E(2) + * | + * G + */ + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["ac"]); + t.insert(h["acf"]); + t.insert(h["abde"], 2); + + // B has more branch support + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{1})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{2})->id == h["ab"].id()); + + // But if you last validated D,F or E, you do not yet know + // if someone used that validation to commit to B or C + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["a"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["a"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + + /** + * One of E advancing to G doesn't change anything + * A + * / \ + * B(1) C(1) + * / | | + * H D F(1) + * | + * E(1) + * | + * G(1) + */ + t.remove(h["abde"]); + t.insert(h["abdeg"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{1})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{2})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["a"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["a"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["a"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + + /** + * C advancing to H does advance the seq 3 preferred ledger + * A + * / \ + * B(1) C + * / | | + * H(1)D F(1) + * | + * E(1) + * | + * G(1) + */ + t.remove(h["ac"]); + t.insert(h["abh"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{1})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{2})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["a"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["a"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + + /** + * F advancing to E also moves the preferred ledger forward + * A + * / \ + * B(1) C + * / | | + * H(1)D F + * | + * E(2) + * | + * G(1) + */ + t.remove(h["acf"]); + t.insert(h["abde"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{1})->id == h["abde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{2})->id == h["abde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["ab"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } +} + +TEST(LedgerTrieTest, root_related) +{ + using namespace csf; + // Since the root is a special node that breaks the no-single child + // invariant, do some tests that exercise it. + + LedgerTrie t; + LedgerHistoryHelper h; + EXPECT_TRUE(!t.remove(h[""])); + EXPECT_TRUE(t.branchSupport(h[""]) == 0); + EXPECT_TRUE(t.tipSupport(h[""]) == 0); + + t.insert(h["a"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.branchSupport(h[""]) == 1); + EXPECT_TRUE(t.tipSupport(h[""]) == 0); + + t.insert(h["e"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.branchSupport(h[""]) == 2); + EXPECT_TRUE(t.tipSupport(h[""]) == 0); + + EXPECT_TRUE(t.remove(h["e"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.branchSupport(h[""]) == 1); + EXPECT_TRUE(t.tipSupport(h[""]) == 0); +} + +TEST(LedgerTrieTest, stress) +{ + using namespace csf; + LedgerTrie t; + LedgerHistoryHelper h; + + // Test quasi-randomly add/remove supporting for different ledgers + // from a branching history. + + // Ledgers have sequence 1,2,3,4 + std::uint32_t const depthConst = 4; + // Each ledger has 4 possible children + std::uint32_t const width = 4; + + std::uint32_t const iterations = 10000; + + // Use explicit seed to have same results for CI + // NOLINTNEXTLINE(bugprone-random-generator-seed): fixed seed for reproducible test + std::mt19937 gen{42}; + std::uniform_int_distribution<> depthDist(0, depthConst - 1); + std::uniform_int_distribution<> widthDist(0, width - 1); + std::uniform_int_distribution<> flip(0, 1); + for (std::uint32_t i = 0; i < iterations; ++i) + { + // pick a random ledger history + std::string curr; + char const depth = depthDist(gen); + char offset = 0; + for (char d = 0; d < depth; ++d) + { + char const a = offset + widthDist(gen); + curr += a; + offset = (a + 1) * width; + } + + // 50-50 to add remove + if (flip(gen) == 0) + { + t.insert(h[curr]); + } + else + { + t.remove(h[curr]); + } + EXPECT_TRUE(t.checkInvariants()); + if (!(t.checkInvariants())) + return; + } +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/ScaleFreeSim.cpp b/src/tests/libxrpl/consensus/ScaleFreeSim.cpp new file mode 100644 index 0000000000..2b07b29900 --- /dev/null +++ b/src/tests/libxrpl/consensus/ScaleFreeSim.cpp @@ -0,0 +1,100 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace xrpl::test { + +TEST(ScaleFreeSimTest, DISABLED_scale_free_sim) +{ + using namespace std::chrono; + using namespace csf; + + std::ostream& log = std::cout; + + // Generate a quasi-random scale free network and simulate consensus + // as we vary transaction submission rates + + int const n = 100; // Peers + + int const numUNLs = 15; // UNL lists + int const minUNLSize = n / 4, maxUNLSize = n / 2; + + ConsensusParms const parms{}; + Sim sim; + PeerGroup network = sim.createGroup(n); + + // generate trust ranks + std::vector const ranks = sample(network.size(), PowerLawDistribution{1, 3}, sim.rng); + + // generate scale-free trust graph + randomRankedTrust( + network, ranks, numUNLs, std::uniform_int_distribution<>{minUNLSize, maxUNLSize}, sim.rng); + + // nodes with a trust line in either direction are network-connected + network.connectFromTrust(round(0.2 * parms.ledgerGRANULARITY)); + + // Initialize collectors to track statistics to report + TxCollector txCollector; + LedgerCollector ledgerCollector; + auto colls = makeCollectors(txCollector, ledgerCollector); + sim.collectors.add(colls); + + // Initial round to set prior state + sim.run(1); + + // Initialize timers + HeartbeatTimer heart(sim.scheduler, seconds(10s)); + + // Run for 10 minutes, submitting 100 tx/second + std::chrono::nanoseconds const simDuration = 10min; + std::chrono::nanoseconds const quiet = 10s; + Rate const rate{.count = 100, .duration = 1000ms}; + + // txs, start/stop/step, target + auto peerSelector = makeSelector(network.begin(), network.end(), ranks, sim.rng); + auto txSubmitter = makeSubmitter( + ConstantDistribution{rate.inv()}, + sim.scheduler.now() + quiet, + sim.scheduler.now() + (simDuration - quiet), + peerSelector, + sim.scheduler, + sim.rng); + + // run simulation for given duration + heart.start(); + sim.run(simDuration); + + EXPECT_TRUE(sim.branches() == 1); + EXPECT_TRUE(sim.synchronized()); + + // TODO: Clean up this formatting mess!! + + log << "Peers: " << network.size() << std::endl; + log << "Simulated Duration: " << duration_cast(simDuration).count() << " ms" + << std::endl; + log << "Branches: " << sim.branches() << std::endl; + log << "Synchronized: " << (sim.synchronized() ? "Y" : "N") << std::endl; + log << std::endl; + + txCollector.report(simDuration, log); + ledgerCollector.report(simDuration, log); + // Print summary? + // # forks? # of LCLs? + // # peers + // # tx submitted + // # ledgers/sec etc.? +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/Validations.cpp b/src/tests/libxrpl/consensus/Validations.cpp new file mode 100644 index 0000000000..56964f56a6 --- /dev/null +++ b/src/tests/libxrpl/consensus/Validations.cpp @@ -0,0 +1,1030 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::test::csf { + +namespace { + +beast::Journal +journal() +{ + return beast::Journal{TestSink::instance()}; +} + +template +void +expireValidations(ValidationStore& validations) +{ + auto j = journal(); + validations.expire(j); +} + +using clock_type = beast::AbstractClock const; + +// Helper to convert steady_clock to a reasonable NetClock +// This allows a single manual clock in the unit tests +NetClock::time_point +toNetClock(clock_type const& c) +{ + // We don't care about the actual epochs, but do want the + // generated NetClock time to be well past its epoch to ensure + // any subtractions are positive + using namespace std::chrono; + return NetClock::time_point( + duration_cast(c.now().time_since_epoch() + 86400s)); +} + +// Represents a node that can issue validations +class Node +{ + clock_type const& c_; + PeerID nodeID_; + bool trusted_ = true; + std::size_t signIdx_{1}; + std::optional loadFee_; + +public: + Node(PeerID nodeID, clock_type const& c) : c_(c), nodeID_(nodeID) + { + } + + void + untrust() + { + trusted_ = false; + } + + void + trust() + { + trusted_ = true; + } + + void + setLoadFee(std::uint32_t fee) + { + loadFee_ = fee; + } + + [[nodiscard]] PeerID + nodeID() const + { + return nodeID_; + } + + void + advanceKey() + { + signIdx_++; + } + + [[nodiscard]] PeerKey + currKey() const + { + return std::make_pair(nodeID_, signIdx_); + } + + [[nodiscard]] PeerKey + masterKey() const + { + return std::make_pair(nodeID_, 0); + } + [[nodiscard]] NetClock::time_point + now() const + { + return toNetClock(c_); + } + + // Issue a new validation with given sequence number and id and + // with signing and seen times offset from the common clock + [[nodiscard]] Validation + validate( + Ledger::ID id, + Ledger::Seq seq, + NetClock::duration signOffset, + NetClock::duration seenOffset, + bool full) const + { + Validation v{ + id, seq, now() + signOffset, now() + seenOffset, currKey(), nodeID_, full, loadFee_}; + if (trusted_) + v.setTrusted(); + return v; + } + + [[nodiscard]] Validation + validate(Ledger ledger, NetClock::duration signOffset, NetClock::duration seenOffset) const + { + return validate(ledger.id(), ledger.seq(), signOffset, seenOffset, true); + } + + [[nodiscard]] Validation + validate(Ledger ledger) const + { + return validate( + ledger.id(), ledger.seq(), NetClock::duration{0}, NetClock::duration{0}, true); + } + + [[nodiscard]] Validation + partial(Ledger ledger) const + { + return validate( + ledger.id(), ledger.seq(), NetClock::duration{0}, NetClock::duration{0}, false); + } +}; + +// Generic Validations adaptor +class Adaptor +{ + clock_type& c_; + LedgerOracle& oracle_; + +public: + // Non-locking mutex to avoid locks in generic Validations + struct Mutex + { + void + lock() + { + } + + void + unlock() + { + } + }; + + using Validation = csf::Validation; + using Ledger = csf::Ledger; + + Adaptor(clock_type& c, LedgerOracle& o) : c_{c}, oracle_{o} + { + } + + [[nodiscard]] NetClock::time_point + now() const + { + return toNetClock(c_); + } + + std::optional + acquire(Ledger::ID const& id) + { + return oracle_.lookup(id); + } +}; + +// Specialize generic Validations using the above types +using TestValidations = Validations; + +// Gather the dependencies of TestValidations in a single class and provide +// accessors for simplifying test logic +class TestHarness +{ + ValidationParms p_; + beast::ManualClock clock_; + TestValidations tv_; + PeerID nextNodeId_{0}; + +public: + explicit TestHarness(LedgerOracle& o) : tv_(p_, clock_, clock_, o) + { + } + + ValStatus + add(Validation const& v) + { + return tv_.add(v.nodeID(), v); + } + + TestValidations& + vals() + { + return tv_; + } + + Node + makeNode() + { + return Node(nextNodeId_++, clock_); + } + + ValidationParms + parms() const + { + return p_; + } + + auto& + clock() + { + return clock_; + } +}; + +Ledger const kGenesisLedger{Ledger::MakeGenesis{}}; + +} // namespace + +TEST(ValidationsTest, add_validation) +{ + using namespace std::chrono_literals; + + SCOPED_TRACE("Add validation"); + LedgerHistoryHelper h; + Ledger const ledgerA = h["a"]; + Ledger ledgerAB = h["ab"]; + Ledger ledgerAZ = h["az"]; + Ledger ledgerABC = h["abc"]; + Ledger const ledgerABCD = h["abcd"]; + Ledger const ledgerABCDE = h["abcde"]; + + { + TestHarness harness(h.oracle); + Node n = harness.makeNode(); + + auto const v = n.validate(ledgerA); + + // Add a current validation + EXPECT_TRUE(ValStatus::Current == harness.add(v)); + + // Re-adding violates the increasing seq requirement for full + // validations + EXPECT_TRUE(ValStatus::BadSeq == harness.add(v)); + + harness.clock().advance(1s); + + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerAB))); + + // Test the node changing signing key + + // Confirm old ledger on hand, but not new ledger + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerAB.id()) == 1); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerABC.id()) == 0); + + // Rotate signing keys + n.advanceKey(); + + harness.clock().advance(1s); + + // Cannot re-do the same full validation sequence + EXPECT_TRUE(ValStatus::Conflicting == harness.add(n.validate(ledgerAB))); + // Cannot send the same partial validation sequence + EXPECT_TRUE(ValStatus::Conflicting == harness.add(n.partial(ledgerAB))); + + // Now trusts the newest ledger too + harness.clock().advance(1s); + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerABC))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerAB.id()) == 1); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerABC.id()) == 1); + + // Processing validations out of order should ignore the older + // validation + harness.clock().advance(2s); + auto const valABCDE = n.validate(ledgerABCDE); + + harness.clock().advance(4s); + auto const valABCD = n.validate(ledgerABCD); + + EXPECT_TRUE(ValStatus::Current == harness.add(valABCD)); + + EXPECT_TRUE(ValStatus::Stale == harness.add(valABCDE)); + } + + { + // Process validations out of order with shifted times + + TestHarness harness(h.oracle); + Node const n = harness.makeNode(); + + // Establish a new current validation + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerA))); + + // Process a validation that has "later" seq but early sign time + EXPECT_TRUE(ValStatus::Stale == harness.add(n.validate(ledgerAB, -1s, -1s))); + + // Process a validation that has a later seq and later sign + // time + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerABC, 1s, 1s))); + } + + { + // Test stale on arrival validations + TestHarness harness(h.oracle); + Node const n = harness.makeNode(); + + EXPECT_TRUE( + ValStatus::Stale == + harness.add(n.validate(ledgerA, -harness.parms().validationCurrentEarly, 0s))); + + EXPECT_TRUE( + ValStatus::Stale == + harness.add(n.validate(ledgerA, harness.parms().validationCurrentWall, 0s))); + + EXPECT_TRUE( + ValStatus::Stale == + harness.add(n.validate(ledgerA, 0s, harness.parms().validationCurrentLocal))); + } + + { + // Test that full or partials cannot be sent for older sequence + // numbers, unless time-out has happened + for (bool doFull : {true, false}) + { + TestHarness harness(h.oracle); + Node n = harness.makeNode(); + + auto process = [&](Ledger& lgr) { + if (doFull) + return harness.add(n.validate(lgr)); + return harness.add(n.partial(lgr)); + }; + + EXPECT_TRUE(ValStatus::Current == process(ledgerABC)); + harness.clock().advance(1s); + EXPECT_TRUE(ledgerAB.seq() < ledgerABC.seq()); + EXPECT_TRUE(ValStatus::BadSeq == process(ledgerAB)); + + // If we advance far enough for AB to expire, we can fully + // validate or partially validate that sequence number again + EXPECT_TRUE(ValStatus::Conflicting == process(ledgerAZ)); + harness.clock().advance(harness.parms().validationSetExpires + 1ms); + EXPECT_TRUE(ValStatus::Current == process(ledgerAZ)); + } + } +} + +TEST(ValidationsTest, on_stale) +{ + SCOPED_TRACE("Stale validation"); + // Verify validation becomes stale based solely on time passing, but + // use different functions to trigger the check for staleness + + LedgerHistoryHelper h; + Ledger ledgerA = h["a"]; + Ledger const ledgerAB = h["ab"]; + + using Trigger = std::function; + + std::vector const triggers = { + [&](TestValidations& vals) { vals.currentTrusted(); }, + [&](TestValidations& vals) { vals.getCurrentNodeIDs(); }, + [&](TestValidations& vals) { vals.getPreferred(kGenesisLedger); }, + [&](TestValidations& vals) { vals.getNodesAfter(ledgerA, ledgerA.id()); }}; + for (Trigger const& trigger : triggers) + { + TestHarness harness(h.oracle); + Node const n = harness.makeNode(); + + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerAB))); + trigger(harness.vals()); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 1); + EXPECT_TRUE( + harness.vals().getPreferred(kGenesisLedger) == + std::make_pair(ledgerAB.seq(), ledgerAB.id())); + harness.clock().advance(harness.parms().validationCurrentLocal); + + // trigger check for stale + trigger(harness.vals()); + + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 0); + EXPECT_TRUE(harness.vals().getPreferred(kGenesisLedger) == std::nullopt); + } +} + +TEST(ValidationsTest, get_nodes_after) +{ + // Test getting number of nodes working on a validation descending + // a prescribed one. This count should only be for trusted nodes, but + // includes partial and full validations + + using namespace std::chrono_literals; + SCOPED_TRACE("Get nodes after"); + + LedgerHistoryHelper h; + Ledger const ledgerA = h["a"]; + Ledger const ledgerAB = h["ab"]; + Ledger const ledgerABC = h["abc"]; + Ledger const ledgerAD = h["ad"]; + + TestHarness harness(h.oracle); + Node const trustedNode1 = harness.makeNode(); + Node const trustedNode2 = harness.makeNode(); + Node const trustedNode3 = harness.makeNode(); + + Node notTrustedNode = harness.makeNode(); + notTrustedNode.untrust(); + + // first round a,b,c agree, d has is partial + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode1.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode2.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode3.partial(ledgerA))); + + for (Ledger const& ledger : {ledgerA, ledgerAB, ledgerABC, ledgerAD}) + EXPECT_TRUE(harness.vals().getNodesAfter(ledger, ledger.id()) == 0); + + harness.clock().advance(5s); + + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode1.validate(ledgerAB))); + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode2.validate(ledgerABC))); + EXPECT_TRUE(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerAB))); + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode3.partial(ledgerABC))); + + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 3); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerAB, ledgerAB.id()) == 2); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerABC, ledgerABC.id()) == 0); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerAD, ledgerAD.id()) == 0); + + // If given a ledger inconsistent with the id, is still able to check using slower method + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerAD, ledgerA.id()) == 1); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerAD, ledgerAB.id()) == 2); +} + +TEST(ValidationsTest, current_trusted) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Current trusted validations"); + + LedgerHistoryHelper h; + Ledger const ledgerA = h["a"]; + Ledger const ledgerB = h["b"]; + Ledger const ledgerAC = h["ac"]; + + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Node b = harness.makeNode(); + b.untrust(); + + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(b.validate(ledgerB))); + + // Only a is trusted + EXPECT_TRUE(harness.vals().currentTrusted().size() == 1); + EXPECT_TRUE(harness.vals().currentTrusted()[0].ledgerID() == ledgerA.id()); + EXPECT_TRUE(harness.vals().currentTrusted()[0].seq() == ledgerA.seq()); + + harness.clock().advance(3s); + + for (auto const& node : {a, b}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.validate(ledgerAC))); + + // New validation for a + EXPECT_TRUE(harness.vals().currentTrusted().size() == 1); + EXPECT_TRUE(harness.vals().currentTrusted()[0].ledgerID() == ledgerAC.id()); + EXPECT_TRUE(harness.vals().currentTrusted()[0].seq() == ledgerAC.seq()); + + // Pass enough time for it to go stale + harness.clock().advance(harness.parms().validationCurrentLocal); + EXPECT_TRUE(harness.vals().currentTrusted().empty()); +} + +TEST(ValidationsTest, get_current_public_keys) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Current public keys"); + + LedgerHistoryHelper h; + Ledger const ledgerA = h["a"]; + Ledger const ledgerAC = h["ac"]; + + TestHarness harness(h.oracle); + Node a = harness.makeNode(), b = harness.makeNode(); + b.untrust(); + + for (auto const& node : {a, b}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.validate(ledgerA))); + + { + hash_set const expectedKeys = {a.nodeID(), b.nodeID()}; + EXPECT_TRUE(harness.vals().getCurrentNodeIDs() == expectedKeys); + } + + harness.clock().advance(3s); + + // Change keys and issue partials + a.advanceKey(); + b.advanceKey(); + + for (auto const& node : {a, b}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.partial(ledgerAC))); + + { + hash_set const expectedKeys = {a.nodeID(), b.nodeID()}; + EXPECT_TRUE(harness.vals().getCurrentNodeIDs() == expectedKeys); + } + + // Pass enough time for them to go stale + harness.clock().advance(harness.parms().validationCurrentLocal); + EXPECT_TRUE(harness.vals().getCurrentNodeIDs().empty()); +} + +TEST(ValidationsTest, trusted_by_ledger_functions) +{ + // Test the Validations functions that calculate a value by ledger ID + using namespace std::chrono_literals; + SCOPED_TRACE("By ledger functions"); + + // Several Validations functions return a set of values associated + // with trusted ledgers sharing the same ledger ID. The tests below + // exercise this logic by saving the set of trusted Validations, and + // verifying that the Validations member functions all calculate the + // proper transformation of the available ledgers. + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + + Node a = harness.makeNode(), b = harness.makeNode(), c = harness.makeNode(), + d = harness.makeNode(), e = harness.makeNode(); + + c.untrust(); + // Mix of load fees + a.setLoadFee(12); + b.setLoadFee(1); + c.setLoadFee(12); + e.setLoadFee(12); + + hash_map, std::vector> trustedValidations; + + //---------------------------------------------------------------------- + // checkers + auto sorted = [](auto vec) { + std::sort(vec.begin(), vec.end()); + return vec; + }; + auto compare = [&]() { + for (auto& it : trustedValidations) + { + auto const& id = it.first.first; + auto const& seq = it.first.second; + auto const& expectedValidations = it.second; + + EXPECT_TRUE(harness.vals().numTrustedForLedger(id) == expectedValidations.size()); + EXPECT_TRUE( + sorted(harness.vals().getTrustedForLedger(id, seq)) == sorted(expectedValidations)); + + std::uint32_t const baseFee = 0; + std::vector expectedFees; + expectedFees.reserve(expectedValidations.size()); + for (auto const& val : expectedValidations) + { + expectedFees.push_back(val.loadFee().value_or(baseFee)); + } + + EXPECT_TRUE(sorted(harness.vals().fees(id, baseFee)) == sorted(expectedFees)); + } + }; + + //---------------------------------------------------------------------- + Ledger const ledgerA = h["a"]; + Ledger const ledgerB = h["b"]; + Ledger const ledgerAC = h["ac"]; + + // Add a dummy ID to cover unknown ledger identifiers + trustedValidations[{Ledger::ID{100}, Ledger::Seq{100}}] = {}; + + // first round a,b,c agree + for (auto const& node : {a, b, c}) + { + auto const val = node.validate(ledgerA); + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + if (val.trusted()) + trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); + } + // d disagrees + { + auto const val = d.validate(ledgerB); + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); + } + // e only issues partials + { + EXPECT_TRUE(ValStatus::Current == harness.add(e.partial(ledgerA))); + } + + harness.clock().advance(5s); + // second round, a,b,c move to ledger 2 + for (auto const& node : {a, b, c}) + { + auto const val = node.validate(ledgerAC); + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + if (val.trusted()) + trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); + } + // d now thinks ledger 1, but cannot re-issue a previously used seq + // and attempting it should generate a conflict. + { + EXPECT_TRUE(ValStatus::Conflicting == harness.add(d.partial(ledgerA))); + } + // e only issues partials + { + EXPECT_TRUE(ValStatus::Current == harness.add(e.partial(ledgerAC))); + } + + compare(); +} + +TEST(ValidationsTest, expire) +{ + // Verify expiring clears out validations stored by ledger + SCOPED_TRACE("Expire validations"); + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + constexpr Ledger::Seq kOne(1); + constexpr Ledger::Seq kTwo(2); + + // simple cases + Ledger const ledgerA = h["a"]; + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerA))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); + expireValidations(harness.vals()); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); + harness.clock().advance(harness.parms().validationSetExpires); + expireValidations(harness.vals()); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 0); + + // use setSeqToKeep to keep the validation from expire + Ledger const ledgerB = h["ab"]; + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerB))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); + harness.vals().setSeqToKeep(ledgerB.seq(), ledgerB.seq() + kOne); + harness.clock().advance(harness.parms().validationSetExpires); + expireValidations(harness.vals()); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); + // change toKeep + harness.vals().setSeqToKeep(ledgerB.seq() + kOne, ledgerB.seq() + kTwo); + // advance clock slowly + int const loops = + harness.parms().validationSetExpires / harness.parms().validationFRESHNESS + 1; + for (int i = 0; i < loops; ++i) + { + harness.clock().advance(harness.parms().validationFRESHNESS); + expireValidations(harness.vals()); + } + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerB.id()) == 0); + + // Allow the validation with high seq to expire + Ledger const ledgerC = h["abc"]; + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerC))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerC.id()) == 1); + harness.vals().setSeqToKeep(ledgerC.seq() - kOne, ledgerC.seq()); + harness.clock().advance(harness.parms().validationSetExpires); + expireValidations(harness.vals()); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerC.id()) == 0); +} + +TEST(ValidationsTest, flush) +{ + // Test final flush of validations + using namespace std::chrono_literals; + SCOPED_TRACE("Flush validations"); + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const trustedNode1 = harness.makeNode(); + Node const trustedNode2 = harness.makeNode(); + Node notTrustedNode = harness.makeNode(); + notTrustedNode.untrust(); + + Ledger const ledgerA = h["a"]; + Ledger const ledgerAB = h["ab"]; + + hash_map expected; + for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode}) + { + auto const val = node.validate(ledgerA); + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + expected.emplace(node.nodeID(), val); + } + + // Send in a new validation for a, saving the new one into the expected + // map after setting the proper prior ledger ID it replaced + harness.clock().advance(1s); + auto newVal = trustedNode1.validate(ledgerAB); + EXPECT_TRUE(ValStatus::Current == harness.add(newVal)); + expected.find(trustedNode1.nodeID())->second = newVal; +} + +TEST(ValidationsTest, get_preferred_ledger) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Preferred Ledger"); + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const trustedNode1 = harness.makeNode(); + Node const trustedNode2 = harness.makeNode(); + Node const trustedNode3 = harness.makeNode(); + + Node notTrustedNode = harness.makeNode(); + notTrustedNode.untrust(); + + Ledger const ledgerA = h["a"]; + Ledger const ledgerB = h["b"]; + Ledger const ledgerAC = h["ac"]; + Ledger const ledgerACD = h["acd"]; + + using Seq = Ledger::Seq; + + auto pref = [](Ledger ledger) { return std::make_pair(ledger.seq(), ledger.id()); }; + + // Empty (no ledgers) + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == std::nullopt); + + // Single ledger + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode1.validate(ledgerB))); + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == pref(ledgerB)); + EXPECT_TRUE(harness.vals().getPreferred(ledgerB) == pref(ledgerB)); + + // Minimum valid sequence + EXPECT_TRUE(harness.vals().getPreferred(ledgerA, Seq{10}) == ledgerA.id()); + + // Untrusted doesn't impact preferred ledger + // (ledgerB has tie-break over ledgerA) + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode2.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerA))); + EXPECT_TRUE(ledgerB.id() > ledgerA.id()); + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == pref(ledgerB)); + EXPECT_TRUE(harness.vals().getPreferred(ledgerB) == pref(ledgerB)); + + // Partial does break ties + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode3.partial(ledgerA))); + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == pref(ledgerA)); + EXPECT_TRUE(harness.vals().getPreferred(ledgerB) == pref(ledgerA)); + + harness.clock().advance(5s); + + // Parent of preferred-> stick with ledger + for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode, trustedNode3}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.validate(ledgerAC))); + // Parent of preferred stays put + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == pref(ledgerA)); + // Earlier different chain, switch + EXPECT_TRUE(harness.vals().getPreferred(ledgerB) == pref(ledgerAC)); + // Later on chain, stays where it is + EXPECT_TRUE(harness.vals().getPreferred(ledgerACD) == pref(ledgerACD)); + + // Any later grandchild or different chain is preferred + harness.clock().advance(5s); + for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode, trustedNode3}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.validate(ledgerACD))); + for (auto const& ledger : {ledgerA, ledgerB, ledgerACD}) + EXPECT_TRUE(harness.vals().getPreferred(ledger) == pref(ledgerACD)); +} + +TEST(ValidationsTest, get_preferred_lcl) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Get preferred LCL"); + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + + Ledger const ledgerA = h["a"]; + Ledger const ledgerB = h["b"]; + Ledger const ledgerC = h["c"]; + + using ID = Ledger::ID; + using Seq = Ledger::Seq; + + hash_map peerCounts; + + // No trusted validations or counts sticks with current ledger + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerA.id()); + + ++peerCounts[ledgerB.id()]; + + // No trusted validations, rely on peer counts + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerB.id()); + + ++peerCounts[ledgerC.id()]; + // No trusted validations, tied peers goes with larger ID + EXPECT_TRUE(ledgerC.id() > ledgerB.id()); + + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerC.id()); + + peerCounts[ledgerC.id()] += 1000; + + // Single trusted always wins over peer counts + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerA))); + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerA.id()); + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerB, Seq{0}, peerCounts) == ledgerA.id()); + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerC, Seq{0}, peerCounts) == ledgerA.id()); + + // Stick with current ledger if trusted validation ledger has too old + // of a sequence + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerB, Seq{2}, peerCounts) == ledgerB.id()); +} + +TEST(ValidationsTest, acquire_validated_ledger) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Acquire validated ledger"); + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Node const b = harness.makeNode(); + + using ID = Ledger::ID; + using Seq = Ledger::Seq; + + // Validate the ledger before it is actually available + Validation const val = a.validate(ID{2}, Seq{2}, 0s, 0s, true); + + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + // Validation is available + EXPECT_TRUE(harness.vals().numTrustedForLedger(ID{2}) == 1); + // but ledger based data is not + EXPECT_TRUE(harness.vals().getNodesAfter(kGenesisLedger, ID{0}) == 0); + // Initial preferred branch falls back to the ledger we are trying to + // acquire + EXPECT_TRUE(harness.vals().getPreferred(kGenesisLedger) == std::make_pair(Seq{2}, ID{2})); + + // After adding another unavailable validation, the preferred ledger + // breaks ties via higher ID + EXPECT_TRUE(ValStatus::Current == harness.add(b.validate(ID{3}, Seq{2}, 0s, 0s, true))); + EXPECT_TRUE(harness.vals().getPreferred(kGenesisLedger) == std::make_pair(Seq{2}, ID{3})); + + // Create the ledger + Ledger const ledgerAB = h["ab"]; + // Now it should be available + EXPECT_TRUE(harness.vals().getNodesAfter(kGenesisLedger, ID{0}) == 1); + + // Create a validation that is not available + harness.clock().advance(5s); + Validation const val2 = a.validate(ID{4}, Seq{4}, 0s, 0s, true); + EXPECT_TRUE(ValStatus::Current == harness.add(val2)); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ID{4}) == 1); + EXPECT_TRUE( + harness.vals().getPreferred(kGenesisLedger) == + std::make_pair(ledgerAB.seq(), ledgerAB.id())); + + // Another node requesting that ledger still doesn't change things + Validation const val3 = b.validate(ID{4}, Seq{4}, 0s, 0s, true); + EXPECT_TRUE(ValStatus::Current == harness.add(val3)); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ID{4}) == 2); + EXPECT_TRUE( + harness.vals().getPreferred(kGenesisLedger) == + std::make_pair(ledgerAB.seq(), ledgerAB.id())); + + // Switch to validation that is available + harness.clock().advance(5s); + Ledger const ledgerABCDE = h["abcde"]; + EXPECT_TRUE(ValStatus::Current == harness.add(a.partial(ledgerABCDE))); + EXPECT_TRUE(ValStatus::Current == harness.add(b.partial(ledgerABCDE))); + EXPECT_TRUE( + harness.vals().getPreferred(kGenesisLedger) == + std::make_pair(ledgerABCDE.seq(), ledgerABCDE.id())); +} + +TEST(ValidationsTest, num_trusted_for_ledger) +{ + SCOPED_TRACE("NumTrustedForLedger"); + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Node const b = harness.makeNode(); + Ledger const ledgerA = h["a"]; + + EXPECT_TRUE(ValStatus::Current == harness.add(a.partial(ledgerA))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 0); + + EXPECT_TRUE(ValStatus::Current == harness.add(b.validate(ledgerA))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); +} + +TEST(ValidationsTest, seq_enforcer) +{ + SCOPED_TRACE("SeqEnforcer"); + using Seq = Ledger::Seq; + using namespace std::chrono; + + beast::ManualClock clock; + SeqEnforcer enforcer; + + ValidationParms const p; + + EXPECT_TRUE(enforcer(clock.now(), Seq{1}, p)); + EXPECT_TRUE(enforcer(clock.now(), Seq{10}, p)); + EXPECT_TRUE(!enforcer(clock.now(), Seq{5}, p)); + EXPECT_TRUE(!enforcer(clock.now(), Seq{9}, p)); + clock.advance(p.validationSetExpires - 1ms); + EXPECT_TRUE(!enforcer(clock.now(), Seq{1}, p)); + clock.advance(2ms); + EXPECT_TRUE(enforcer(clock.now(), Seq{1}, p)); +} + +TEST(ValidationsTest, trust_changed) +{ + SCOPED_TRACE("TrustChanged"); + using namespace std::chrono; + + auto checker = [&](TestValidations& vals, + hash_set const& listed, + std::vector const& trustedVals) { + Ledger::ID const testID = + trustedVals.empty() ? kGenesisLedger.id() : trustedVals[0].ledgerID(); + Ledger::Seq const testSeq = + trustedVals.empty() ? kGenesisLedger.seq() : trustedVals[0].seq(); + EXPECT_TRUE(vals.currentTrusted() == trustedVals); + EXPECT_TRUE(vals.getCurrentNodeIDs() == listed); + EXPECT_TRUE(vals.getNodesAfter(kGenesisLedger, kGenesisLedger.id()) == trustedVals.size()); + if (trustedVals.empty()) + { + EXPECT_TRUE(vals.getPreferred(kGenesisLedger) == std::nullopt); + } + else + { + EXPECT_TRUE(vals.getPreferred(kGenesisLedger)->second == testID); + } + EXPECT_TRUE(vals.getTrustedForLedger(testID, testSeq) == trustedVals); + EXPECT_TRUE(vals.numTrustedForLedger(testID) == trustedVals.size()); + }; + + { + // Trusted to untrusted + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Ledger const ledgerAB = h["ab"]; + Validation const v = a.validate(ledgerAB); + EXPECT_TRUE(ValStatus::Current == harness.add(v)); + + hash_set const listed({a.nodeID()}); + std::vector trustedVals({v}); + checker(harness.vals(), listed, trustedVals); + + trustedVals.clear(); + harness.vals().trustChanged({}, {a.nodeID()}); + checker(harness.vals(), listed, trustedVals); + } + + { + // Untrusted to trusted + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node a = harness.makeNode(); + a.untrust(); + Ledger const ledgerAB = h["ab"]; + Validation const v = a.validate(ledgerAB); + EXPECT_TRUE(ValStatus::Current == harness.add(v)); + + hash_set const listed({a.nodeID()}); + std::vector trustedVals; + checker(harness.vals(), listed, trustedVals); + + trustedVals.push_back(v); + harness.vals().trustChanged({a.nodeID()}, {}); + checker(harness.vals(), listed, trustedVals); + } + + { + // Trusted but not acquired -> untrusted + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Validation const v = a.validate(Ledger::ID{2}, Ledger::Seq{2}, 0s, 0s, true); + EXPECT_TRUE(ValStatus::Current == harness.add(v)); + + hash_set const listed({a.nodeID()}); + std::vector trustedVals({v}); + auto& vals = harness.vals(); + EXPECT_TRUE(vals.currentTrusted() == trustedVals); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(vals.getPreferred(kGenesisLedger)->second == v.ledgerID()); + EXPECT_TRUE(vals.getNodesAfter(kGenesisLedger, kGenesisLedger.id()) == 0); + + trustedVals.clear(); + harness.vals().trustChanged({}, {a.nodeID()}); + // make acquiring ledger available + h["ab"]; + EXPECT_TRUE(vals.currentTrusted() == trustedVals); + EXPECT_TRUE(vals.getPreferred(kGenesisLedger) == std::nullopt); + EXPECT_TRUE(vals.getNodesAfter(kGenesisLedger, kGenesisLedger.id()) == 0); + } +} + +} // namespace xrpl::test::csf diff --git a/src/tests/libxrpl/csf/BasicNetwork.cpp b/src/tests/libxrpl/csf/BasicNetwork.cpp new file mode 100644 index 0000000000..a6fc2d90e6 --- /dev/null +++ b/src/tests/libxrpl/csf/BasicNetwork.cpp @@ -0,0 +1,122 @@ +#include + +#include +#include + +#include +#include + +namespace xrpl::test { + +namespace { + +struct Peer +{ + int id; + std::set set; + + Peer(Peer const&) = default; + Peer(Peer&&) = default; + + explicit Peer(int id) : id(id) + { + } + + template + void + start(csf::Scheduler& scheduler, Net& net) + { + using namespace std::chrono_literals; + auto t = scheduler.in(1s, [&] { set.insert(0); }); + if (id == 0) + { + for (auto const link : net.links(this)) + { + net.send(this, link.target, [&, to = link.target] { to->receive(net, this, 1); }); + } + } + else + { + scheduler.cancel(t); + } + } + + template + void + receive(Net& net, Peer* from, int m) + { + set.insert(m); + ++m; + if (m < 5) + { + for (auto const link : net.links(this)) + { + net.send(this, link.target, [&, mm = m, to = link.target] { + to->receive(net, this, mm); + }); + } + } + } +}; + +} // namespace + +TEST(BasicNetworkTest, network) +{ + using namespace std::chrono_literals; + std::vector pv; + pv.emplace_back(0); + pv.emplace_back(1); + pv.emplace_back(2); + csf::Scheduler scheduler; + csf::BasicNetwork net(scheduler); + EXPECT_TRUE(!net.connect(&pv[0], &pv[0])); + EXPECT_TRUE(net.connect(&pv[0], &pv[1], 1s)); + EXPECT_TRUE(net.connect(&pv[1], &pv[2], 1s)); + EXPECT_TRUE(!net.connect(&pv[0], &pv[1])); + for (auto& peer : pv) + peer.start(scheduler, net); + EXPECT_TRUE(scheduler.stepFor(0s)); + EXPECT_TRUE(scheduler.stepFor(1s)); + EXPECT_TRUE(scheduler.step()); + EXPECT_TRUE(!scheduler.step()); + EXPECT_TRUE(!scheduler.stepFor(1s)); + net.send(&pv[0], &pv[1], [] {}); + net.send(&pv[1], &pv[0], [] {}); + EXPECT_TRUE(net.disconnect(&pv[0], &pv[1])); + EXPECT_TRUE(!net.disconnect(&pv[0], &pv[1])); + for (;;) + { + auto const links = net.links(&pv[1]); + if (links.empty()) + break; + EXPECT_TRUE(net.disconnect(&pv[1], links[0].target)); + } + EXPECT_TRUE(pv[0].set == std::set({0, 2, 4})); + EXPECT_TRUE(pv[1].set == std::set({1, 3})); + EXPECT_TRUE(pv[2].set == std::set({2, 4})); +} + +TEST(BasicNetworkTest, disconnect) +{ + using namespace std::chrono_literals; + csf::Scheduler scheduler; + csf::BasicNetwork net(scheduler); + EXPECT_TRUE(net.connect(0, 1, 1s)); + EXPECT_TRUE(net.connect(0, 2, 2s)); + + std::set delivered; + net.send(0, 1, [&]() { delivered.insert(1); }); + net.send(0, 2, [&]() { delivered.insert(2); }); + + scheduler.in(1000ms, [&]() { EXPECT_TRUE(net.disconnect(0, 2)); }); + scheduler.in(1100ms, [&]() { EXPECT_TRUE(net.connect(0, 2)); }); + + scheduler.step(); + + // only the first message is delivered because the disconnect at 1 s + // purges all pending messages from 0 to 2 + EXPECT_TRUE(delivered == std::set({1})); +} + +} // namespace xrpl::test diff --git a/src/test/csf/BasicNetwork.h b/src/tests/libxrpl/csf/BasicNetwork.h similarity index 99% rename from src/test/csf/BasicNetwork.h rename to src/tests/libxrpl/csf/BasicNetwork.h index 0428475504..2c68bd282d 100644 --- a/src/test/csf/BasicNetwork.h +++ b/src/tests/libxrpl/csf/BasicNetwork.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include diff --git a/src/test/csf/CollectorRef.h b/src/tests/libxrpl/csf/CollectorRef.h similarity index 97% rename from src/test/csf/CollectorRef.h rename to src/tests/libxrpl/csf/CollectorRef.h index 3aef4d617f..b1da962f3d 100644 --- a/src/test/csf/CollectorRef.h +++ b/src/tests/libxrpl/csf/CollectorRef.h @@ -1,11 +1,11 @@ #pragma once -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/tests/libxrpl/csf/Digraph.cpp b/src/tests/libxrpl/csf/Digraph.cpp new file mode 100644 index 0000000000..83c15ec06a --- /dev/null +++ b/src/tests/libxrpl/csf/Digraph.cpp @@ -0,0 +1,72 @@ +#include + +#include + +#include +#include +#include +#include + +namespace xrpl::test { + +TEST(DigraphTest, digraph) +{ + using namespace csf; + using Graph = Digraph; + Graph graph; + + EXPECT_TRUE(!graph.connected('a', 'b')); + EXPECT_TRUE(!graph.edge('a', 'b')); + EXPECT_TRUE(!graph.disconnect('a', 'b')); + + EXPECT_TRUE(graph.connect('a', 'b', "foobar")); + EXPECT_TRUE(graph.connected('a', 'b')); + EXPECT_TRUE(*graph.edge('a', 'b') == "foobar"); // NOLINT(bugprone-unchecked-optional-access) + + EXPECT_TRUE(!graph.connect('a', 'b', "repeat")); + EXPECT_TRUE(graph.disconnect('a', 'b')); + EXPECT_TRUE(graph.connect('a', 'b', "repeat")); + EXPECT_TRUE(graph.connected('a', 'b')); + EXPECT_TRUE(*graph.edge('a', 'b') == "repeat"); // NOLINT(bugprone-unchecked-optional-access) + + EXPECT_TRUE(graph.connect('a', 'c', "tree")); + + { + std::vector> edges; + + for (auto const& edge : graph.outEdges('a')) + { + edges.emplace_back(edge.source, edge.target, edge.data); + } + + std::vector> expected; + expected.emplace_back('a', 'b', "repeat"); + expected.emplace_back('a', 'c', "tree"); + EXPECT_TRUE(edges == expected); + EXPECT_TRUE(graph.outDegree('a') == expected.size()); + } + + EXPECT_TRUE(graph.outEdges('r').size() == 0); + EXPECT_TRUE(graph.outDegree('r') == 0); + EXPECT_TRUE(graph.outDegree('c') == 0); + + // only 'a' has out edges + EXPECT_TRUE(graph.outVertices().size() == 1); + std::vector const expected = {'b', 'c'}; + + EXPECT_TRUE((graph.outVertices('a') == expected)); + EXPECT_TRUE(graph.outVertices('b').size() == 0); + EXPECT_TRUE(graph.outVertices('c').size() == 0); + EXPECT_TRUE(graph.outVertices('r').size() == 0); + + std::stringstream ss; + graph.saveDot(ss, [](char v) { return v; }); + std::string const expectedDot = + "digraph {\n" + "a -> b;\n" + "a -> c;\n" + "}\n"; + EXPECT_TRUE(ss.str() == expectedDot); +} + +} // namespace xrpl::test diff --git a/src/test/csf/Digraph.h b/src/tests/libxrpl/csf/Digraph.h similarity index 100% rename from src/test/csf/Digraph.h rename to src/tests/libxrpl/csf/Digraph.h diff --git a/src/tests/libxrpl/csf/Histogram.cpp b/src/tests/libxrpl/csf/Histogram.cpp new file mode 100644 index 0000000000..6de1cde593 --- /dev/null +++ b/src/tests/libxrpl/csf/Histogram.cpp @@ -0,0 +1,59 @@ +#include + +#include + +namespace xrpl::test { + +TEST(HistogramTest, histogram) +{ + using namespace csf; + Histogram hist; + + EXPECT_TRUE(hist.size() == 0); + EXPECT_TRUE(hist.numBins() == 0); + EXPECT_TRUE(hist.minValue() == 0); + EXPECT_TRUE(hist.maxValue() == 0); + EXPECT_TRUE(hist.avg() == 0); + EXPECT_TRUE(hist.percentile(0.0f) == hist.minValue()); + EXPECT_TRUE(hist.percentile(0.5f) == 0); + EXPECT_TRUE(hist.percentile(0.9f) == 0); + EXPECT_TRUE(hist.percentile(1.0f) == hist.maxValue()); + + hist.insert(1); + + EXPECT_TRUE(hist.size() == 1); + EXPECT_TRUE(hist.numBins() == 1); + EXPECT_TRUE(hist.minValue() == 1); + EXPECT_TRUE(hist.maxValue() == 1); + EXPECT_TRUE(hist.avg() == 1); + EXPECT_TRUE(hist.percentile(0.0f) == hist.minValue()); + EXPECT_TRUE(hist.percentile(0.5f) == 1); + EXPECT_TRUE(hist.percentile(0.9f) == 1); + EXPECT_TRUE(hist.percentile(1.0f) == hist.maxValue()); + + hist.insert(9); + + EXPECT_TRUE(hist.size() == 2); + EXPECT_TRUE(hist.numBins() == 2); + EXPECT_TRUE(hist.minValue() == 1); + EXPECT_TRUE(hist.maxValue() == 9); + EXPECT_TRUE(hist.avg() == 5); + EXPECT_TRUE(hist.percentile(0.0f) == hist.minValue()); + EXPECT_TRUE(hist.percentile(0.5f) == 1); + EXPECT_TRUE(hist.percentile(0.9f) == 9); + EXPECT_TRUE(hist.percentile(1.0f) == hist.maxValue()); + + hist.insert(1); + + EXPECT_TRUE(hist.size() == 3); + EXPECT_TRUE(hist.numBins() == 2); + EXPECT_TRUE(hist.minValue() == 1); + EXPECT_TRUE(hist.maxValue() == 9); + EXPECT_TRUE(hist.avg() == 11 / 3); + EXPECT_TRUE(hist.percentile(0.0f) == hist.minValue()); + EXPECT_TRUE(hist.percentile(0.5f) == 1); + EXPECT_TRUE(hist.percentile(0.9f) == 9); + EXPECT_TRUE(hist.percentile(1.0f) == hist.maxValue()); +} + +} // namespace xrpl::test diff --git a/src/test/csf/Histogram.h b/src/tests/libxrpl/csf/Histogram.h similarity index 100% rename from src/test/csf/Histogram.h rename to src/tests/libxrpl/csf/Histogram.h diff --git a/src/test/csf/Peer.h b/src/tests/libxrpl/csf/Peer.h similarity index 98% rename from src/test/csf/Peer.h rename to src/tests/libxrpl/csf/Peer.h index 79bffec9cb..d4b6f42bdb 100644 --- a/src/test/csf/Peer.h +++ b/src/tests/libxrpl/csf/Peer.h @@ -1,33 +1,32 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - #include #include #include #include #include #include +#include +#include +#include +#include #include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include #include diff --git a/src/test/csf/PeerGroup.h b/src/tests/libxrpl/csf/PeerGroup.h similarity index 98% rename from src/test/csf/PeerGroup.h rename to src/tests/libxrpl/csf/PeerGroup.h index 1c31209ef3..ff99b779a3 100644 --- a/src/test/csf/PeerGroup.h +++ b/src/tests/libxrpl/csf/PeerGroup.h @@ -1,9 +1,9 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/src/test/csf/Proposal.h b/src/tests/libxrpl/csf/Proposal.h similarity index 66% rename from src/test/csf/Proposal.h rename to src/tests/libxrpl/csf/Proposal.h index ecf430ae8d..b2a97f9731 100644 --- a/src/test/csf/Proposal.h +++ b/src/tests/libxrpl/csf/Proposal.h @@ -1,10 +1,10 @@ #pragma once -#include -#include -#include +#include -#include +#include +#include +#include namespace xrpl::test::csf { /** diff --git a/src/test/csf/README.md b/src/tests/libxrpl/csf/README.md similarity index 100% rename from src/test/csf/README.md rename to src/tests/libxrpl/csf/README.md diff --git a/src/tests/libxrpl/csf/Scheduler.cpp b/src/tests/libxrpl/csf/Scheduler.cpp new file mode 100644 index 0000000000..63871e0623 --- /dev/null +++ b/src/tests/libxrpl/csf/Scheduler.cpp @@ -0,0 +1,61 @@ +#include + +#include + +#include + +namespace xrpl::test { + +TEST(SchedulerTest, scheduler) +{ + using namespace std::chrono_literals; + csf::Scheduler scheduler; + std::set seen; + + scheduler.in(1s, [&] { seen.insert(1); }); + scheduler.in(2s, [&] { seen.insert(2); }); + auto token = scheduler.in(3s, [&] { seen.insert(3); }); + scheduler.at(scheduler.now() + 4s, [&] { seen.insert(4); }); + scheduler.at(scheduler.now() + 8s, [&] { seen.insert(8); }); + + auto start = scheduler.now(); + + // Process first event + EXPECT_TRUE(seen.empty()); + EXPECT_TRUE(scheduler.stepOne()); + EXPECT_TRUE(seen == std::set({1})); + EXPECT_TRUE(scheduler.now() == (start + 1s)); + + // No processing if stepping until current time + EXPECT_TRUE(scheduler.stepUntil(scheduler.now())); + EXPECT_TRUE(seen == std::set({1})); + EXPECT_TRUE(scheduler.now() == (start + 1s)); + + // Process next event + EXPECT_TRUE(scheduler.stepFor(1s)); + EXPECT_TRUE(seen == std::set({1, 2})); + EXPECT_TRUE(scheduler.now() == (start + 2s)); + + // Don't process cancelled event, but advance clock + scheduler.cancel(token); + EXPECT_TRUE(scheduler.stepFor(1s)); + EXPECT_TRUE(seen == std::set({1, 2})); + EXPECT_TRUE(scheduler.now() == (start + 3s)); + + // Process until 3 seen ints + EXPECT_TRUE(scheduler.stepWhile([&]() { return seen.size() < 3; })); + EXPECT_TRUE(seen == std::set({1, 2, 4})); + EXPECT_TRUE(scheduler.now() == (start + 4s)); + + // Process the rest + EXPECT_TRUE(scheduler.step()); + EXPECT_TRUE(seen == std::set({1, 2, 4, 8})); + EXPECT_TRUE(scheduler.now() == (start + 8s)); + + // Process the rest again doesn't advance + EXPECT_TRUE(!scheduler.step()); + EXPECT_TRUE(seen == std::set({1, 2, 4, 8})); + EXPECT_TRUE(scheduler.now() == (start + 8s)); +} + +} // namespace xrpl::test diff --git a/src/test/csf/Scheduler.h b/src/tests/libxrpl/csf/Scheduler.h similarity index 100% rename from src/test/csf/Scheduler.h rename to src/tests/libxrpl/csf/Scheduler.h diff --git a/src/test/csf/Sim.h b/src/tests/libxrpl/csf/Sim.h similarity index 94% rename from src/test/csf/Sim.h rename to src/tests/libxrpl/csf/Sim.h index 94d26d5e06..774537138a 100644 --- a/src/test/csf/Sim.h +++ b/src/tests/libxrpl/csf/Sim.h @@ -1,16 +1,16 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include - #include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include #include diff --git a/src/test/csf/SimTime.h b/src/tests/libxrpl/csf/SimTime.h similarity index 100% rename from src/test/csf/SimTime.h rename to src/tests/libxrpl/csf/SimTime.h diff --git a/src/test/csf/TrustGraph.h b/src/tests/libxrpl/csf/TrustGraph.h similarity index 99% rename from src/test/csf/TrustGraph.h rename to src/tests/libxrpl/csf/TrustGraph.h index d46a887364..d010b954e0 100644 --- a/src/test/csf/TrustGraph.h +++ b/src/tests/libxrpl/csf/TrustGraph.h @@ -1,9 +1,9 @@ #pragma once -#include - #include +#include + #include #include #include diff --git a/src/test/csf/Tx.h b/src/tests/libxrpl/csf/Tx.h similarity index 100% rename from src/test/csf/Tx.h rename to src/tests/libxrpl/csf/Tx.h diff --git a/src/test/csf/Validation.h b/src/tests/libxrpl/csf/Validation.h similarity index 99% rename from src/test/csf/Validation.h rename to src/tests/libxrpl/csf/Validation.h index 0b9fc94890..4325f96511 100644 --- a/src/test/csf/Validation.h +++ b/src/tests/libxrpl/csf/Validation.h @@ -1,10 +1,10 @@ #pragma once -#include - #include #include +#include + #include #include #include diff --git a/src/test/csf/collectors.h b/src/tests/libxrpl/csf/collectors.h similarity index 99% rename from src/test/csf/collectors.h rename to src/tests/libxrpl/csf/collectors.h index f85854e5dd..05b63f592b 100644 --- a/src/test/csf/collectors.h +++ b/src/tests/libxrpl/csf/collectors.h @@ -1,13 +1,13 @@ #pragma once -#include -#include -#include -#include -#include - #include +#include +#include +#include +#include +#include + #include #include #include diff --git a/src/test/csf/csf_graph.png b/src/tests/libxrpl/csf/csf_graph.png similarity index 100% rename from src/test/csf/csf_graph.png rename to src/tests/libxrpl/csf/csf_graph.png diff --git a/src/test/csf/csf_overview.png b/src/tests/libxrpl/csf/csf_overview.png similarity index 100% rename from src/test/csf/csf_overview.png rename to src/tests/libxrpl/csf/csf_overview.png diff --git a/src/test/csf/events.h b/src/tests/libxrpl/csf/events.h similarity index 96% rename from src/test/csf/events.h rename to src/tests/libxrpl/csf/events.h index 2cf4fd9e9b..4ff15ec987 100644 --- a/src/test/csf/events.h +++ b/src/tests/libxrpl/csf/events.h @@ -1,8 +1,8 @@ #pragma once -#include -#include -#include +#include +#include +#include namespace xrpl::test::csf { diff --git a/src/test/csf/impl/Sim.cpp b/src/tests/libxrpl/csf/impl/Sim.cpp similarity index 93% rename from src/test/csf/impl/Sim.cpp rename to src/tests/libxrpl/csf/impl/Sim.cpp index bf4706927e..28ac5f126a 100644 --- a/src/test/csf/impl/Sim.cpp +++ b/src/tests/libxrpl/csf/impl/Sim.cpp @@ -1,7 +1,7 @@ -#include +#include -#include -#include +#include +#include #include #include diff --git a/src/test/csf/impl/ledgers.cpp b/src/tests/libxrpl/csf/impl/ledgers.cpp similarity index 98% rename from src/test/csf/impl/ledgers.cpp rename to src/tests/libxrpl/csf/impl/ledgers.cpp index 46a3600307..ed1cd927c4 100644 --- a/src/test/csf/impl/ledgers.cpp +++ b/src/tests/libxrpl/csf/impl/ledgers.cpp @@ -1,11 +1,11 @@ -#include - -#include +#include #include #include #include +#include + #include #include #include diff --git a/src/test/csf/ledgers.h b/src/tests/libxrpl/csf/ledgers.h similarity index 99% rename from src/test/csf/ledgers.h rename to src/tests/libxrpl/csf/ledgers.h index 09f5fa54de..ca4e44d5a6 100644 --- a/src/test/csf/ledgers.h +++ b/src/tests/libxrpl/csf/ledgers.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include #include @@ -9,6 +7,8 @@ #include +#include + #include #include #include diff --git a/src/test/csf/random.h b/src/tests/libxrpl/csf/random.h similarity index 98% rename from src/test/csf/random.h rename to src/tests/libxrpl/csf/random.h index f8df253642..007bdecb1b 100644 --- a/src/test/csf/random.h +++ b/src/tests/libxrpl/csf/random.h @@ -135,13 +135,13 @@ class PowerLawDistribution { double xmin_; double a_; - double inv_; + double inv_{1.0 / (1.0 - a_)}; std::uniform_real_distribution uf_{0, 1}; public: using result_type = double; - PowerLawDistribution(double xmin, double a) : xmin_{xmin}, a_{a}, inv_(1.0 / (1.0 - a_)) + PowerLawDistribution(double xmin, double a) : xmin_{xmin}, a_{a} { } diff --git a/src/test/csf/submitters.h b/src/tests/libxrpl/csf/submitters.h similarity index 96% rename from src/test/csf/submitters.h rename to src/tests/libxrpl/csf/submitters.h index 160d0bcd9f..4f27f0f665 100644 --- a/src/test/csf/submitters.h +++ b/src/tests/libxrpl/csf/submitters.h @@ -1,8 +1,8 @@ #pragma once -#include -#include -#include +#include +#include +#include #include #include diff --git a/src/test/csf/timers.h b/src/tests/libxrpl/csf/timers.h similarity index 96% rename from src/test/csf/timers.h rename to src/tests/libxrpl/csf/timers.h index 4f13b21b25..e89ea33698 100644 --- a/src/test/csf/timers.h +++ b/src/tests/libxrpl/csf/timers.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include #include diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 4abf77f578..20421ab916 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -17,8 +16,6 @@ #include #include #include -#include -#include #include #include @@ -32,6 +29,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -389,7 +389,7 @@ RCLConsensus::Adaptor::onClose( if (!wrongLCL) { LedgerIndex const seq = prevLedger->header().seq + 1; - RCLCensorshipDetector::TxIDSeqVec proposed; + CensorshipDetector::TxIDSeqVec proposed; initialSet->visitLeaves( [&proposed, seq](boost::intrusive_ptr const& item) { diff --git a/src/xrpld/app/consensus/RCLConsensus.h b/src/xrpld/app/consensus/RCLConsensus.h index 4ffe18a7a8..4c07e7e646 100644 --- a/src/xrpld/app/consensus/RCLConsensus.h +++ b/src/xrpld/app/consensus/RCLConsensus.h @@ -1,20 +1,20 @@ #pragma once -#include #include #include #include #include #include #include -#include -#include -#include #include #include #include #include +#include +#include +#include +#include #include #include #include @@ -83,7 +83,7 @@ class RCLConsensus std::atomic prevRoundTime_{std::chrono::milliseconds{0}}; std::atomic mode_{ConsensusMode::Observing}; - RCLCensorshipDetector censorshipDetector_; + CensorshipDetector censorshipDetector_; NegativeUNLVote nUnlVote_; public: diff --git a/src/xrpld/app/consensus/RCLCxPeerPos.h b/src/xrpld/app/consensus/RCLCxPeerPos.h index 050bdf6d36..078556dea8 100644 --- a/src/xrpld/app/consensus/RCLCxPeerPos.h +++ b/src/xrpld/app/consensus/RCLCxPeerPos.h @@ -1,11 +1,10 @@ #pragma once -#include - #include #include #include #include +#include #include #include #include diff --git a/src/xrpld/app/consensus/RCLValidations.cpp b/src/xrpld/app/consensus/RCLValidations.cpp index 9d40e60b00..c587a04cf0 100644 --- a/src/xrpld/app/consensus/RCLValidations.cpp +++ b/src/xrpld/app/consensus/RCLValidations.cpp @@ -5,12 +5,12 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include diff --git a/src/xrpld/app/consensus/RCLValidations.h b/src/xrpld/app/consensus/RCLValidations.h index 7eadaf0dff..963b6c150e 100644 --- a/src/xrpld/app/consensus/RCLValidations.h +++ b/src/xrpld/app/consensus/RCLValidations.h @@ -1,10 +1,9 @@ #pragma once -#include - #include #include #include +#include #include #include #include diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 4b0091dff6..47cbebb901 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -21,8 +21,6 @@ #include #include #include -#include -#include #include #include #include @@ -54,6 +52,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index c720bdf30b..d7a9a9e449 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -35,6 +34,7 @@ #include #include #include +#include #include #include #include From 7908aec2ec342a17924d81e373cc0b291318051d Mon Sep 17 00:00:00 2001 From: Ed Hennis Date: Thu, 23 Jul 2026 17:39:11 -0400 Subject: [PATCH 09/13] feat: Check default fields are not default when serializing (#6267) Co-authored-by: Vito <5780819+Tapanito@users.noreply.github.com> Co-authored-by: Ayaz Salikhov Co-authored-by: Bart --- src/libxrpl/protocol/STObject.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libxrpl/protocol/STObject.cpp b/src/libxrpl/protocol/STObject.cpp index 4b3ace2be3..c2543d2cca 100644 --- a/src/libxrpl/protocol/STObject.cpp +++ b/src/libxrpl/protocol/STObject.cpp @@ -899,6 +899,10 @@ STObject::add(Serializer& s, WhichFields whichFields) const XRPL_ASSERT( (sType != STI_OBJECT) || (field->getFName().fieldType == STI_OBJECT), "xrpl::STObject::add : valid field type"); + XRPL_ASSERT( + getStyle(field->getFName()) != SoeDefault || !field->isDefault(), + "xrpl::STObject::add : non-default value"); + field->addFieldID(s); field->add(s); if (sType == STI_ARRAY || sType == STI_OBJECT) From 9afa1cf4d15665e5597ef69d91917baa69c79bce Mon Sep 17 00:00:00 2001 From: Kassaking7 <96991820+Kassaking7@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:40:21 -0400 Subject: [PATCH 10/13] fix: Update PermissionedDEX invariant domain tracking for valid offer replacement (#7387) Co-authored-by: Bart --- .../tx/invariants/PermissionedDEXInvariant.h | 3 +- .../invariants/PermissionedDEXInvariant.cpp | 14 ++++- src/test/app/PermissionedDEX_test.cpp | 57 +++++++++++++++++++ 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/include/xrpl/tx/invariants/PermissionedDEXInvariant.h b/include/xrpl/tx/invariants/PermissionedDEXInvariant.h index 2763b80a94..ae0d573385 100644 --- a/include/xrpl/tx/invariants/PermissionedDEXInvariant.h +++ b/include/xrpl/tx/invariants/PermissionedDEXInvariant.h @@ -17,7 +17,8 @@ class ValidPermissionedDEX bool regularOffers_ = false; // post-fixCleanup3_2_0: excludes deleted offers bool badHybridsOld_ = false; // pre-fixCleanup3_1_3: missing field/domain or size > 1 bool badHybrids_ = false; // post-fixCleanup3_1_3: also catches size == 0 (size != 1) - hash_set domains_; + hash_set domainsOld_; // pre-fixCleanup3_4_0: also flags deleted domains + hash_set domains_; // post-fixCleanup3_4_0: excludes deleted domains public: void diff --git a/src/libxrpl/tx/invariants/PermissionedDEXInvariant.cpp b/src/libxrpl/tx/invariants/PermissionedDEXInvariant.cpp index 1014642b36..44f623f284 100644 --- a/src/libxrpl/tx/invariants/PermissionedDEXInvariant.cpp +++ b/src/libxrpl/tx/invariants/PermissionedDEXInvariant.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -19,17 +20,23 @@ namespace xrpl { void ValidPermissionedDEX::visitEntry(bool isDelete, SLE::const_ref before, SLE::const_ref after) { + auto trackDomain = [this, isDelete](uint256 const& domain) { + domainsOld_.insert(domain); + if (!isDelete) + domains_.insert(domain); + }; + if (after && after->getType() == ltDIR_NODE) { if (after->isFieldPresent(sfDomainID)) - domains_.insert(after->getFieldH256(sfDomainID)); + trackDomain(after->getFieldH256(sfDomainID)); } if (after && after->getType() == ltOFFER) { if (after->isFieldPresent(sfDomainID)) { - domains_.insert(after->getFieldH256(sfDomainID)); + trackDomain(after->getFieldH256(sfDomainID)); } else { @@ -87,7 +94,8 @@ ValidPermissionedDEX::finalize( // for both payment and offercreate, there shouldn't be another domain // that's different from the domain specified - for (auto const& d : domains_) + auto const& domains = view.rules().enabled(fixCleanup3_4_0) ? domains_ : domainsOld_; + for (auto const& d : domains) { if (d != domain) { diff --git a/src/test/app/PermissionedDEX_test.cpp b/src/test/app/PermissionedDEX_test.cpp index 998b7b1c7f..67cb7602a0 100644 --- a/src/test/app/PermissionedDEX_test.cpp +++ b/src/test/app/PermissionedDEX_test.cpp @@ -2001,6 +2001,61 @@ class PermissionedDEX_test : public beast::unit_test::Suite } } + void + testReplaceDomainOfferWithOtherDomainOffer(FeatureBitset features) + { + bool const fixEnabled = features[fixCleanup3_4_0]; + + testcase << "Replace domain offer via OfferCreate" + << (fixEnabled ? " (fixCleanup3_4_0 enabled)" : " (fixCleanup3_4_0 disabled)"); + + Env env(*this, features); + auto const& [gw, domainOwner, alice, bob, carol, USD, domainA, credType] = + PermissionedDEX(env); + + Account const domainOwnerB("permdex-domainOwnerB"); + auto const domainB = + setupDomain(env, {alice, bob, carol, gw}, domainOwnerB, "permdex-other-domain"); + BEAST_EXPECT(domainA != domainB); + + auto const oldSeq = env.seq(alice); + env(offer(alice, USD(100), XRP(1)), Domain(domainA)); + env.close(); + + BEAST_EXPECT(checkOffer(env, alice, oldSeq, USD(100), XRP(1), 0, true)); + auto const oldOffer = env.le(keylet::offer(alice.id(), oldSeq)); + if (!BEAST_EXPECT(oldOffer)) + return; + BEAST_EXPECT(oldOffer->getFieldH256(sfDomainID) == domainA); + + auto const newSeq = env.seq(alice); + // The invariant should reject mixing active Permissioned DEX domains, + // not a domain that is only touched because its offer is being deleted. + if (fixEnabled) + { + env(offer(alice, USD(100), XRP(2)), Domain(domainB), Json(jss::OfferSequence, oldSeq)); + env.close(); + + BEAST_EXPECT(!offerExists(env, alice, oldSeq)); + BEAST_EXPECT(checkOffer(env, alice, newSeq, USD(100), XRP(2), 0, true)); + auto const newOffer = env.le(keylet::offer(alice.id(), newSeq)); + if (!BEAST_EXPECT(newOffer)) + return; + BEAST_EXPECT(newOffer->getFieldH256(sfDomainID) == domainB); + } + else + { + env(offer(alice, USD(100), XRP(2)), + Domain(domainB), + Json(jss::OfferSequence, oldSeq), + Ter(tecINVARIANT_FAILED)); + env.close(); + + BEAST_EXPECT(checkOffer(env, alice, oldSeq, USD(100), XRP(1), 0, true)); + BEAST_EXPECT(!offerExists(env, alice, newSeq)); + } + } + public: void run() override @@ -2038,6 +2093,8 @@ public: // only after fixCleanup3_2_0. testCancelRegularOfferWithDomainCreate(all); testCancelRegularOfferWithDomainCreate(all - fixCleanup3_2_0); + testReplaceDomainOfferWithOtherDomainOffer(all); + testReplaceDomainOfferWithOtherDomainOffer(all - fixCleanup3_4_0); } }; From 29d74142aee00930efcbf809efd0104f6c688fe5 Mon Sep 17 00:00:00 2001 From: Ayaz Salikhov Date: Fri, 24 Jul 2026 15:59:24 +0100 Subject: [PATCH 11/13] =?UTF-8?q?build:=20Pat=D1=81h=20binary=20in=20local?= =?UTF-8?q?=20Linux=20nix=20environment=20(#7859)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/scripts/strategy-matrix/linux.json | 2 +- .github/workflows/build-nix-images.yml | 2 ++ .github/workflows/on-pr.yml | 1 + .github/workflows/on-trigger.yml | 1 + .github/workflows/publish-docs.yml | 2 +- .github/workflows/reusable-clang-tidy.yml | 2 +- .github/workflows/reusable-upload-recipe.yml | 2 +- .../default-loader-path.sh | 0 cmake/CompilationEnv.cmake | 21 +++++++++++ cmake/PatchNixBinary.cmake | 35 ++++++++++++------- cmake/XrplCompiler.cmake | 3 +- cmake/XrplSanity.cmake | 13 +++++++ nix/devshell.nix | 34 +++++++++++++----- nix/docker/Dockerfile | 2 +- nix/docker/README.md | 7 ++-- 15 files changed, 97 insertions(+), 30 deletions(-) rename nix/docker/loader-path.sh => bin/default-loader-path.sh (100%) diff --git a/.github/scripts/strategy-matrix/linux.json b/.github/scripts/strategy-matrix/linux.json index 992f314686..2a0b5e8e0e 100644 --- a/.github/scripts/strategy-matrix/linux.json +++ b/.github/scripts/strategy-matrix/linux.json @@ -1,5 +1,5 @@ { - "image_tag": "sha-3122de8", + "image_tag": "sha-40cdf49", "configs": { "ubuntu": [ { diff --git a/.github/workflows/build-nix-images.yml b/.github/workflows/build-nix-images.yml index 8574182a7e..fe2f43fdcc 100644 --- a/.github/workflows/build-nix-images.yml +++ b/.github/workflows/build-nix-images.yml @@ -13,6 +13,7 @@ on: - "!nix/docker/README.md" - "!nix/devshell.nix" - "bin/check-tools.sh" + - "bin/default-loader-path.sh" - "bin/install-sanitizer-libs.sh" pull_request: paths: @@ -24,6 +25,7 @@ on: - "!nix/docker/README.md" - "!nix/devshell.nix" - "bin/check-tools.sh" + - "bin/default-loader-path.sh" - "bin/install-sanitizer-libs.sh" workflow_dispatch: diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 442a202a44..13c807ffca 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -90,6 +90,7 @@ jobs: .clang-tidy .codecov.yml bin/check-tools.sh + bin/default-loader-path.sh cfg/** cmake/** conan/** diff --git a/.github/workflows/on-trigger.yml b/.github/workflows/on-trigger.yml index 49a93d2746..b8899cec72 100644 --- a/.github/workflows/on-trigger.yml +++ b/.github/workflows/on-trigger.yml @@ -28,6 +28,7 @@ on: - ".clang-tidy" - ".codecov.yml" - "bin/check-tools.sh" + - "bin/default-loader-path.sh" - "cfg/**" - "cmake/**" - "conan/**" diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 19c73f93d6..49f5c021a3 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -41,7 +41,7 @@ env: jobs: build: runs-on: ubuntu-latest - container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-3122de8 + container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-40cdf49 steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/.github/workflows/reusable-clang-tidy.yml b/.github/workflows/reusable-clang-tidy.yml index e81bbea367..90f24bc464 100644 --- a/.github/workflows/reusable-clang-tidy.yml +++ b/.github/workflows/reusable-clang-tidy.yml @@ -34,7 +34,7 @@ jobs: needs: [determine-files] if: ${{ needs.determine-files.outputs.cpp_changed_files != '' || needs.determine-files.outputs.need_full_run == 'true' }} runs-on: ["self-hosted", "Linux", "X64", "heavy"] - container: "ghcr.io/xrplf/xrpld/nix-debian:sha-3122de8" + container: "ghcr.io/xrplf/xrpld/nix-debian:sha-40cdf49" permissions: contents: read issues: write diff --git a/.github/workflows/reusable-upload-recipe.yml b/.github/workflows/reusable-upload-recipe.yml index df4a2d9516..0f00ce7ca0 100644 --- a/.github/workflows/reusable-upload-recipe.yml +++ b/.github/workflows/reusable-upload-recipe.yml @@ -40,7 +40,7 @@ defaults: jobs: upload: runs-on: ubuntu-latest - container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-3122de8 + container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-40cdf49 env: REMOTE_NAME: ${{ inputs.remote_name }} CONAN_LOGIN_USERNAME_XRPLF: ${{ secrets.remote_username }} diff --git a/nix/docker/loader-path.sh b/bin/default-loader-path.sh similarity index 100% rename from nix/docker/loader-path.sh rename to bin/default-loader-path.sh diff --git a/cmake/CompilationEnv.cmake b/cmake/CompilationEnv.cmake index 8e69a4dfdd..471c43d6c6 100644 --- a/cmake/CompilationEnv.cmake +++ b/cmake/CompilationEnv.cmake @@ -29,6 +29,27 @@ if(CMAKE_GENERATOR STREQUAL "Xcode") set(is_xcode TRUE) endif() +# -------------------------------------------------------------------- +# Nix toolchain detection +# -------------------------------------------------------------------- +# True when the C++ compiler resolves into the Nix store. CMAKE_CXX_COMPILER may +# be referenced through a symlink outside the store (a Nix profile, a /usr/bin +# alternative, ...), so resolve the real path before matching. +set(is_nix_compiler FALSE) +get_filename_component(_cxx_real "${CMAKE_CXX_COMPILER}" REALPATH) +if(_cxx_real MATCHES "^/nix/store/") + set(is_nix_compiler TRUE) +endif() +unset(_cxx_real) + +# True inside the Nix CI Docker image, identified by the /nix/ci-env tree it +# ships (see nix/docker/Dockerfile). The dev shell and bare systems don't have +# it, so it distinguishes the CI image from other Nix-compiler environments. +set(is_ci_image FALSE) +if(EXISTS "/nix/ci-env/bin") + set(is_ci_image TRUE) +endif() + # -------------------------------------------------------------------- # Operating system detection # -------------------------------------------------------------------- diff --git a/cmake/PatchNixBinary.cmake b/cmake/PatchNixBinary.cmake index 79ca0b150c..2490416f1f 100644 --- a/cmake/PatchNixBinary.cmake +++ b/cmake/PatchNixBinary.cmake @@ -1,26 +1,37 @@ #[===================================================================[ Patch executables to run in non-Nix environments. - The Nix-based CI image links binaries against an ELF interpreter (loader) - that lives in the Nix store, so the resulting binaries don't run elsewhere - (including once installed from the .deb package). `patch_nix_binary` adds a - POST_BUILD step that resets the interpreter to the system default loader and - drops the rpath. + The Nix toolchain links binaries against an ELF interpreter (loader) + that lives in the Nix store, so the resulting binaries don't run elsewhere. + `patch_nix_binary` adds a POST_BUILD step that resets the interpreter + to the system default loader and drops the rpath. - This is only active inside the Nix-based image, detected by the presence of - /tmp/loader-path.sh (shipped by that image, resolves the default loader). It - is skipped for sanitizer builds, whose runtime libraries are resolved through - the rpath. Everywhere else `patch_nix_binary` is a no-op. + This runs by default for Nix-toolchain builds (determined by whether the compiler resolves under /nix/store/). + Those builds are where binaries get a Nix-store loader. + It is opted out of by setting the XRPLD_NO_PATCH_NIX_BINARY environment variable — + the plain Nix dev shells set it, since their binaries link a newer glibc + and must not be retargeted to the system loader. + + Non-Nix builds (a system compiler, already using the system loader) and sanitizer builds + (runtime libraries resolved through the rpath) are skipped too. + Everywhere else `patch_nix_binary` is a no-op. + + The default loader is resolved by bin/default-loader-path.sh. #]===================================================================] include_guard(GLOBAL) include(CompilationEnv) -# Provided by the Nix-based CI image; prints the system default ELF loader path. -set(_loader_path_script "/tmp/loader-path.sh") +# Resolves the system default ELF loader path for the current architecture. +set(_loader_path_script "${CMAKE_SOURCE_DIR}/bin/default-loader-path.sh") -if(is_linux AND NOT SANITIZERS_ENABLED AND EXISTS "${_loader_path_script}") +if( + is_linux + AND NOT SANITIZERS_ENABLED + AND is_nix_compiler + AND NOT DEFINED ENV{XRPLD_NO_PATCH_NIX_BINARY} +) execute_process( COMMAND "${_loader_path_script}" OUTPUT_VARIABLE DEFAULT_LOADER_PATH diff --git a/cmake/XrplCompiler.cmake b/cmake/XrplCompiler.cmake index cb4e797137..e262acf1c9 100644 --- a/cmake/XrplCompiler.cmake +++ b/cmake/XrplCompiler.cmake @@ -171,9 +171,8 @@ else() # Clang wrapper supplies those paths itself (via -nostdinc++), so at compile time the # flag is unused -> Clang errors under our -Werror. At link time the flag IS consumed # (it selects the C++ runtime), so we move it there instead of dropping it entirely. - get_filename_component(_cxx_real "${CMAKE_CXX_COMPILER}" REALPATH) if( - _cxx_real MATCHES "^/nix/store/" + is_nix_compiler AND is_linux AND is_clang AND CMAKE_CXX_FLAGS MATCHES "stdlib=libstdc" diff --git a/cmake/XrplSanity.cmake b/cmake/XrplSanity.cmake index a35645ad5c..ba9f7988bc 100644 --- a/cmake/XrplSanity.cmake +++ b/cmake/XrplSanity.cmake @@ -36,6 +36,19 @@ elseif(is_gcc) endif() endif() +# A Nix compiler is only meant to be used from a managed environment: the xrpld +# dev shell (which exports XRPL_DEVSHELL) or the CI image. Using one from a bare +# shell usually means a leaked toolchain (picked up via PATH or a Conan profile) +# and leads to confusing breakage, so fail early with guidance. +if(is_nix_compiler AND NOT is_ci_image AND NOT DEFINED ENV{XRPL_DEVSHELL}) + message( + FATAL_ERROR + "A Nix compiler (${CMAKE_CXX_COMPILER}) is being used outside the xrpld " + "dev shell. Enter it with `nix develop` (see docs/build/nix.md) before " + "configuring the build." + ) +endif() + # check for in-source build and fail if("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") message( diff --git a/nix/devshell.nix b/nix/devshell.nix index 9b453ddef8..cb4a99c76a 100644 --- a/nix/devshell.nix +++ b/nix/devshell.nix @@ -50,12 +50,17 @@ let # compilerName is the command used to print the version, or null for none. makeShell = { + shellName, stdenv, compilerName, version ? null, versionedTools ? [ ], extraPackages ? [ ], warningHook ? "", + # Opt out of PatchNixBinary.cmake retargeting binaries to the system + # loader. The plain toolchain links a newer glibc, so it must not be + # patched; the custom toolchain patches by default. + noPatchNixBinary ? false, }: let compilerVersionHook = @@ -73,14 +78,20 @@ let tools = versionedTools; }); in - (pkgs.mkShell.override { inherit stdenv; }) { - packages = commonPackages ++ versionedLinks ++ extraPackages; - shellHook = '' - echo "Welcome to xrpld development shell"; - ${compilerVersionHook} - ${warningHook} - ''; - }; + (pkgs.mkShell.override { inherit stdenv; }) ( + { + packages = commonPackages ++ versionedLinks ++ extraPackages; + # Marks a managed dev shell, so the build (XrplSanity.cmake) can tell an + # intentional Nix toolchain from one leaked into a bare shell. + XRPL_DEVSHELL = shellName; + shellHook = '' + echo "Welcome to xrpld development shell"; + ${compilerVersionHook} + ${warningHook} + ''; + } + // pkgs.lib.optionalAttrs noPatchNixBinary { XRPLD_NO_PATCH_NIX_BINARY = "1"; } + ); in rec { # macOS: Nix Clang. Linux: Nix GCC. @@ -89,6 +100,7 @@ rec { # gcc/clang use the custom-glibc toolchain, matching CI. On darwin there is no # custom glibc, so they fall back to the plain nixpkgs toolchain. gcc = makeShell { + shellName = "gcc"; stdenv = customGccStdenv; compilerName = "gcc"; version = gccVersion; @@ -97,6 +109,7 @@ rec { }; clang = makeShell { + shellName = "clang"; stdenv = customClangStdenv; compilerName = "clang"; version = llvmVersion; @@ -105,6 +118,7 @@ rec { # Nix provides no compiler; use the one from your system (e.g. Apple Clang). no-compiler = makeShell { + shellName = "no-compiler"; stdenv = pkgs.stdenvNoCC; compilerName = null; }; @@ -115,19 +129,23 @@ rec { # makes `nix develop .#gcc-plain` fail there rather than silently aliasing gcc. // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { gcc-plain = makeShell { + shellName = "gcc-plain"; stdenv = plainGccStdenv; compilerName = "gcc"; version = gccVersion; versionedTools = gccVersionedTools; extraPackages = [ plainGcov ]; warningHook = plainWarningHook; + noPatchNixBinary = true; }; clang-plain = makeShell { + shellName = "clang-plain"; stdenv = plainClangStdenv; compilerName = "clang"; version = llvmVersion; versionedTools = clangVersionedTools; warningHook = plainWarningHook; + noPatchNixBinary = true; }; } diff --git a/nix/docker/Dockerfile b/nix/docker/Dockerfile index 53b646ac7c..74c630cb61 100644 --- a/nix/docker/Dockerfile +++ b/nix/docker/Dockerfile @@ -58,7 +58,7 @@ ENV GIT_SSL_CAINFO="/nix/ci-env/etc/ssl/certs/ca-bundle.crt" # Externally-built dynamically-linked ELF binaries hard-code the loader path # (e.g. /lib64/ld-linux-x86-64.so.2) in their PT_INTERP header. Install it # from the Nix store when the base image doesn't already provide one. -COPY nix/docker/loader-path.sh /tmp/loader-path.sh +COPY bin/default-loader-path.sh /tmp/loader-path.sh RUN < Date: Fri, 24 Jul 2026 16:52:45 +0100 Subject: [PATCH 12/13] chore: Verify tooling version for Nix-managed environments (#7862) --- .github/workflows/check-tools.yml | 114 ++++++++++++++++++++++++++++ bin/check-tools.sh | 21 +++-- docs/build/nix.md | 8 ++ nix/check-tools/README.md | 49 ++++++++++++ nix/check-tools/macos.txt | 47 ++++++++++++ nix/check-tools/nix-nixos-amd64.txt | 55 ++++++++++++++ nix/check-tools/nix-nixos-arm64.txt | 55 ++++++++++++++ 7 files changed, 341 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/check-tools.yml create mode 100644 nix/check-tools/README.md create mode 100644 nix/check-tools/macos.txt create mode 100644 nix/check-tools/nix-nixos-amd64.txt create mode 100644 nix/check-tools/nix-nixos-arm64.txt diff --git a/.github/workflows/check-tools.yml b/.github/workflows/check-tools.yml new file mode 100644 index 0000000000..02e2038de0 --- /dev/null +++ b/.github/workflows/check-tools.yml @@ -0,0 +1,114 @@ +# Verifies the committed snapshots of `bin/check-tools.sh` output for each Nix +# environment (see nix/check-tools/). If the environment changes — a new image +# tag, an updated flake.lock, a different tool list — without the matching +# snapshot being regenerated and committed, this workflow fails so the drift is +# caught in review. +# +# To regenerate the snapshots, see nix/check-tools/README.md. +name: Check tools + +on: + pull_request: + paths: + - ".github/workflows/check-tools.yml" + - ".github/scripts/strategy-matrix/linux.json" + - "bin/check-tools.sh" + - "nix/check-tools/**" + - "flake.nix" + - "flake.lock" + - "rust-toolchain.toml" + push: + branches: + - "develop" + paths: + - ".github/workflows/check-tools.yml" + - ".github/scripts/strategy-matrix/linux.json" + - "bin/check-tools.sh" + - "nix/check-tools/**" + - "flake.nix" + - "flake.lock" + - "rust-toolchain.toml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + # The nix-nixos image tag is pinned alongside the build matrix in linux.json, + # so snapshots are checked against the exact image CI builds against. + linux-image-tag: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Read nix image tag + id: tag + run: echo "tag=$(jq -r .image_tag .github/scripts/strategy-matrix/linux.json)" >>"${GITHUB_OUTPUT}" + + # One job for all environments; they differ only in whether the tools come + # from the nix-nixos container (Linux) or `nix develop` (macOS). + check-tools: + needs: linux-image-tag + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu + snapshot: nix/check-tools/nix-nixos-amd64.txt + nix_develop: false + - runner: ubuntu-arm64 + snapshot: nix/check-tools/nix-nixos-arm64.txt + nix_develop: false + - runner: macos-26-apple-clang-21 + snapshot: nix/check-tools/macos.txt + nix_develop: true + runs-on: ${{ matrix.runner }} + # Linux runs inside the pinned nix-nixos image; macOS runs natively and uses + # the flake's dev shell instead (see the run step below). + container: ${{ !matrix.nix_develop && format('ghcr.io/xrplf/xrpld/nix-nixos:{0}', needs.linux-image-tag.outputs.tag) || null }} + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Prepare runner + uses: XRPLF/actions/prepare-runner@e4b6449d55a61c002d7c3fdfa6c20f721ede0606 + with: + enable_ccache: false + + - name: Regenerate snapshot + env: + CHECK_TOOLS_SKIP_CLONE: "1" + # check-tools.sh skips some macOS tools when CI is set; the snapshots + # capture the full `nix develop` environment, so unset it here. + CI: "" + run: | + if [ "${{ matrix.nix_develop }}" = "true" ]; then + # `nix develop` prints the dev-shell greeting first; keep only the + # check-tools.sh output (from the "Detected OS:" line onward). + nix --extra-experimental-features "nix-command flakes" develop \ + -c bash bin/check-tools.sh | sed -n '/^Detected OS:/,$p' >"${{ matrix.snapshot }}" + else + bash bin/check-tools.sh >"${{ matrix.snapshot }}" + fi + + - name: Verify snapshot is up to date + run: | + if ! git diff --exit-code -- "${{ matrix.snapshot }}"; then + echo "::error::${{ matrix.snapshot }} is out of date. Regenerate it (see nix/check-tools/README.md) and commit the result." + exit 1 + fi + + - name: Upload regenerated snapshot + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: check-tools-${{ runner.os }}-${{ runner.arch }} + path: ${{ matrix.snapshot }} diff --git a/bin/check-tools.sh b/bin/check-tools.sh index 7886bcf8b0..e230302742 100755 --- a/bin/check-tools.sh +++ b/bin/check-tools.sh @@ -30,8 +30,10 @@ missing=() checked=0 # check [probe-command...] -# Runs the probe (default: " --version") quietly. Records as -# missing if the command is not found or exits non-zero. +# Runs the probe (default: " --version"), capturing both stdout and +# stderr, and prints one aligned line: the status, the name, and the first +# non-blank line of the probe output (its version). Records as missing +# if the command is not found or exits non-zero. check() { local name="$1" shift @@ -40,10 +42,11 @@ check() { probe=("${name}" --version) fi - echo "Checking ${name}..." checked=$((checked + 1)) - if "${probe[@]}" | head -n 1; then - printf ' [ ok ] %s\n' "${name}" + local output version + if output="$("${probe[@]}" 2>&1)"; then + version="$(printf '%s\n' "${output}" | grep -m1 '[^[:space:]]' || true)" + printf ' [ ok ] %-20s %s\n' "${name}" "${version}" else printf ' [MISS] %s\n' "${name}" missing+=("${name}") @@ -85,12 +88,14 @@ if [ "${os}" = "linux" ] || [ "${os}" = "macos" ]; then check file check less check make - check netstat which netstat + # net-tools netstat reports "net-tools X.Y"; macOS ships BSD netstat with no + # version flag, so fall back to a presence marker there. + check netstat sh -c 'command -v netstat >/dev/null && { netstat --version 2>&1 | grep -m1 -oE "net-tools [0-9.]+" || echo present; }' check ninja - check perl + check perl perl -e 'print "$^V\n"' check pkg-config check vim - check zip + check zip bash -c 'zip --version 2>&1 | grep -m1 -oE "Zip [0-9.]+"' # These tools are present in our Linux CI images and in local development # setups, but not in the macOS CI environment. So check them everywhere diff --git a/docs/build/nix.md b/docs/build/nix.md index b95b82fc42..d0001294e3 100644 --- a/docs/build/nix.md +++ b/docs/build/nix.md @@ -154,6 +154,14 @@ conan install .. --output-folder . --build '*' --settings build_type=Release To update `flake.lock` to the latest revision use `nix flake update` command. +## Tooling snapshots + +The tool versions in each Nix environment are recorded in +[`nix/check-tools/`](../../nix/check-tools) and verified by CI. If you change the +environment (bump the CI image tag, update `flake.lock`, or edit the tool list in +`bin/check-tools.sh`), CI fails until you regenerate and commit the affected +snapshot — see [`nix/check-tools/README.md`](../../nix/check-tools/README.md). + ## Troubleshooting See [Troubleshooting Nix problems](./nix_troubleshooting.md) for common issues, diff --git a/nix/check-tools/README.md b/nix/check-tools/README.md new file mode 100644 index 0000000000..fe9ab3e250 --- /dev/null +++ b/nix/check-tools/README.md @@ -0,0 +1,49 @@ +# check-tools snapshots + +These files capture the output of [`bin/check-tools.sh`](../../bin/check-tools.sh) +— the versions of the development tooling — in each Nix environment: + +| File | Environment | +| --------------------- | ----------------------------------- | +| `nix-nixos-amd64.txt` | `nix-nixos` CI image, `linux/amd64` | +| `nix-nixos-arm64.txt` | `nix-nixos` CI image, `linux/arm64` | +| `macos.txt` | macOS, inside `nix develop` | + +The [`check-tools`](../../.github/workflows/check-tools.yml) workflow regenerates +each snapshot in its environment and fails if it differs from the committed file. +So if you change the environment (bump the image tag in +[`linux.json`](../../.github/scripts/strategy-matrix/linux.json), update +`flake.lock`, change the tool list in `check-tools.sh`, …) you must regenerate +and commit the affected snapshots. + +Each snapshot is `check-tools.sh` stdout with the git-clone connectivity check +skipped (`CHECK_TOOLS_SKIP_CLONE=1`), so it contains only deterministic version +data. On macOS the dev-shell greeting that `nix develop` prints first is dropped +with `sed -n '/^Detected OS:/,$p'`. + +## Regenerating + +The two Linux snapshots come from the `nix-nixos` image (Docker or a compatible +runtime such as Apple `container`). The image tag is pinned in `linux.json`: + +```bash +img="ghcr.io/xrplf/xrpld/nix-nixos:$(jq -r .image_tag .github/scripts/strategy-matrix/linux.json)" + +for arch in amd64 arm64; do + container run --rm -i -e CHECK_TOOLS_SKIP_CLONE=1 -a "${arch}" --entrypoint bash "${img}" -s \ + "nix/check-tools/nix-nixos-${arch}.txt" +done +``` + +(With Docker, replace `container run … -a "${arch}"` with +`docker run … --platform "linux/${arch}"`.) + +The macOS snapshot is generated locally. `CI=` is unset so `check-tools.sh` +checks the full dev-shell tool set (it otherwise skips some tools when `CI` is +set): + +```bash +CI= nix develop -c bash -c 'CHECK_TOOLS_SKIP_CLONE=1 bash bin/check-tools.sh' | + sed -n '/^Detected OS:/,$p' \ + >nix/check-tools/macos.txt +``` diff --git a/nix/check-tools/macos.txt b/nix/check-tools/macos.txt new file mode 100644 index 0000000000..af9814e9cb --- /dev/null +++ b/nix/check-tools/macos.txt @@ -0,0 +1,47 @@ +Detected OS: macos (Darwin arm64) + +Core build tools: + [ ok ] cmake cmake version 4.1.2 + [ ok ] conan Conan version 2.28.1 + [ ok ] git git version 2.54.0 + [ ok ] python3 Python 3.13.13 + +Development tooling: + [ ok ] ccache ccache version 4.13.6 + [ ok ] clang clang version 21.1.8 + [ ok ] clang++ clang version 21.1.8 + [ ok ] ClangBuildAnalyzer ClangBuildAnalyzer 1.6.0 + [ ok ] curl curl 8.20.0 (aarch64-apple-darwin25.3.0) libcurl/8.20.0 OpenSSL/3.6.2 zlib/1.3.2 libssh2/1.11.1 nghttp2/1.69.0 mit-krb5/1.22.1 + [ ok ] file file-5.47 + [ ok ] less less 692 (PCRE2 regular expressions) + [ ok ] make GNU Make 4.4.1 + [ ok ] netstat present + [ ok ] ninja 1.13.2 + [ ok ] perl v5.42.0 + [ ok ] pkg-config 0.29.2 + [ ok ] vim VIM - Vi IMproved 9.2 (2026 Feb 14, compiled Jan 01 1980 00:00:00) + [ ok ] zip Zip 3.0 + [ ok ] clang-format clang-format version 21.1.8 + [ ok ] dot dot - graphviz version 12.2.1 (0) + [ ok ] doxygen 1.16.1 + [ ok ] gcovr gcovr 8.4 + [ ok ] gh gh version 2.94.0 (nixpkgs) + [ ok ] git-cliff git-cliff 2.13.1 + [ ok ] git-lfs git-lfs/3.7.1 (3.7.1; darwin arm64; go 1.26.3) + [ ok ] gpg gpg (GnuPG) 2.4.9 + [ ok ] pre-commit pre-commit 4.5.1 + [ ok ] run-clang-tidy usage: run-clang-tidy [-h] [-allow-enabling-alpha-checkers] + +Rust toolchain: + [ ok ] cargo cargo 1.95.0 (f2d3ce0bd 2026-03-21) + [ ok ] cargo-audit cargo-audit-audit 0.22.1 + [ ok ] cargo-llvm-cov cargo-llvm-cov 0.8.5 + [ ok ] cargo-nextest cargo-nextest 0.9.137 + [ ok ] clippy clippy 0.1.95 (59807616e1 2026-04-14) + [ ok ] rust-analyzer rust-analyzer 1.95.0 (59807616 2026-04-14) + [ ok ] rustc rustc 1.95.0 (59807616e 2026-04-14) + [ ok ] rustfmt rustfmt 1.9.0-stable (59807616e1 2026-04-14) + +Skipping git-over-HTTPS check (CHECK_TOOLS_SKIP_CLONE is set). + +All 36 checked tools are present and runnable. diff --git a/nix/check-tools/nix-nixos-amd64.txt b/nix/check-tools/nix-nixos-amd64.txt new file mode 100644 index 0000000000..b922cca4a8 --- /dev/null +++ b/nix/check-tools/nix-nixos-amd64.txt @@ -0,0 +1,55 @@ +Detected OS: linux (Linux x86_64) + +Core build tools: + [ ok ] cmake cmake version 4.1.2 + [ ok ] conan Conan version 2.28.1 + [ ok ] git git version 2.54.0 + [ ok ] python3 Python 3.13.13 + +Development tooling: + [ ok ] ccache ccache version 4.13.6 + [ ok ] clang clang version 22.1.7 + [ ok ] clang++ clang version 22.1.7 + [ ok ] ClangBuildAnalyzer ClangBuildAnalyzer 1.6.0 + [ ok ] curl curl 8.20.0 (x86_64-pc-linux-gnu) libcurl/8.20.0 OpenSSL/3.6.2 zlib/1.3.2 libssh2/1.11.1 nghttp2/1.69.0 mit-krb5/1.22.1 + [ ok ] file file-5.47 + [ ok ] less less 692 (PCRE2 regular expressions) + [ ok ] make GNU Make 4.4.1 + [ ok ] netstat net-tools 2.10 + [ ok ] ninja 1.13.2 + [ ok ] perl v5.42.0 + [ ok ] pkg-config 0.29.2 + [ ok ] vim VIM - Vi IMproved 9.2 (2026 Feb 14, compiled Jan 01 1980 00:00:00) + [ ok ] zip Zip 3.0 + [ ok ] clang-format clang-format version 22.1.7 + [ ok ] dot dot - graphviz version 12.2.1 (0) + [ ok ] doxygen 1.16.1 + [ ok ] gcovr gcovr 8.4 + [ ok ] gh gh version 2.94.0 (nixpkgs) + [ ok ] git-cliff git-cliff 2.13.1 + [ ok ] git-lfs git-lfs/3.7.1 (3.7.1; linux amd64; go 1.26.3) + [ ok ] gpg gpg (GnuPG) 2.4.9 + [ ok ] pre-commit pre-commit 4.5.1 + [ ok ] run-clang-tidy usage: run-clang-tidy [-h] [-allow-enabling-alpha-checkers] + +Rust toolchain: + [ ok ] cargo cargo 1.95.0 (f2d3ce0bd 2026-03-21) + [ ok ] cargo-audit cargo-audit-audit 0.22.1 + [ ok ] cargo-llvm-cov cargo-llvm-cov 0.8.5 + [ ok ] cargo-nextest cargo-nextest 0.9.137 + [ ok ] clippy clippy 0.1.95 (59807616e1 2026-04-14) + [ ok ] rust-analyzer rust-analyzer 1.95.0 (5980761 2026-04-14) + [ ok ] rustc rustc 1.95.0 (59807616e 2026-04-14) + [ ok ] rustfmt rustfmt 1.9.0-stable (59807616e1 2026-04-14) + +GCC toolchain: + [ ok ] gcc gcc (GCC) 15.2.0 + [ ok ] g++ g++ (GCC) 15.2.0 + [ ok ] gcov gcov (GCC) 15.2.0 + +Mold: + [ ok ] mold mold 2.41.0 (compatible with GNU ld) + +Skipping git-over-HTTPS check (CHECK_TOOLS_SKIP_CLONE is set). + +All 40 checked tools are present and runnable. diff --git a/nix/check-tools/nix-nixos-arm64.txt b/nix/check-tools/nix-nixos-arm64.txt new file mode 100644 index 0000000000..5267839682 --- /dev/null +++ b/nix/check-tools/nix-nixos-arm64.txt @@ -0,0 +1,55 @@ +Detected OS: linux (Linux aarch64) + +Core build tools: + [ ok ] cmake cmake version 4.1.2 + [ ok ] conan Conan version 2.28.1 + [ ok ] git git version 2.54.0 + [ ok ] python3 Python 3.13.13 + +Development tooling: + [ ok ] ccache ccache version 4.13.6 + [ ok ] clang clang version 22.1.7 + [ ok ] clang++ clang version 22.1.7 + [ ok ] ClangBuildAnalyzer ClangBuildAnalyzer 1.6.0 + [ ok ] curl curl 8.20.0 (aarch64-unknown-linux-gnu) libcurl/8.20.0 OpenSSL/3.6.2 zlib/1.3.2 libssh2/1.11.1 nghttp2/1.69.0 mit-krb5/1.22.1 + [ ok ] file file-5.47 + [ ok ] less less 692 (PCRE2 regular expressions) + [ ok ] make GNU Make 4.4.1 + [ ok ] netstat net-tools 2.10 + [ ok ] ninja 1.13.2 + [ ok ] perl v5.42.0 + [ ok ] pkg-config 0.29.2 + [ ok ] vim VIM - Vi IMproved 9.2 (2026 Feb 14, compiled Jan 01 1980 00:00:00) + [ ok ] zip Zip 3.0 + [ ok ] clang-format clang-format version 22.1.7 + [ ok ] dot dot - graphviz version 12.2.1 (0) + [ ok ] doxygen 1.16.1 + [ ok ] gcovr gcovr 8.4 + [ ok ] gh gh version 2.94.0 (nixpkgs) + [ ok ] git-cliff git-cliff 2.13.1 + [ ok ] git-lfs git-lfs/3.7.1 (3.7.1; linux arm64; go 1.26.3) + [ ok ] gpg gpg (GnuPG) 2.4.9 + [ ok ] pre-commit pre-commit 4.5.1 + [ ok ] run-clang-tidy usage: run-clang-tidy [-h] [-allow-enabling-alpha-checkers] + +Rust toolchain: + [ ok ] cargo cargo 1.95.0 (f2d3ce0bd 2026-03-21) + [ ok ] cargo-audit cargo-audit-audit 0.22.1 + [ ok ] cargo-llvm-cov cargo-llvm-cov 0.8.5 + [ ok ] cargo-nextest cargo-nextest 0.9.137 + [ ok ] clippy clippy 0.1.95 (59807616e1 2026-04-14) + [ ok ] rust-analyzer rust-analyzer 1.95.0 (5980761 2026-04-14) + [ ok ] rustc rustc 1.95.0 (59807616e 2026-04-14) + [ ok ] rustfmt rustfmt 1.9.0-stable (59807616e1 2026-04-14) + +GCC toolchain: + [ ok ] gcc gcc (GCC) 15.2.0 + [ ok ] g++ g++ (GCC) 15.2.0 + [ ok ] gcov gcov (GCC) 15.2.0 + +Mold: + [ ok ] mold mold 2.41.0 (compatible with GNU ld) + +Skipping git-over-HTTPS check (CHECK_TOOLS_SKIP_CLONE is set). + +All 40 checked tools are present and runnable. From fecfc0cf3faf88799e1bc3b40dff821b1572ffc0 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Fri, 24 Jul 2026 18:30:20 +0100 Subject: [PATCH 13/13] chore: Fix clang version in devshell (#7860) Co-authored-by: Ayaz Salikhov --- .github/workflows/check-tools.yml | 14 +++++++------- nix/check-tools/macos.txt | 6 +++--- ...ix-nixos-amd64.txt => nix-ubuntu-amd64.txt} | 0 ...ix-nixos-arm64.txt => nix-ubuntu-arm64.txt} | 0 nix/packages.nix | 18 +++++++++++++++++- 5 files changed, 27 insertions(+), 11 deletions(-) rename nix/check-tools/{nix-nixos-amd64.txt => nix-ubuntu-amd64.txt} (100%) rename nix/check-tools/{nix-nixos-arm64.txt => nix-ubuntu-arm64.txt} (100%) diff --git a/.github/workflows/check-tools.yml b/.github/workflows/check-tools.yml index 02e2038de0..6daaf98114 100644 --- a/.github/workflows/check-tools.yml +++ b/.github/workflows/check-tools.yml @@ -13,7 +13,7 @@ on: - ".github/workflows/check-tools.yml" - ".github/scripts/strategy-matrix/linux.json" - "bin/check-tools.sh" - - "nix/check-tools/**" + - "nix/**" - "flake.nix" - "flake.lock" - "rust-toolchain.toml" @@ -24,7 +24,7 @@ on: - ".github/workflows/check-tools.yml" - ".github/scripts/strategy-matrix/linux.json" - "bin/check-tools.sh" - - "nix/check-tools/**" + - "nix/**" - "flake.nix" - "flake.lock" - "rust-toolchain.toml" @@ -61,11 +61,11 @@ jobs: fail-fast: false matrix: include: - - runner: ubuntu - snapshot: nix/check-tools/nix-nixos-amd64.txt + - runner: ubuntu-latest + snapshot: nix/check-tools/nix-ubuntu-amd64.txt nix_develop: false - - runner: ubuntu-arm64 - snapshot: nix/check-tools/nix-nixos-arm64.txt + - runner: ubuntu-24.04-arm + snapshot: nix/check-tools/nix-ubuntu-arm64.txt nix_develop: false - runner: macos-26-apple-clang-21 snapshot: nix/check-tools/macos.txt @@ -73,7 +73,7 @@ jobs: runs-on: ${{ matrix.runner }} # Linux runs inside the pinned nix-nixos image; macOS runs natively and uses # the flake's dev shell instead (see the run step below). - container: ${{ !matrix.nix_develop && format('ghcr.io/xrplf/xrpld/nix-nixos:{0}', needs.linux-image-tag.outputs.tag) || null }} + container: ${{ !matrix.nix_develop && format('ghcr.io/xrplf/xrpld/nix-ubuntu:{0}', needs.linux-image-tag.outputs.tag) || null }} steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/nix/check-tools/macos.txt b/nix/check-tools/macos.txt index af9814e9cb..93cc926181 100644 --- a/nix/check-tools/macos.txt +++ b/nix/check-tools/macos.txt @@ -8,8 +8,8 @@ Core build tools: Development tooling: [ ok ] ccache ccache version 4.13.6 - [ ok ] clang clang version 21.1.8 - [ ok ] clang++ clang version 21.1.8 + [ ok ] clang clang version 22.1.7 + [ ok ] clang++ clang version 22.1.7 [ ok ] ClangBuildAnalyzer ClangBuildAnalyzer 1.6.0 [ ok ] curl curl 8.20.0 (aarch64-apple-darwin25.3.0) libcurl/8.20.0 OpenSSL/3.6.2 zlib/1.3.2 libssh2/1.11.1 nghttp2/1.69.0 mit-krb5/1.22.1 [ ok ] file file-5.47 @@ -21,7 +21,7 @@ Development tooling: [ ok ] pkg-config 0.29.2 [ ok ] vim VIM - Vi IMproved 9.2 (2026 Feb 14, compiled Jan 01 1980 00:00:00) [ ok ] zip Zip 3.0 - [ ok ] clang-format clang-format version 21.1.8 + [ ok ] clang-format clang-format version 22.1.7 [ ok ] dot dot - graphviz version 12.2.1 (0) [ ok ] doxygen 1.16.1 [ ok ] gcovr gcovr 8.4 diff --git a/nix/check-tools/nix-nixos-amd64.txt b/nix/check-tools/nix-ubuntu-amd64.txt similarity index 100% rename from nix/check-tools/nix-nixos-amd64.txt rename to nix/check-tools/nix-ubuntu-amd64.txt diff --git a/nix/check-tools/nix-nixos-arm64.txt b/nix/check-tools/nix-ubuntu-arm64.txt similarity index 100% rename from nix/check-tools/nix-nixos-arm64.txt rename to nix/check-tools/nix-ubuntu-arm64.txt diff --git a/nix/packages.nix b/nix/packages.nix index 3cf0f57c3e..01ab2ecf9a 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -16,7 +16,23 @@ let exec ${pkgs.python3}/bin/python3 ${llvmPackages.clang-unwrapped}/bin/run-clang-tidy "$@" ''; - rustToolchain = pkgs.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml; + # rust-overlay's toolchain propagates the *default* stdenv.cc onto the PATH (so + # cargo has a linker). That default may be different from the clang we pin here, + # so it shadows our clang and the build can silently use a different compiler + # version. Drop that cc from every propagation channel instead of pinning a + # replacement: the toolchain then carries no compiler and cargo just uses the + # active shell's stdenv cc. Must cover all channels — rust-overlay uses both + # propagatedBuildInputs and depsHostHostPropagated. + rustToolchainBase = pkgs.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml; + rustToolchain = + let + defaultCc = pkgs.stdenv.cc; # default compiler from nixpkgs stdenv + withoutDefaultCc = builtins.filter (dep: (dep.outPath or "") != defaultCc.outPath); + in + rustToolchainBase.overrideAttrs (old: { + propagatedBuildInputs = withoutDefaultCc (old.propagatedBuildInputs or [ ]); + depsHostHostPropagated = withoutDefaultCc (old.depsHostHostPropagated or [ ]); + }); # Nix wraps its toolchain so that binaries are exposed only under unsuffixed # names (gcc, g++, clang-tidy, ...). Several tools probe for a