diff --git a/.cspell.config.yaml b/.cspell.config.yaml index 081bdae2e4..4081002552 100644 --- a/.cspell.config.yaml +++ b/.cspell.config.yaml @@ -30,7 +30,9 @@ ignoreRegExpList: - ABCDEFGHIJKLMNOPQRSTUVWXYZ - ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz overrides: - - filename: "**/*_test.cpp" # all test files + - filename: + - "**/*_test.cpp" # legacy boost.test files + - "src/tests/**/*.cpp" # gtest test files ignoreRegExpList: - /"[^"]*"/g # double-quoted strings - /'[^']*'/g # single-quoted strings @@ -220,6 +222,7 @@ words: - onlatest - ostr - otelc + - oxalica - pargs - partitioner - paychan @@ -347,6 +350,7 @@ words: - unsquelch - unsquelched - unsquelching + - unsuffixed - unvalidated - unveto - unvetoed diff --git a/.github/dependabot.yml b/.github/dependabot.yml index da7a30dc77..fcac44c44c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,3 +15,7 @@ updates: commit-message: prefix: "ci: [DEPENDABOT] " target-branch: develop + groups: + github-actions: + patterns: + - "*" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index f1f7aa18f7..95d75c04b4 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,10 +1,10 @@ @@ -15,7 +15,7 @@ https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your- Please include a summary of the changes. This may be a direct input to the release notes. If too broad, please consider splitting into multiple PRs. -If a relevant task or issue, please link it here. +If there is a relevant task or issue, please link it here. --> ### Context of Change @@ -65,5 +65,5 @@ This section may not be needed if your change includes thoroughly commented unit diff --git a/.github/scripts/levelization/README.md b/.github/scripts/levelization/README.md index f657344827..93748c43e1 100644 --- a/.github/scripts/levelization/README.md +++ b/.github/scripts/levelization/README.md @@ -40,18 +40,18 @@ listed later. | 04 | xrpl/protocol | | 05 | xrpl/core xrpl/resource xrpl/server | | 06 | xrpl/ledger xrpl/nodestore xrpl/net | -| 07 | xrpl/shamap | +| 07 | xrpl/shamap xrpl/consensus | ## xrpld Modules (Application Implementation) -| Level / Tier | Module(s) | -| ------------ | -------------------------------- | -| 05 | xrpld/conditions xrpld/consensus | -| 06 | xrpld/core xrpld/peerfinder | -| 07 | xrpld/shamap xrpld/overlay | -| 08 | xrpld/app | -| 09 | xrpld/rpc | -| 10 | xrpld/perflog | +| Level / Tier | Module(s) | +| ------------ | --------------------------- | +| 05 | xrpld/conditions | +| 06 | xrpld/core xrpld/peerfinder | +| 07 | xrpld/shamap xrpld/overlay | +| 08 | xrpld/app | +| 09 | xrpld/rpc | +| 10 | xrpld/perflog | ## Test Modules diff --git a/.github/scripts/levelization/results/loops.txt b/.github/scripts/levelization/results/loops.txt index cf70468e32..ea7b8a372a 100644 --- a/.github/scripts/levelization/results/loops.txt +++ b/.github/scripts/levelization/results/loops.txt @@ -1,9 +1,6 @@ Loop: xrpld.app xrpld.overlay xrpld.app > xrpld.overlay -Loop: xrpld.app xrpld.peerfinder - xrpld.peerfinder ~= xrpld.app - Loop: xrpld.app xrpld.rpc xrpld.rpc > xrpld.app diff --git a/.github/scripts/levelization/results/ordering.txt b/.github/scripts/levelization/results/ordering.txt index 9538b32802..72b93ac5ec 100644 --- a/.github/scripts/levelization/results/ordering.txt +++ b/.github/scripts/levelization/results/ordering.txt @@ -6,6 +6,8 @@ libxrpl.conditions > xrpl.basics libxrpl.conditions > xrpl.conditions libxrpl.config > xrpl.basics libxrpl.config > xrpl.config +libxrpl.consensus > xrpl.basics +libxrpl.consensus > xrpl.consensus libxrpl.core > xrpl.basics libxrpl.core > xrpl.core libxrpl.core > xrpl.json @@ -25,6 +27,9 @@ libxrpl.nodestore > xrpl.config libxrpl.nodestore > xrpl.json libxrpl.nodestore > xrpl.nodestore libxrpl.nodestore > xrpl.protocol +libxrpl.peerfinder > xrpl.basics +libxrpl.peerfinder > xrpl.peerfinder +libxrpl.peerfinder > xrpl.protocol libxrpl.protocol > xrpl.basics libxrpl.protocol > xrpl.json libxrpl.protocol > xrpl.protocol @@ -63,9 +68,9 @@ test.app > test.jtx test.app > test.unit_test test.app > xrpl.basics test.app > xrpl.config +test.app > xrpl.consensus test.app > xrpl.core test.app > xrpld.app -test.app > xrpld.consensus test.app > xrpld.core test.app > xrpld.overlay test.app > xrpld.rpc @@ -86,12 +91,9 @@ test.basics > xrpl.protocol test.beast > xrpl.basics test.conditions > xrpl.basics test.conditions > xrpl.conditions -test.consensus > test.csf test.consensus > test.jtx -test.consensus > test.unit_test test.consensus > xrpl.basics test.consensus > xrpld.app -test.consensus > xrpld.consensus test.consensus > xrpl.ledger test.consensus > xrpl.protocol test.consensus > xrpl.shamap @@ -106,10 +108,6 @@ test.core > xrpl.json test.core > xrpl.protocol test.core > xrpl.rdb test.core > xrpl.server -test.csf > xrpl.basics -test.csf > xrpld.consensus -test.csf > xrpl.json -test.csf > xrpl.ledger test.json > test.jtx test.json > xrpl.json test.jtx > test.unit_test @@ -135,12 +133,9 @@ test.ledger > xrpl.json test.ledger > xrpl.ledger test.ledger > xrpl.protocol test.nodestore > test.jtx -test.nodestore > test.unit_test test.nodestore > xrpl.basics -test.nodestore > xrpl.config test.nodestore > xrpld.core test.nodestore > xrpl.nodestore -test.nodestore > xrpl.protocol test.nodestore > xrpl.rdb test.overlay > test.jtx test.overlay > test.unit_test @@ -149,19 +144,13 @@ test.overlay > xrpl.config test.overlay > xrpld.app test.overlay > xrpld.core test.overlay > xrpld.overlay -test.overlay > xrpld.peerfinder test.overlay > xrpl.json test.overlay > xrpl.nodestore +test.overlay > xrpl.peerfinder test.overlay > xrpl.protocol test.overlay > xrpl.resource test.overlay > xrpl.server test.overlay > xrpl.shamap -test.peerfinder > test.beast -test.peerfinder > test.unit_test -test.peerfinder > xrpl.basics -test.peerfinder > xrpld.core -test.peerfinder > xrpld.peerfinder -test.peerfinder > xrpl.protocol test.protocol > test.jtx test.protocol > test.unit_test test.protocol > xrpl.basics @@ -195,11 +184,13 @@ test.unit_test > xrpl.basics test.unit_test > xrpl.protocol tests.libxrpl > xrpl.basics tests.libxrpl > xrpl.config +tests.libxrpl > xrpl.consensus tests.libxrpl > xrpl.core tests.libxrpl > xrpl.json tests.libxrpl > xrpl.ledger tests.libxrpl > xrpl.net tests.libxrpl > xrpl.nodestore +tests.libxrpl > xrpl.peerfinder tests.libxrpl > xrpl.protocol tests.libxrpl > xrpl.protocol_autogen tests.libxrpl > xrpl.resource @@ -210,6 +201,10 @@ tests.libxrpl > xrpl.tx xrpl.conditions > xrpl.basics xrpl.conditions > xrpl.protocol xrpl.config > xrpl.basics +xrpl.consensus > xrpl.basics +xrpl.consensus > xrpl.json +xrpl.consensus > xrpl.ledger +xrpl.consensus > xrpl.protocol xrpl.core > xrpl.basics xrpl.core > xrpl.json xrpl.core > xrpl.protocol @@ -224,6 +219,8 @@ xrpl.nodestore > xrpl.basics xrpl.nodestore > xrpl.config xrpl.nodestore > xrpl.json xrpl.nodestore > xrpl.protocol +xrpl.peerfinder > xrpl.basics +xrpl.peerfinder > xrpl.protocol xrpl.protocol > xrpl.basics xrpl.protocol > xrpl.json xrpl.protocol_autogen > xrpl.json @@ -252,13 +249,14 @@ xrpl.tx > xrpl.protocol xrpld.app > test.unit_test xrpld.app > xrpl.basics xrpld.app > xrpl.config +xrpld.app > xrpl.consensus xrpld.app > xrpl.core -xrpld.app > xrpld.consensus xrpld.app > xrpld.core xrpld.app > xrpl.json xrpld.app > xrpl.ledger xrpld.app > xrpl.net xrpld.app > xrpl.nodestore +xrpld.app > xrpl.peerfinder xrpld.app > xrpl.protocol xrpld.app > xrpl.rdb xrpld.app > xrpl.resource @@ -266,10 +264,6 @@ xrpld.app > xrpl.server xrpld.app > xrpl.shamap xrpld.app > xrpl.telemetry xrpld.app > xrpl.tx -xrpld.consensus > xrpl.basics -xrpld.consensus > xrpl.json -xrpld.consensus > xrpl.ledger -xrpld.consensus > xrpl.protocol xrpld.core > xrpl.basics xrpld.core > xrpl.config xrpld.core > xrpl.core @@ -278,21 +272,22 @@ xrpld.core > xrpl.protocol xrpld.core > xrpl.rdb xrpld.overlay > xrpl.basics xrpld.overlay > xrpl.config +xrpld.overlay > xrpl.consensus xrpld.overlay > xrpl.core -xrpld.overlay > xrpld.consensus xrpld.overlay > xrpld.core xrpld.overlay > xrpld.peerfinder xrpld.overlay > xrpl.json xrpld.overlay > xrpl.ledger +xrpld.overlay > xrpl.peerfinder xrpld.overlay > xrpl.protocol xrpld.overlay > xrpl.resource xrpld.overlay > xrpl.server xrpld.overlay > xrpl.shamap xrpld.overlay > xrpl.tx xrpld.peerfinder > xrpl.basics -xrpld.peerfinder > xrpl.config +xrpld.peerfinder > xrpld.app xrpld.peerfinder > xrpld.core -xrpld.peerfinder > xrpl.protocol +xrpld.peerfinder > xrpl.peerfinder xrpld.peerfinder > xrpl.rdb xrpld.perflog > xrpl.basics xrpld.perflog > xrpl.config diff --git a/.github/scripts/rename/namespace.sh b/.github/scripts/rename/namespace.sh index bb186bc8bc..94a9205f76 100755 --- a/.github/scripts/rename/namespace.sh +++ b/.github/scripts/rename/namespace.sh @@ -46,9 +46,6 @@ for DIRECTORY in "${DIRECTORIES[@]}"; do done done -# Special case for NuDBFactory that has ripple twice in the test suite name. -${SED_COMMAND} -i -E 's/(BEAST_DEFINE_TESTSUITE.+)ripple(.+)/\1xrpl\2/g' src/test/nodestore/NuDBFactory_test.cpp - DIRECTORY=$1 find "${DIRECTORY}" -type f -name "*.md" | while read -r FILE; do echo "Processing file: ${FILE}" diff --git a/.github/scripts/strategy-matrix/linux.json b/.github/scripts/strategy-matrix/linux.json index 7edbf96ef6..60f3da09f1 100644 --- a/.github/scripts/strategy-matrix/linux.json +++ b/.github/scripts/strategy-matrix/linux.json @@ -1,5 +1,5 @@ { - "image_tag": "sha-2e25435", + "image_tag": "sha-fecfc0c", "configs": { "ubuntu": [ { diff --git a/.github/workflows/build-nix-images.yml b/.github/workflows/build-nix-images.yml index 54911ef6e0..fe2f43fdcc 100644 --- a/.github/workflows/build-nix-images.yml +++ b/.github/workflows/build-nix-images.yml @@ -1,4 +1,4 @@ -name: Build Nix Docker images +name: Build `nix` Docker images on: push: @@ -8,20 +8,24 @@ on: - ".github/workflows/build-nix-images.yml" - "flake.nix" - "flake.lock" + - "rust-toolchain.toml" - "nix/**" - "!nix/docker/README.md" - "!nix/devshell.nix" - "bin/check-tools.sh" + - "bin/default-loader-path.sh" - "bin/install-sanitizer-libs.sh" pull_request: paths: - ".github/workflows/build-nix-images.yml" - "flake.nix" - "flake.lock" + - "rust-toolchain.toml" - "nix/**" - "!nix/docker/README.md" - "!nix/devshell.nix" - "bin/check-tools.sh" + - "bin/default-loader-path.sh" - "bin/install-sanitizer-libs.sh" workflow_dispatch: @@ -36,7 +40,7 @@ defaults: jobs: build-merge: - name: Build and push nix-${{ matrix.distro.name }} + name: Build and push `nix-${{ matrix.distro.name }}` image permissions: contents: read packages: write @@ -54,7 +58,7 @@ jobs: base_image: debian:bookworm - name: rhel base_image: registry.access.redhat.com/ubi9/ubi:latest - uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@ee03d31bcc4501d7599dc1b1ecd7a34af582ad1c + uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32 with: image_name: xrpld/nix-${{ matrix.distro.name }} dockerfile: nix/docker/Dockerfile diff --git a/.github/workflows/build-packaging-images.yml b/.github/workflows/build-packaging-images.yml index 3633847ef3..43b276bdf1 100644 --- a/.github/workflows/build-packaging-images.yml +++ b/.github/workflows/build-packaging-images.yml @@ -1,4 +1,4 @@ -name: Build packaging Docker images +name: Build `packaging` Docker images on: push: @@ -26,7 +26,7 @@ defaults: jobs: build-merge: - name: Build and push packaging-${{ matrix.distro.name }} + name: Build and push `packaging-${{ matrix.distro.name }}` image permissions: contents: read packages: write @@ -38,7 +38,7 @@ jobs: base_image: debian:bookworm - name: rhel base_image: registry.access.redhat.com/ubi9/ubi:latest - uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@ee03d31bcc4501d7599dc1b1ecd7a34af582ad1c + uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32 with: image_name: xrpld/packaging-${{ matrix.distro.name }} dockerfile: package/Dockerfile diff --git a/.github/workflows/build-pre-commit-image.yml b/.github/workflows/build-pre-commit-image.yml new file mode 100644 index 0000000000..d0eba6b495 --- /dev/null +++ b/.github/workflows/build-pre-commit-image.yml @@ -0,0 +1,38 @@ +name: Build `pre-commit` Docker image + +on: + push: + branches: + - develop + paths: + - ".github/workflows/build-pre-commit-image.yml" + - "bin/pre-commit/Dockerfile" + - "rust-toolchain.toml" + pull_request: + paths: + - ".github/workflows/build-pre-commit-image.yml" + - "bin/pre-commit/Dockerfile" + - "rust-toolchain.toml" + workflow_dispatch: + +concurrency: + # Read `on-trigger.yml` for the rationale behind this concurrency group name. + group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' && github.sha || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + build-merge: + name: Build and push `pre-commit` image + permissions: + contents: read + packages: write + uses: XRPLF/actions/.github/workflows/build-multiarch-image.yml@9e7e4e80af9e684c116b38369add8eea64451f32 + with: + image_name: xrpld/pre-commit + dockerfile: bin/pre-commit/Dockerfile + base_image: ubuntu:26.04 + push: ${{ github.event_name == 'push' }} diff --git a/.github/workflows/check-pr-description.yml b/.github/workflows/check-pr-description.yml index 744449f216..f8e7b6cdc4 100644 --- a/.github/workflows/check-pr-description.yml +++ b/.github/workflows/check-pr-description.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Write PR body to file env: diff --git a/.github/workflows/check-pr-title.yml b/.github/workflows/check-pr-title.yml index 4b5f679df1..cc80982440 100644 --- a/.github/workflows/check-pr-title.yml +++ b/.github/workflows/check-pr-title.yml @@ -20,4 +20,4 @@ on: jobs: check_title: if: ${{ github.event.pull_request.draft != true }} - uses: XRPLF/actions/.github/workflows/check-pr-title.yml@cba1f0891650baf1a9c88624dc2d72573be2eb81 + uses: XRPLF/actions/.github/workflows/check-pr-title.yml@d7c65e49225a38f6d8010eacf017bb5a98d7476c diff --git a/.github/workflows/check-tools.yml b/.github/workflows/check-tools.yml new file mode 100644 index 0000000000..99dddd7d96 --- /dev/null +++ b/.github/workflows/check-tools.yml @@ -0,0 +1,114 @@ +# Verifies the committed snapshots of `bin/check-tools.sh` output for each Nix +# environment (see nix/check-tools/). If the environment changes — a new image +# tag, an updated flake.lock, a different tool list — without the matching +# snapshot being regenerated and committed, this workflow fails so the drift is +# caught in review. +# +# To regenerate the snapshots, see nix/check-tools/README.md. +name: Check tools + +on: + pull_request: + paths: + - ".github/workflows/check-tools.yml" + - ".github/scripts/strategy-matrix/linux.json" + - "bin/check-tools.sh" + - "nix/**" + - "flake.nix" + - "flake.lock" + - "rust-toolchain.toml" + push: + branches: + - "develop" + paths: + - ".github/workflows/check-tools.yml" + - ".github/scripts/strategy-matrix/linux.json" + - "bin/check-tools.sh" + - "nix/**" + - "flake.nix" + - "flake.lock" + - "rust-toolchain.toml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + # The nix-nixos image tag is pinned alongside the build matrix in linux.json, + # so snapshots are checked against the exact image CI builds against. + linux-image-tag: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.tag.outputs.tag }} + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Read nix image tag + id: tag + run: echo "tag=$(jq -r .image_tag .github/scripts/strategy-matrix/linux.json)" >>"${GITHUB_OUTPUT}" + + # One job for all environments; they differ only in whether the tools come + # from the nix-nixos container (Linux) or `nix develop` (macOS). + check-tools: + needs: linux-image-tag + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-latest + snapshot: nix/check-tools/nix-ubuntu-amd64.txt + nix_develop: false + - runner: ubuntu-24.04-arm + snapshot: nix/check-tools/nix-ubuntu-arm64.txt + nix_develop: false + - runner: macos-26-apple-clang-21 + snapshot: nix/check-tools/macos.txt + nix_develop: true + runs-on: ${{ matrix.runner }} + # Linux runs inside the pinned nix-nixos image; macOS runs natively and uses + # the flake's dev shell instead (see the run step below). + container: ${{ !matrix.nix_develop && format('ghcr.io/xrplf/xrpld/nix-ubuntu:{0}', needs.linux-image-tag.outputs.tag) || null }} + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Prepare runner + uses: XRPLF/actions/prepare-runner@c00c22ada3bd6bcda48fcb0d62fbbab49fec8a0f + with: + enable_ccache: false + + - name: Regenerate snapshot + env: + CHECK_TOOLS_SKIP_CLONE: "1" + # check-tools.sh skips some macOS tools when CI is set; the snapshots + # capture the full `nix develop` environment, so unset it here. + CI: "" + run: | + if [ "${{ matrix.nix_develop }}" = "true" ]; then + # `nix develop` prints the dev-shell greeting first; keep only the + # check-tools.sh output (from the "Detected OS:" line onward). + nix --extra-experimental-features "nix-command flakes" develop \ + -c bash bin/check-tools.sh | sed -n '/^Detected OS:/,$p' >"${{ matrix.snapshot }}" + else + bash bin/check-tools.sh >"${{ matrix.snapshot }}" + fi + + - name: Verify snapshot is up to date + run: | + if ! git diff --exit-code -- "${{ matrix.snapshot }}"; then + echo "::error::${{ matrix.snapshot }} is out of date. Regenerate it (see nix/check-tools/README.md) and commit the result." + exit 1 + fi + + - name: Upload regenerated snapshot + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: check-tools-${{ runner.os }}-${{ runner.arch }} + path: ${{ matrix.snapshot }} diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 442a202a44..1cd97305da 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -52,7 +52,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Determine changed files # This step checks whether any files have changed that should # cause the next jobs to run. We do it this way rather than @@ -90,6 +90,7 @@ jobs: .clang-tidy .codecov.yml bin/check-tools.sh + bin/default-loader-path.sh cfg/** cmake/** conan/** diff --git a/.github/workflows/on-trigger.yml b/.github/workflows/on-trigger.yml index 49a93d2746..b8899cec72 100644 --- a/.github/workflows/on-trigger.yml +++ b/.github/workflows/on-trigger.yml @@ -28,6 +28,7 @@ on: - ".clang-tidy" - ".codecov.yml" - "bin/check-tools.sh" + - "bin/default-loader-path.sh" - "cfg/**" - "cmake/**" - "conan/**" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 1acd28208e..ac5fe46722 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -14,7 +14,7 @@ on: jobs: # Call the workflow in the XRPLF/actions repo that runs the pre-commit hooks. run-hooks: - uses: XRPLF/actions/.github/workflows/pre-commit.yml@1bde119a1ab71305ba5d3716e7a82cea1c7bdede + uses: XRPLF/actions/.github/workflows/pre-commit.yml@3ba08d6ddf114092891d48491fc2e26c3ba15552 with: runs_on: ubuntu-latest - container: '{ "image": "ghcr.io/xrplf/ci/tools-rippled-pre-commit:sha-41ec7c1" }' + container: '{ "image": "ghcr.io/xrplf/xrpld/pre-commit:sha-f56b79f" }' diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 90182e7cbb..a3e096315c 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -41,13 +41,13 @@ env: jobs: build: runs-on: ubuntu-latest - container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-2e25435 + container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-fecfc0c steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Prepare runner - uses: XRPLF/actions/prepare-runner@ad188deb3dae79dc39816e16ddfdad1e06c6fab2 + uses: XRPLF/actions/prepare-runner@c00c22ada3bd6bcda48fcb0d62fbbab49fec8a0f with: enable_ccache: false diff --git a/.github/workflows/reusable-build-test-config.yml b/.github/workflows/reusable-build-test-config.yml index d4135207fe..c1b8e4195b 100644 --- a/.github/workflows/reusable-build-test-config.yml +++ b/.github/workflows/reusable-build-test-config.yml @@ -110,10 +110,10 @@ jobs: uses: XRPLF/actions/cleanup-workspace@c7d9ce5ebb03c752a354889ecd870cadfc2b1cd4 - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Prepare runner - uses: XRPLF/actions/prepare-runner@ad188deb3dae79dc39816e16ddfdad1e06c6fab2 + uses: XRPLF/actions/prepare-runner@c00c22ada3bd6bcda48fcb0d62fbbab49fec8a0f with: enable_ccache: ${{ inputs.ccache_enabled }} @@ -223,11 +223,13 @@ jobs: BUILD_TYPE: ${{ inputs.build_type }} CMAKE_TARGET: ${{ inputs.cmake_target }} run: | + set -o pipefail cmake \ --build . \ --config "${BUILD_TYPE}" \ --parallel "${BUILD_NPROC}" \ - --target "${CMAKE_TARGET}" + --target "${CMAKE_TARGET}" \ + 2>&1 | tee "${GITHUB_WORKSPACE}/build.log" - name: Show ccache statistics if: ${{ inputs.ccache_enabled }} @@ -322,7 +324,7 @@ jobs: PRELOAD="" fi - LD_PRELOAD="$PRELOAD" ./xrpld --unittest --unittest-jobs "${BUILD_NPROC}" 2>&1 | tee unittest.log + LD_PRELOAD="$PRELOAD" ./xrpld --unittest --unittest-jobs "${BUILD_NPROC}" 2>&1 | tee "${GITHUB_WORKSPACE}/unittest.log" # Smoke-run every benchmark module with a single repetition to confirm the # benchmarks still build and execute. This is a correctness check, not a @@ -341,25 +343,27 @@ jobs: done < <(find src/benchmarks -type f -perm -u+x -name 'xrpl.bench.*') exit "${rc}" - - name: Show test failure summary - if: ${{ failure() && !inputs.build_only }} - env: - WORKING_DIR: ${{ runner.os == 'Windows' && format('{0}\{1}', env.BUILD_DIR, inputs.build_type) || env.BUILD_DIR }} + - name: Show build/test failure summary + if: ${{ failure() }} run: | - if [ ! -d "${WORKING_DIR}" ]; then - echo "Working directory '${WORKING_DIR}' does not exist." - exit 0 - fi + cd "${GITHUB_WORKSPACE}" - cd "${WORKING_DIR}" - - if [ ! -f unittest.log ]; then - echo "unittest.log not found; embedded tests may not have run." - exit 0 - fi - - if ! grep -E "failed" unittest.log; then - echo "Log present but no failure lines found in unittest.log." + if [ -f unittest.log ]; then + if ! grep -E "failed" unittest.log | grep -vE "^I[0-9]|^[0-9]+> (ERR:|FTL:)"; then + echo "unittest.log present but no failure lines found." + fi + elif [ -f build.log ]; then + # GCC/Clang emit "error:" (covers "fatal error:"); MSVC emits + # "error C####:", "error LNK####:", and "fatal error LNK####:". + # -A6 prints the lines that follow each match (source line, caret, + # notes, and the "N errors generated" tally) to capture the whole + # diagnostic block. + if ! grep -E -A6 "error:|error C[0-9]{4}|error LNK[0-9]{4}|fatal error" build.log; then + echo "build.log present but no compile errors found." + fi + else + echo "unittest.log/build.log not found; something went wrong." + exit 1 fi - name: Debug failure (Linux) if: ${{ failure() && runner.os == 'Linux' && !inputs.build_only }} diff --git a/.github/workflows/reusable-check-levelization.yml b/.github/workflows/reusable-check-levelization.yml index 88c95ac3ba..7f547f2ab6 100644 --- a/.github/workflows/reusable-check-levelization.yml +++ b/.github/workflows/reusable-check-levelization.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Check levelization run: python .github/scripts/levelization/generate.py - name: Check for differences diff --git a/.github/workflows/reusable-check-rename.yml b/.github/workflows/reusable-check-rename.yml index 9a91e98ee3..874c8adcde 100644 --- a/.github/workflows/reusable-check-rename.yml +++ b/.github/workflows/reusable-check-rename.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Check definitions run: .github/scripts/rename/definitions.sh . - name: Check copyright notices diff --git a/.github/workflows/reusable-clang-tidy.yml b/.github/workflows/reusable-clang-tidy.yml index 4a10e4b0f7..a6c1e6ae56 100644 --- a/.github/workflows/reusable-clang-tidy.yml +++ b/.github/workflows/reusable-clang-tidy.yml @@ -34,16 +34,16 @@ jobs: needs: [determine-files] if: ${{ needs.determine-files.outputs.cpp_changed_files != '' || needs.determine-files.outputs.need_full_run == 'true' }} runs-on: ["self-hosted", "Linux", "X64", "heavy"] - container: "ghcr.io/xrplf/xrpld/nix-debian:sha-2e25435" + container: "ghcr.io/xrplf/xrpld/nix-debian:sha-fecfc0c" permissions: contents: read issues: write steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Prepare runner - uses: XRPLF/actions/prepare-runner@ad188deb3dae79dc39816e16ddfdad1e06c6fab2 + uses: XRPLF/actions/prepare-runner@c00c22ada3bd6bcda48fcb0d62fbbab49fec8a0f with: enable_ccache: false diff --git a/.github/workflows/reusable-package.yml b/.github/workflows/reusable-package.yml index 6feecbfb75..e1c11ac677 100644 --- a/.github/workflows/reusable-package.yml +++ b/.github/workflows/reusable-package.yml @@ -27,10 +27,10 @@ jobs: matrix: ${{ steps.generate.outputs.matrix }} steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.13" @@ -54,7 +54,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download pre-built binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/workflows/reusable-strategy-matrix.yml b/.github/workflows/reusable-strategy-matrix.yml index b6091b99d9..12f11b0fbe 100644 --- a/.github/workflows/reusable-strategy-matrix.yml +++ b/.github/workflows/reusable-strategy-matrix.yml @@ -23,10 +23,10 @@ jobs: matrix: ${{ steps.generate.outputs.matrix }} steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.13" diff --git a/.github/workflows/reusable-upload-recipe.yml b/.github/workflows/reusable-upload-recipe.yml index 07077163ed..b4ab638dee 100644 --- a/.github/workflows/reusable-upload-recipe.yml +++ b/.github/workflows/reusable-upload-recipe.yml @@ -40,14 +40,14 @@ defaults: jobs: upload: runs-on: ubuntu-latest - container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-2e25435 + container: ghcr.io/xrplf/xrpld/nix-ubuntu:sha-fecfc0c env: REMOTE_NAME: ${{ inputs.remote_name }} CONAN_LOGIN_USERNAME_XRPLF: ${{ secrets.remote_username }} CONAN_PASSWORD_XRPLF: ${{ secrets.remote_password }} steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Generate build version number id: version diff --git a/.github/workflows/upload-conan-deps.yml b/.github/workflows/upload-conan-deps.yml index abc0867b15..eb58650bdf 100644 --- a/.github/workflows/upload-conan-deps.yml +++ b/.github/workflows/upload-conan-deps.yml @@ -65,10 +65,10 @@ jobs: uses: XRPLF/actions/cleanup-workspace@c7d9ce5ebb03c752a354889ecd870cadfc2b1cd4 - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Prepare runner - uses: XRPLF/actions/prepare-runner@ad188deb3dae79dc39816e16ddfdad1e06c6fab2 + uses: XRPLF/actions/prepare-runner@c00c22ada3bd6bcda48fcb0d62fbbab49fec8a0f with: enable_ccache: false diff --git a/.gitignore b/.gitignore index de59dd3712..0b4e68af74 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,9 @@ DerivedData # Python __pycache__ +# Rust build artifacts. +target/ + # Direnv's directory /.direnv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a689c0f8b..d339cb29ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,7 +55,7 @@ repos: types_or: [c++, c] - repo: https://github.com/pre-commit/mirrors-clang-format - rev: dd18dad857d6133e90bbe478f4f2f22ec0030269 # frozen: v22.1.5 + rev: f4d7745e17a28aad7eed2f4874ca8d1568c11c4c # frozen: v22.1.8 hooks: - id: clang-format args: [--style=file] @@ -68,7 +68,7 @@ repos: - id: gersemi - repo: https://github.com/rbubley/mirrors-prettier - rev: 39e2973981e6d2f9b6c543b0086a2d2393abdc89 # frozen: v3.9.4 + rev: 9337a74165b178ae2c766f60bee7252a0f06f3e8 # frozen: v3.9.5 hooks: - id: prettier args: [--end-of-line=auto] diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000000..f92cb81924 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,4 @@ +# By default, anyone can review changes. + +# The CI tooling team should review changes to the CI configuration. +/.github/ @XRPLF/ci-tooling diff --git a/bin/check-tools.sh b/bin/check-tools.sh index 7886bcf8b0..e230302742 100755 --- a/bin/check-tools.sh +++ b/bin/check-tools.sh @@ -30,8 +30,10 @@ missing=() checked=0 # check [probe-command...] -# Runs the probe (default: " --version") quietly. Records as -# missing if the command is not found or exits non-zero. +# Runs the probe (default: " --version"), capturing both stdout and +# stderr, and prints one aligned line: the status, the name, and the first +# non-blank line of the probe output (its version). Records as missing +# if the command is not found or exits non-zero. check() { local name="$1" shift @@ -40,10 +42,11 @@ check() { probe=("${name}" --version) fi - echo "Checking ${name}..." checked=$((checked + 1)) - if "${probe[@]}" | head -n 1; then - printf ' [ ok ] %s\n' "${name}" + local output version + if output="$("${probe[@]}" 2>&1)"; then + version="$(printf '%s\n' "${output}" | grep -m1 '[^[:space:]]' || true)" + printf ' [ ok ] %-20s %s\n' "${name}" "${version}" else printf ' [MISS] %s\n' "${name}" missing+=("${name}") @@ -85,12 +88,14 @@ if [ "${os}" = "linux" ] || [ "${os}" = "macos" ]; then check file check less check make - check netstat which netstat + # net-tools netstat reports "net-tools X.Y"; macOS ships BSD netstat with no + # version flag, so fall back to a presence marker there. + check netstat sh -c 'command -v netstat >/dev/null && { netstat --version 2>&1 | grep -m1 -oE "net-tools [0-9.]+" || echo present; }' check ninja - check perl + check perl perl -e 'print "$^V\n"' check pkg-config check vim - check zip + check zip bash -c 'zip --version 2>&1 | grep -m1 -oE "Zip [0-9.]+"' # These tools are present in our Linux CI images and in local development # setups, but not in the macOS CI environment. So check them everywhere diff --git a/nix/docker/loader-path.sh b/bin/default-loader-path.sh similarity index 100% rename from nix/docker/loader-path.sh rename to bin/default-loader-path.sh diff --git a/bin/pre-commit/Dockerfile b/bin/pre-commit/Dockerfile new file mode 100644 index 0000000000..a96f3e4c25 --- /dev/null +++ b/bin/pre-commit/Dockerfile @@ -0,0 +1,55 @@ +ARG BASE_IMAGE=ubuntu:26.04 + +FROM ${BASE_IMAGE} + +SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] +ENTRYPOINT ["/bin/bash"] + +ARG DEBIAN_FRONTEND=noninteractive + +RUN < Clang errors under our -Werror. At link time the flag IS consumed # (it selects the C++ runtime), so we move it there instead of dropping it entirely. - get_filename_component(_cxx_real "${CMAKE_CXX_COMPILER}" REALPATH) if( - _cxx_real MATCHES "^/nix/store/" + is_nix_compiler AND is_linux AND is_clang AND CMAKE_CXX_FLAGS MATCHES "stdlib=libstdc" diff --git a/cmake/XrplCore.cmake b/cmake/XrplCore.cmake index b061a0b82a..9db2e3a06e 100644 --- a/cmake/XrplCore.cmake +++ b/cmake/XrplCore.cmake @@ -133,6 +133,12 @@ target_link_libraries( add_module(xrpl resource) target_link_libraries(xrpl.libxrpl.resource PUBLIC xrpl.libxrpl.protocol) +add_module(xrpl peerfinder) +target_link_libraries( + xrpl.libxrpl.peerfinder + PUBLIC xrpl.libxrpl.basics xrpl.libxrpl.protocol +) + # Level 08 add_module(xrpl net) target_link_libraries( @@ -201,6 +207,16 @@ target_link_libraries( add_module(xrpl tx) target_link_libraries(xrpl.libxrpl.tx PUBLIC xrpl.libxrpl.ledger) +add_module(xrpl consensus) +target_link_libraries( + xrpl.libxrpl.consensus + PUBLIC + xrpl.libxrpl.basics + xrpl.libxrpl.json + xrpl.libxrpl.protocol + xrpl.libxrpl.ledger +) + # Telemetry module — OpenTelemetry distributed tracing support. # Sources: include/xrpl/telemetry/ (headers), src/libxrpl/telemetry/ (impl). # When telemetry=ON, links the Conan-provided umbrella target @@ -237,6 +253,7 @@ target_link_modules( beast conditions config + consensus core crypto git @@ -244,6 +261,7 @@ target_link_modules( ledger net nodestore + peerfinder protocol protocol_autogen rdb diff --git a/cmake/XrplSanity.cmake b/cmake/XrplSanity.cmake index a35645ad5c..ba9f7988bc 100644 --- a/cmake/XrplSanity.cmake +++ b/cmake/XrplSanity.cmake @@ -36,6 +36,19 @@ elseif(is_gcc) endif() endif() +# A Nix compiler is only meant to be used from a managed environment: the xrpld +# dev shell (which exports XRPL_DEVSHELL) or the CI image. Using one from a bare +# shell usually means a leaked toolchain (picked up via PATH or a Conan profile) +# and leads to confusing breakage, so fail early with guidance. +if(is_nix_compiler AND NOT is_ci_image AND NOT DEFINED ENV{XRPL_DEVSHELL}) + message( + FATAL_ERROR + "A Nix compiler (${CMAKE_CXX_COMPILER}) is being used outside the xrpld " + "dev shell. Enter it with `nix develop` (see docs/build/nix.md) before " + "configuring the build." + ) +endif() + # check for in-source build and fail if("${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") message( diff --git a/conan/profiles/default b/conan/profiles/default index cc937ce511..40e95dc42d 100644 --- a/conan/profiles/default +++ b/conan/profiles/default @@ -20,6 +20,22 @@ compiler.libcxx={{ detect_api.detect_libcxx(compiler, version, compiler_exe) }} {% endif %} [conf] +{# The Boost recipe builds with b2, which doesn't use Conan's toolchain files. #} +{# Instead it hand-rolls the compiler for user-config.jam, #} +{# and its fallback probes a version-suffixed binary (e.g. `g++-15`) before plain `g++`. #} +{# Inside the Nix shell the wrapper only provides `g++`/`gcc` (no `-15` suffix), #} +{# so on a host that also has a system `g++-15` the probe escapes Nix #} +{# and picks the system compiler, which is mismatched with the Nix libraries #} +{# and breaks the build (e.g. Boost.Stacktrace link checks fail). #} +{# Pinning the executables here short-circuits that probe so Boost (and the rest of the toolchain) #} +{# resolve the same compiler. #} +{# Not part of the package ID, so binaries stay shareable. #} +{% if os != "Windows" %} +{% set cc_exe = {"gcc": "gcc", "clang": "clang", "apple-clang": "clang"}.get(compiler) %} +{% set cxx_exe = {"gcc": "g++", "clang": "clang++", "apple-clang": "clang++"}.get(compiler) %} +tools.build:compiler_executables={'c':'{{ cc_exe }}','cpp':'{{ cxx_exe }}'} +{% endif %} + {# By default, Conan tries to reuse binaries built with different cppstd versions. #} {# We want to avoid that to improve reproduceability, so we add the cppstd version to the package ID. #} {# More info: https://docs.conan.io/2/reference/extensions/binary_compatibility.html #} diff --git a/docs/build/nix.md b/docs/build/nix.md index d6e53a254a..d0001294e3 100644 --- a/docs/build/nix.md +++ b/docs/build/nix.md @@ -38,8 +38,10 @@ The first time you run this command, it will take a few minutes to download and ### Platform notes -- **Linux**: `nix develop` gives you a shell with all the tooling necessary to - develop xrpld and with GCC 15.2 (also provided by Nix). There are no caveats. +- **Linux**: `nix develop` gives you a shell with all the tooling necessary to develop xrpld + and with the same GCC/glibc toolchain that Nix builds for CI. + See [Choosing a different compiler](#choosing-a-different-compiler) + for the custom-vs-plain toolchain trade-off. - **macOS**: `nix develop` gives you a full environment too, with Clang (and every other tool, including Conan) provided by Nix. To use your system-wide Apple Clang instead, enter `nix develop .#apple-clang`. Conan has no binary in @@ -63,8 +65,16 @@ The first time you run this command, it will take a few minutes to download and ### Choosing a different compiler A compiler can be chosen by providing its name with the `.#` prefix, e.g. `nix develop .#clang`. -The `.#gcc` and `.#clang` shells provide the same GCC and Clang versions used in CI -(pinned in [`nix/packages.nix`](../../nix/packages.nix)). + +On Linux, `.#gcc` and `.#clang` provide the exact toolchain CI uses: +the compiler (pinned in [`nix/packages.nix`](../../nix/packages.nix)) +rebuilt against the pinned custom glibc (see [`nix/compilers.nix`](../../nix/compilers.nix)). +Building that toolchain the first time is slow unless it is fetched from a Nix binary cache. +If you don't need the custom glibc, the Linux-only `.#gcc-plain` and `.#clang-plain` +give you the stock nixpkgs compilers of the same versions. +On macOS there is no custom glibc, so `.#gcc` and `.#clang` are already the plain nixpkgs toolchain, +and the `-plain` variants do not exist. + Use `nix flake show` to see all the available development shells. Use `nix develop .#no-compiler` to use the compiler from your system. @@ -72,14 +82,18 @@ Use `nix develop .#no-compiler` to use the compiler from your system. ### Example Usage ```bash -# Use GCC (same version as CI) +# Use GCC — same toolchain as CI (custom glibc on Linux) nix develop .#gcc -# Use Clang (same version as CI) +# Use Clang — same toolchain as CI (custom glibc on Linux) nix develop .#clang # Use default for your platform nix develop + +# Stock nixpkgs GCC/Clang, Linux only — skips the custom-glibc build, but does not match CI +nix develop .#gcc-plain +nix develop .#clang-plain ``` ### Using a different shell @@ -110,6 +124,10 @@ nix develop -c "$SHELL" Once inside the Nix development shell, follow the standard [build instructions](../../BUILD.md#steps). The Nix shell provides all necessary tools (CMake, Ninja, Conan, etc.). +Coverage builds (`-Dcoverage=ON`) work in the `gcc` shell (and `gcc-plain` on Linux): +each ships a `gcov` matching its compiler, since Nix's cc-wrapper does not expose one. +The `clang` shells do not include `llvm-cov`, so use a `gcc` shell for coverage. + ## Automatic Activation with direnv [direnv](https://direnv.net/) or [nix-direnv](https://github.com/nix-community/nix-direnv) can automatically activate the Nix development shell when you enter the repository directory. @@ -136,6 +154,14 @@ conan install .. --output-folder . --build '*' --settings build_type=Release To update `flake.lock` to the latest revision use `nix flake update` command. +## Tooling snapshots + +The tool versions in each Nix environment are recorded in +[`nix/check-tools/`](../../nix/check-tools) and verified by CI. If you change the +environment (bump the CI image tag, update `flake.lock`, or edit the tool list in +`bin/check-tools.sh`), CI fails until you regenerate and commit the affected +snapshot — see [`nix/check-tools/README.md`](../../nix/check-tools/README.md). + ## Troubleshooting See [Troubleshooting Nix problems](./nix_troubleshooting.md) for common issues, diff --git a/flake.lock b/flake.lock index 80243ccf15..cd9289c998 100644 --- a/flake.lock +++ b/flake.lock @@ -36,7 +36,28 @@ "root": { "inputs": { "nixpkgs": "nixpkgs", - "nixpkgs-custom-glibc": "nixpkgs-custom-glibc" + "nixpkgs-custom-glibc": "nixpkgs-custom-glibc", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1784611586, + "narHash": "sha256-OfqgY+0hp/zseZB7uyH0U8kIDPS4scZZCyAurEplvG0=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "14f58845249f3552a89b07772626b8d3c632fa86", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index c52f4d050e..ee2fd13efc 100644 --- a/flake.nix +++ b/flake.nix @@ -10,12 +10,25 @@ url = "github:NixOS/nixpkgs/9cd98386a38891d1074fc18036b842dc4416f562"; flake = false; }; + # Pinned Rust toolchains, delivered from the Nix store. Lets the Nix CI + # image and dev shell honour the single `rust-toolchain.toml` pin (shared + # with the rustup-based non-Nix runners) while staying hermetic — the + # toolchain lands in the image's Nix closure and is locked by flake.lock. + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = - { nixpkgs, nixpkgs-custom-glibc, ... }: + { + nixpkgs, + nixpkgs-custom-glibc, + rust-overlay, + ... + }: let - forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc; }; + forEachSystem = import ./nix/utils.nix { inherit nixpkgs nixpkgs-custom-glibc rust-overlay; }; in { devShells = forEachSystem (import ./nix/devshell.nix); diff --git a/include/xrpl/beast/container/detail/aged_unordered_container.h b/include/xrpl/beast/container/detail/aged_unordered_container.h index db10e8cc23..c4287b1ca1 100644 --- a/include/xrpl/beast/container/detail/aged_unordered_container.h +++ b/include/xrpl/beast/container/detail/aged_unordered_container.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include diff --git a/src/xrpld/app/consensus/RCLCensorshipDetector.h b/include/xrpl/consensus/CensorshipDetector.h similarity index 98% rename from src/xrpld/app/consensus/RCLCensorshipDetector.h rename to include/xrpl/consensus/CensorshipDetector.h index 6d0e20031e..3d2708f68f 100644 --- a/src/xrpld/app/consensus/RCLCensorshipDetector.h +++ b/include/xrpl/consensus/CensorshipDetector.h @@ -10,7 +10,7 @@ namespace xrpl { template -class RCLCensorshipDetector +class CensorshipDetector { public: struct TxIDSeq @@ -49,7 +49,7 @@ private: TxIDSeqVec tracker_; public: - RCLCensorshipDetector() = default; + CensorshipDetector() = default; /** * Add transactions being proposed for the current consensus round. diff --git a/src/xrpld/consensus/Consensus.h b/include/xrpl/consensus/Consensus.h similarity index 99% rename from src/xrpld/consensus/Consensus.h rename to include/xrpl/consensus/Consensus.h index 440191939b..f9d5f7ef02 100644 --- a/src/xrpld/consensus/Consensus.h +++ b/include/xrpl/consensus/Consensus.h @@ -1,15 +1,14 @@ #pragma once -#include -#include -#include - #include #include #include #include #include #include +#include +#include +#include #include #include #include diff --git a/src/xrpld/consensus/ConsensusParms.h b/include/xrpl/consensus/ConsensusParms.h similarity index 100% rename from src/xrpld/consensus/ConsensusParms.h rename to include/xrpl/consensus/ConsensusParms.h diff --git a/src/xrpld/consensus/ConsensusProposal.h b/include/xrpl/consensus/ConsensusProposal.h similarity index 100% rename from src/xrpld/consensus/ConsensusProposal.h rename to include/xrpl/consensus/ConsensusProposal.h diff --git a/src/xrpld/consensus/ConsensusTypes.h b/include/xrpl/consensus/ConsensusTypes.h similarity index 98% rename from src/xrpld/consensus/ConsensusTypes.h rename to include/xrpl/consensus/ConsensusTypes.h index 4553e46f48..4dac2d9912 100644 --- a/src/xrpld/consensus/ConsensusTypes.h +++ b/include/xrpl/consensus/ConsensusTypes.h @@ -1,11 +1,10 @@ #pragma once -#include -#include - #include #include #include +#include +#include #include #include diff --git a/src/xrpld/consensus/DisputedTx.h b/include/xrpl/consensus/DisputedTx.h similarity index 99% rename from src/xrpld/consensus/DisputedTx.h rename to include/xrpl/consensus/DisputedTx.h index 12ed00d460..c194716d43 100644 --- a/src/xrpld/consensus/DisputedTx.h +++ b/include/xrpl/consensus/DisputedTx.h @@ -1,9 +1,8 @@ #pragma once -#include - #include #include +#include #include #include diff --git a/src/xrpld/consensus/LedgerTrie.h b/include/xrpl/consensus/LedgerTrie.h similarity index 100% rename from src/xrpld/consensus/LedgerTrie.h rename to include/xrpl/consensus/LedgerTrie.h diff --git a/src/xrpld/consensus/README.md b/include/xrpl/consensus/README.md similarity index 100% rename from src/xrpld/consensus/README.md rename to include/xrpl/consensus/README.md diff --git a/src/xrpld/consensus/Validations.h b/include/xrpl/consensus/Validations.h similarity index 99% rename from src/xrpld/consensus/Validations.h rename to include/xrpl/consensus/Validations.h index 2696804c86..ebb13c5e7c 100644 --- a/src/xrpld/consensus/Validations.h +++ b/include/xrpl/consensus/Validations.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include #include @@ -11,6 +9,7 @@ #include #include #include +#include #include #include diff --git a/include/xrpl/core/ServiceRegistry.h b/include/xrpl/core/ServiceRegistry.h index 93ac8e1847..414dff896d 100644 --- a/include/xrpl/core/ServiceRegistry.h +++ b/include/xrpl/core/ServiceRegistry.h @@ -15,9 +15,9 @@ namespace xrpl { // Forward declarations -namespace NodeStore { +namespace node_store { class Database; -} // namespace NodeStore +} // namespace node_store namespace Resource { class Manager; } // namespace Resource @@ -167,7 +167,7 @@ public: getResourceManager() = 0; // Storage services - virtual NodeStore::Database& + virtual node_store::Database& getNodeStore() = 0; virtual SHAMapStore& diff --git a/include/xrpl/nodestore/Backend.h b/include/xrpl/nodestore/Backend.h index 564a874c5e..85d076bcfb 100644 --- a/include/xrpl/nodestore/Backend.h +++ b/include/xrpl/nodestore/Backend.h @@ -13,7 +13,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { /** * A backend used for the NodeStore. @@ -163,4 +163,4 @@ public: fdRequired() const = 0; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/Database.h b/include/xrpl/nodestore/Database.h index 96ba91bd76..902cfc9e03 100644 --- a/include/xrpl/nodestore/Database.h +++ b/include/xrpl/nodestore/Database.h @@ -25,7 +25,7 @@ namespace xrpl { class Section; } // namespace xrpl -namespace xrpl::NodeStore { +namespace xrpl::node_store { /** * Persistency layer for NodeObject @@ -248,7 +248,7 @@ protected: void storeStats(std::uint64_t count, std::uint64_t sz) { - XRPL_ASSERT(count <= sz, "xrpl::NodeStore::Database::storeStats : valid inputs"); + XRPL_ASSERT(count <= sz, "xrpl::node_store::Database::storeStats : valid inputs"); storeCount_ += count; storeSz_ += sz; } @@ -308,4 +308,4 @@ private: threadEntry(); }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/DatabaseRotating.h b/include/xrpl/nodestore/DatabaseRotating.h index 5381b5c435..21b8b422c7 100644 --- a/include/xrpl/nodestore/DatabaseRotating.h +++ b/include/xrpl/nodestore/DatabaseRotating.h @@ -9,7 +9,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { /* This class has two key-value store Backend objects for persisting SHAMap * records. This facilitates online deletion of data. New backends are @@ -38,7 +38,7 @@ public: */ virtual void rotate( - std::unique_ptr&& newBackend, + std::unique_ptr&& newBackend, std::function const& f) = 0; @@ -56,4 +56,4 @@ public: setRotationInFlight(bool inFlight) = 0; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/DummyScheduler.h b/include/xrpl/nodestore/DummyScheduler.h index 49b0d37462..fc7a040b5a 100644 --- a/include/xrpl/nodestore/DummyScheduler.h +++ b/include/xrpl/nodestore/DummyScheduler.h @@ -3,7 +3,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { /** * Simple NodeStore Scheduler that just performs the tasks synchronously. @@ -21,4 +21,4 @@ public: onBatchWrite(BatchWriteReport const& report) override; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/Factory.h b/include/xrpl/nodestore/Factory.h index a18023a8a8..568e9b0712 100644 --- a/include/xrpl/nodestore/Factory.h +++ b/include/xrpl/nodestore/Factory.h @@ -14,7 +14,7 @@ namespace xrpl { class Section; } // namespace xrpl -namespace xrpl::NodeStore { +namespace xrpl::node_store { /** * Base class for backend factories. @@ -70,4 +70,4 @@ public: } }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/Manager.h b/include/xrpl/nodestore/Manager.h index 54d99fe94b..1b869e4bca 100644 --- a/include/xrpl/nodestore/Manager.h +++ b/include/xrpl/nodestore/Manager.h @@ -10,7 +10,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { /** * Singleton for managing NodeStore factories and back ends. @@ -98,4 +98,4 @@ public: beast::Journal journal) = 0; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/NodeObject.h b/include/xrpl/nodestore/NodeObject.h index b96d65fa12..db139b9aa8 100644 --- a/include/xrpl/nodestore/NodeObject.h +++ b/include/xrpl/nodestore/NodeObject.h @@ -8,7 +8,7 @@ #include #include -// VFALCO NOTE Intentionally not in the NodeStore namespace +// VFALCO NOTE Intentionally not in the node_store namespace namespace xrpl { diff --git a/include/xrpl/nodestore/Scheduler.h b/include/xrpl/nodestore/Scheduler.h index 5d93a80eaa..40c36ce8ab 100644 --- a/include/xrpl/nodestore/Scheduler.h +++ b/include/xrpl/nodestore/Scheduler.h @@ -4,7 +4,7 @@ #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { enum class FetchType { Synchronous, Async }; @@ -71,4 +71,4 @@ public: onBatchWrite(BatchWriteReport const& report) = 0; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/Task.h b/include/xrpl/nodestore/Task.h index 59fe648476..22cec62eae 100644 --- a/include/xrpl/nodestore/Task.h +++ b/include/xrpl/nodestore/Task.h @@ -1,6 +1,6 @@ #pragma once -namespace xrpl::NodeStore { +namespace xrpl::node_store { /** * Derived classes perform scheduled tasks. @@ -17,4 +17,4 @@ struct Task performScheduledTask() = 0; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/Types.h b/include/xrpl/nodestore/Types.h index 872d948a36..21af6fa68b 100644 --- a/include/xrpl/nodestore/Types.h +++ b/include/xrpl/nodestore/Types.h @@ -5,7 +5,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { // This is only used to pre-allocate the array for // batch objects and does not affect the amount written. @@ -36,4 +36,4 @@ enum class Status { */ using Batch = std::vector>; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/detail/BatchWriter.h b/include/xrpl/nodestore/detail/BatchWriter.h index b89df0da14..6ac3428752 100644 --- a/include/xrpl/nodestore/detail/BatchWriter.h +++ b/include/xrpl/nodestore/detail/BatchWriter.h @@ -9,7 +9,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { /** * Batch-writing assist logic. @@ -86,4 +86,4 @@ private: Batch writeSet_; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/detail/DatabaseNodeImp.h b/include/xrpl/nodestore/detail/DatabaseNodeImp.h index 6f2fca682f..33a2e27939 100644 --- a/include/xrpl/nodestore/detail/DatabaseNodeImp.h +++ b/include/xrpl/nodestore/detail/DatabaseNodeImp.h @@ -22,7 +22,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { class DatabaseNodeImp : public Database { @@ -68,7 +68,7 @@ public: XRPL_ASSERT( backend_, - "xrpl::NodeStore::DatabaseNodeImp::DatabaseNodeImp : non-null " + "xrpl::node_store::DatabaseNodeImp::DatabaseNodeImp : non-null " "backend"); } @@ -138,4 +138,4 @@ private: } }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/detail/DatabaseRotatingImp.h b/include/xrpl/nodestore/detail/DatabaseRotatingImp.h index ecbe9a513d..9b566195ef 100644 --- a/include/xrpl/nodestore/detail/DatabaseRotatingImp.h +++ b/include/xrpl/nodestore/detail/DatabaseRotatingImp.h @@ -16,7 +16,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { class DatabaseRotatingImp : public DatabaseRotating { @@ -41,7 +41,7 @@ public: void rotate( - std::unique_ptr&& newBackend, + std::unique_ptr&& newBackend, std::function const& f) override; @@ -94,4 +94,4 @@ private: forEach(std::function)> f) override; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/detail/DecodedBlob.h b/include/xrpl/nodestore/detail/DecodedBlob.h index d0cc5e3404..bd90ff2f1b 100644 --- a/include/xrpl/nodestore/detail/DecodedBlob.h +++ b/include/xrpl/nodestore/detail/DecodedBlob.h @@ -4,7 +4,7 @@ #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { /** * Parsed key/value blob into NodeObject components. @@ -49,4 +49,4 @@ private: int dataBytes_; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/detail/EncodedBlob.h b/include/xrpl/nodestore/detail/EncodedBlob.h index d668cdccd8..d171df3cc9 100644 --- a/include/xrpl/nodestore/detail/EncodedBlob.h +++ b/include/xrpl/nodestore/detail/EncodedBlob.h @@ -12,7 +12,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { /** * Convert a NodeObject from in-memory to database format. @@ -68,7 +68,7 @@ class EncodedBlob public: explicit EncodedBlob(std::shared_ptr const& obj) : size_([&obj]() { - XRPL_ASSERT(obj, "xrpl::NodeStore::EncodedBlob::EncodedBlob : non-null input"); + XRPL_ASSERT(obj, "xrpl::node_store::EncodedBlob::EncodedBlob : non-null input"); if (!obj) throw std::runtime_error("EncodedBlob: unseated std::shared_ptr used."); @@ -88,7 +88,7 @@ public: XRPL_ASSERT( ((ptr_ == payload_.data()) && (size_ <= payload_.size())) || ((ptr_ != payload_.data()) && (size_ > payload_.size())), - "xrpl::NodeStore::EncodedBlob::~EncodedBlob : valid payload " + "xrpl::node_store::EncodedBlob::~EncodedBlob : valid payload " "pointer"); if (ptr_ != payload_.data()) @@ -114,4 +114,4 @@ public: } }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/detail/ManagerImp.h b/include/xrpl/nodestore/detail/ManagerImp.h index fc84b0aa57..f1653b45dc 100644 --- a/include/xrpl/nodestore/detail/ManagerImp.h +++ b/include/xrpl/nodestore/detail/ManagerImp.h @@ -13,7 +13,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { class ManagerImp : public Manager { @@ -57,4 +57,4 @@ public: beast::Journal journal) override; }; -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/detail/codec.h b/include/xrpl/nodestore/detail/codec.h index 2f69d532be..a3dfa7c944 100644 --- a/include/xrpl/nodestore/detail/codec.h +++ b/include/xrpl/nodestore/detail/codec.h @@ -21,7 +21,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { template std::pair @@ -269,7 +269,7 @@ nodeobjectCompress(void const* in, std::size_t inSize, BufferFactory&& bf) case 1: // lz4 { std::uint8_t* p = nullptr; - auto const lzr = NodeStore::lz4Compress(in, inSize, [&p, &vn, &bf](std::size_t n) { + auto const lzr = node_store::lz4Compress(in, inSize, [&p, &vn, &bf](std::size_t n) { p = reinterpret_cast(bf(vn + n)); return p + vn; }); @@ -316,4 +316,4 @@ filterInner(void* in, std::size_t inSize) } } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/nodestore/detail/varint.h b/include/xrpl/nodestore/detail/varint.h index 5a65545d3a..afbf71cdea 100644 --- a/include/xrpl/nodestore/detail/varint.h +++ b/include/xrpl/nodestore/detail/varint.h @@ -6,7 +6,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { // This is a variant of the base128 varint format from // google protocol buffers: @@ -123,4 +123,4 @@ write(nudb::detail::ostream& os, std::size_t t) writeVarint(os.data(sizeVarint(t)), t); } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/include/xrpl/peerfinder/Config.h b/include/xrpl/peerfinder/Config.h new file mode 100644 index 0000000000..9ff0d342c3 --- /dev/null +++ b/include/xrpl/peerfinder/Config.h @@ -0,0 +1,163 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace xrpl::PeerFinder { + +struct PeerLimitConfig +{ + std::optional maxPeers; + std::optional inPeers; + std::optional outPeers; +}; + +/** + * PeerFinder configuration settings. + */ +struct Config +{ + /** + * The largest number of public peer slots to allow. + * This includes both inbound and outbound, but does not include + * fixed peers. + */ + std::size_t maxPeers{Tuning::kDefaultMaxPeers}; + + /** + * The number of automatic outbound connections to maintain. + * Outbound connections are only maintained if autoConnect + * is `true`. + */ + std::size_t outPeers = calcOutPeers(); // Note: relies on `maxPeers` being initialized + + /** + * The number of automatic inbound connections to maintain. + * Inbound connections are only maintained if wantIncoming + * is `true`. + */ + std::size_t inPeers{0}; + + /** + * `true` if we want our IP address kept private. + */ + bool peerPrivate = true; + + /** + * `true` if we want to accept incoming connections. + */ + bool wantIncoming{true}; + + /** + * `true` if we want to establish connections automatically + */ + bool autoConnect{true}; + + /** + * The listening port number. + */ + std::uint16_t listeningPort{0}; + + /** + * The set of features we advertise. + */ + std::string features; + + /** + * Limit how many incoming connections we allow per IP + */ + int ipLimit{0}; + + /** + * `true` if we want to verify endpoints in TMEndpoints messages + */ + bool verifyEndpoints = true; + + //-------------------------------------------------------------------------- + + /** + * Returns a suitable value for outPeers according to the rules. + */ + [[nodiscard]] std::size_t + calcOutPeers() const; + + /** + * Adjusts the values so they follow the business rules. + */ + void + applyTuning(); + + /** + * Write the configuration into a property stream + */ + void + onWrite(beast::PropertyStream::Map& map) const; + + /** + * Make PeerFinder::Config from peer limit and server mode parameters. + */ + static Config + makeConfig( + bool peerPrivate, + bool standalone, + PeerLimitConfig const& limits, + std::uint16_t port, + bool validationPublicKey, + int ipLimit, + bool verifyEndpoints); + + /** + * Compares two configurations for equality field by field. + */ + friend bool + operator==(Config const& lhs, Config const& rhs) = default; +}; + +//------------------------------------------------------------------------------ + +/** + * Possible results from activating a slot. + */ +enum class Result { InboundDisabled, DuplicatePeer, IpLimitExceeded, Full, Success }; + +/** + * @brief Converts a `Result` enum value to its string representation. + * + * This function provides a human-readable string for a given `Result` enum, + * which is useful for logging, debugging, or displaying status messages. + * + * @param result The `Result` enum value to convert. + * @return A `std::string_view` representing the enum value. Returns "unknown" + * if the enum value is not explicitly handled. + * + * @note This function returns a `std::string_view` for performance. + * A `std::string` would need to allocate memory on the heap and copy the + * string literal into it every time the function is called. + */ +inline std::string_view +to_string(Result result) noexcept +{ + switch (result) + { + case Result::InboundDisabled: + return "inbound disabled"; + case Result::DuplicatePeer: + return "peer already connected"; + case Result::IpLimitExceeded: + return "ip limit exceeded"; + case Result::Full: + return "slots full"; + case Result::Success: + return "success"; + } + + return "unknown"; +} + +} // namespace xrpl::PeerFinder diff --git a/include/xrpl/peerfinder/PeerfinderManager.h b/include/xrpl/peerfinder/PeerfinderManager.h new file mode 100644 index 0000000000..ed683520c1 --- /dev/null +++ b/include/xrpl/peerfinder/PeerfinderManager.h @@ -0,0 +1,179 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace xrpl::PeerFinder { + +/** + * Maintains a set of IP addresses used for getting into the network. + */ +class Manager : public beast::PropertyStream::Source +{ +protected: + Manager() noexcept; + +public: + /** + * Destroy the object. + * Any pending source fetch operations are aborted. + * There may be some listener calls made before the + * destructor returns. + */ + ~Manager() override = default; + + /** + * Set the configuration for the manager. + * The new settings will be applied asynchronously. + * Thread safety: + * Can be called from any threads at any time. + */ + virtual void + setConfig(Config const& config) = 0; + + /** + * Transition to the started state, synchronously. + */ + virtual void + start() = 0; + + /** + * Transition to the stopped state, synchronously. + */ + virtual void + stop() = 0; + + /** + * Returns the configuration for the manager. + */ + virtual Config + config() = 0; + + /** + * Add a peer that should always be connected. + * This is useful for maintaining a private cluster of peers. + * The string is the name as specified in the configuration + * file, along with the set of corresponding IP addresses. + */ + virtual void + addFixedPeer(std::string_view name, std::vector const& addresses) = 0; + + /** + * Add a set of strings as fallback IP::Endpoint sources. + * @param name A label used for diagnostics. + */ + virtual void + addFallbackStrings(std::string const& name, std::vector const& strings) = 0; + + /** + * Add a URL as a fallback location to obtain IP::Endpoint sources. + * @param name A label used for diagnostics. + */ + /* VFALCO NOTE Unimplemented + virtual void addFallbackURL (std::string const& name, + std::string const& url) = 0; + */ + + //-------------------------------------------------------------------------- + + /** + * Create a new inbound slot with the specified remote endpoint. + * If nullptr is returned, then the slot could not be assigned. + * Usually this is because of a detected self-connection. + */ + virtual std::pair, Result> + newInboundSlot( + beast::IP::Endpoint const& localEndpoint, + beast::IP::Endpoint const& remoteEndpoint) = 0; + + /** + * Create a new outbound slot with the specified remote endpoint. + * If nullptr is returned, then the slot could not be assigned. + * Usually this is because of a duplicate connection. + */ + virtual std::pair, Result> + newOutboundSlot(beast::IP::Endpoint const& remoteEndpoint) = 0; + + /** + * Called when mtENDPOINTS is received. + */ + virtual void + onEndpoints(std::shared_ptr const& slot, Endpoints const& endpoints) = 0; + + /** + * Called when the slot is closed. + * This always happens when the socket is closed, unless the socket + * was canceled. + */ + virtual void + onClosed(std::shared_ptr const& slot) = 0; + + /** + * Called when an outbound connection is deemed to have failed + */ + virtual void + onFailure(std::shared_ptr const& slot) = 0; + + /** + * Called when we received redirect IPs from a busy peer. + */ + virtual void + onRedirects( + boost::asio::ip::tcp::endpoint const& remoteAddress, + std::vector const& eps) = 0; + + //-------------------------------------------------------------------------- + + /** + * Called when an outbound connection attempt succeeds. + * The local endpoint must be valid. If the caller receives an error + * when retrieving the local endpoint from the socket, it should + * proceed as if the connection attempt failed by calling on_closed + * instead of on_connected. + * @return `true` if the connection should be kept + */ + virtual bool + onConnected(std::shared_ptr const& slot, beast::IP::Endpoint const& localEndpoint) = 0; + + /** + * Request an active slot type. + */ + virtual Result + activate(std::shared_ptr const& slot, PublicKey const& key, bool reserved) = 0; + + /** + * Returns a set of endpoints suitable for redirection. + */ + virtual std::vector + redirect(std::shared_ptr const& slot) = 0; + + /** + * Return a set of addresses we should connect to. + */ + virtual std::vector + autoconnect() = 0; + + virtual std::vector, std::vector>> + buildEndpointsForPeers() = 0; + + /** + * Perform periodic activity. + * This should be called once per second. + */ + virtual void + oncePerSecond() = 0; +}; + +} // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/Slot.h b/include/xrpl/peerfinder/Slot.h similarity index 100% rename from src/xrpld/peerfinder/Slot.h rename to include/xrpl/peerfinder/Slot.h diff --git a/include/xrpl/peerfinder/Types.h b/include/xrpl/peerfinder/Types.h new file mode 100644 index 0000000000..9e82d9d65c --- /dev/null +++ b/include/xrpl/peerfinder/Types.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace xrpl::PeerFinder { + +using clock_type = beast::AbstractClock; + +/** + * Represents a set of addresses. + */ +using IPAddresses = std::vector; + +//------------------------------------------------------------------------------ + +/** + * Describes a connectable peer address along with some metadata. + */ +struct Endpoint +{ + Endpoint() = default; + + Endpoint(beast::IP::Endpoint ep, std::uint32_t hops); + + std::uint32_t hops = 0; + beast::IP::Endpoint address; +}; + +inline bool +operator<(Endpoint const& lhs, Endpoint const& rhs) +{ + return lhs.address < rhs.address; +} + +/** + * A set of Endpoint used for connecting. + */ +using Endpoints = std::vector; + +} // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/Bootcache.h b/include/xrpl/peerfinder/detail/Bootcache.h similarity index 97% rename from src/xrpld/peerfinder/detail/Bootcache.h rename to include/xrpl/peerfinder/detail/Bootcache.h index c84fed42c7..2141a374fa 100644 --- a/src/xrpld/peerfinder/detail/Bootcache.h +++ b/include/xrpl/peerfinder/detail/Bootcache.h @@ -1,11 +1,11 @@ #pragma once -#include -#include - #include #include #include +#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Checker.h b/include/xrpl/peerfinder/detail/Checker.h similarity index 100% rename from src/xrpld/peerfinder/detail/Checker.h rename to include/xrpl/peerfinder/detail/Checker.h diff --git a/src/xrpld/peerfinder/detail/Counts.h b/include/xrpl/peerfinder/detail/Counts.h similarity index 98% rename from src/xrpld/peerfinder/detail/Counts.h rename to include/xrpl/peerfinder/detail/Counts.h index c90598c1a1..ce78eadce4 100644 --- a/src/xrpld/peerfinder/detail/Counts.h +++ b/include/xrpl/peerfinder/detail/Counts.h @@ -1,11 +1,10 @@ #pragma once -#include -#include -#include - #include #include +#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Fixed.h b/include/xrpl/peerfinder/detail/Fixed.h similarity index 92% rename from src/xrpld/peerfinder/detail/Fixed.h rename to include/xrpl/peerfinder/detail/Fixed.h index 24d54775ef..6754ec6dbd 100644 --- a/src/xrpld/peerfinder/detail/Fixed.h +++ b/include/xrpl/peerfinder/detail/Fixed.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Handouts.h b/include/xrpl/peerfinder/detail/Handouts.h similarity index 98% rename from src/xrpld/peerfinder/detail/Handouts.h rename to include/xrpl/peerfinder/detail/Handouts.h index 757f1a8e1b..cb5fd7f850 100644 --- a/src/xrpld/peerfinder/detail/Handouts.h +++ b/include/xrpl/peerfinder/detail/Handouts.h @@ -1,12 +1,11 @@ #pragma once -#include -#include -#include - #include #include #include +#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Livecache.h b/include/xrpl/peerfinder/detail/Livecache.h similarity index 95% rename from src/xrpld/peerfinder/detail/Livecache.h rename to include/xrpl/peerfinder/detail/Livecache.h index 2015098847..cac284d1cc 100644 --- a/src/xrpld/peerfinder/detail/Livecache.h +++ b/include/xrpl/peerfinder/detail/Livecache.h @@ -1,9 +1,5 @@ #pragma once -#include -#include -#include - #include #include #include @@ -12,6 +8,8 @@ #include #include #include +#include +#include #include #include @@ -22,6 +20,8 @@ #include #include #include +#include +#include #include #include #include @@ -411,7 +411,7 @@ Livecache::expire() } if (n > 0) { - JLOG(journal_.debug()) << beast::Leftw(18) << "Livecache expired " << n + JLOG(journal_.debug()) << std::left << std::setw(18) << "Livecache expired " << n << ((n > 1) ? " entries" : " entry"); } } @@ -434,7 +434,7 @@ Livecache::insert(Endpoint const& ep) if (result.second) { hops.insert(e); - JLOG(journal_.debug()) << beast::Leftw(18) << "Livecache insert " << ep.address + JLOG(journal_.debug()) << std::left << std::setw(18) << "Livecache insert " << ep.address << " at hops " << ep.hops; return; } @@ -442,7 +442,7 @@ Livecache::insert(Endpoint const& ep) { // Drop duplicates at higher hops std::size_t const excess(ep.hops - e.endpoint.hops); - JLOG(journal_.trace()) << beast::Leftw(18) << "Livecache drop " << ep.address + JLOG(journal_.trace()) << std::left << std::setw(18) << "Livecache drop " << ep.address << " at hops +" << excess; return; } @@ -453,12 +453,12 @@ Livecache::insert(Endpoint const& ep) if (ep.hops < e.endpoint.hops) { hops.reinsert(e, ep.hops); - JLOG(journal_.debug()) << beast::Leftw(18) << "Livecache update " << ep.address + JLOG(journal_.debug()) << std::left << std::setw(18) << "Livecache update " << ep.address << " at hops " << ep.hops; } else { - JLOG(journal_.trace()) << beast::Leftw(18) << "Livecache refresh " << ep.address + JLOG(journal_.trace()) << std::left << std::setw(18) << "Livecache refresh " << ep.address << " at hops " << ep.hops; } } diff --git a/src/xrpld/peerfinder/detail/Logic.h b/include/xrpl/peerfinder/detail/Logic.h similarity index 91% rename from src/xrpld/peerfinder/detail/Logic.h rename to include/xrpl/peerfinder/detail/Logic.h index a7dbbf850d..c623263884 100644 --- a/src/xrpld/peerfinder/detail/Logic.h +++ b/include/xrpl/peerfinder/detail/Logic.h @@ -1,17 +1,5 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include @@ -22,18 +10,34 @@ #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include + #include #include #include #include +#include +#include #include #include #include #include #include #include +#include #include #include #include @@ -197,8 +201,8 @@ public: if (result.second) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Logic add fixed '" << name << "' at " << remoteAddress; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic add fixed '" << name + << "' at " << remoteAddress; return; } } @@ -221,7 +225,7 @@ public: if (iter == slots.end()) { // The slot disconnected before we finished the check - JLOG(journal.debug()) << beast::Leftw(18) << "Logic tested " << checkedAddress + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic tested " << checkedAddress << " but the connection was closed"; return; } @@ -255,7 +259,7 @@ public: beast::IP::Endpoint const& localEndpoint, beast::IP::Endpoint const& remoteEndpoint) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic accept" << remoteEndpoint + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic accept" << remoteEndpoint << " on local " << localEndpoint; std::scoped_lock const _(lock); @@ -266,7 +270,7 @@ public: auto const count = connectedAddresses.count(remoteEndpoint.address()); if (count + 1 > config_.ipLimit) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic dropping inbound " + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic dropping inbound " << remoteEndpoint << " because of ip limits."; return {SlotImp::ptr(), Result::IpLimitExceeded}; } @@ -275,8 +279,8 @@ public: // Check for duplicate connection if (slots.contains(remoteEndpoint)) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic dropping " << remoteEndpoint - << " as duplicate incoming"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic dropping " + << remoteEndpoint << " as duplicate incoming"; return {SlotImp::ptr(), Result::DuplicatePeer}; } @@ -304,15 +308,15 @@ public: std::pair newOutboundSlot(beast::IP::Endpoint const& remoteEndpoint) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic connect " << remoteEndpoint; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic connect " << remoteEndpoint; std::scoped_lock const _(lock); // Check for duplicate connection if (slots.contains(remoteEndpoint)) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic dropping " << remoteEndpoint - << " as duplicate connect"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic dropping " + << remoteEndpoint << " as duplicate connect"; return {SlotImp::ptr(), Result::DuplicatePeer}; } @@ -506,15 +510,15 @@ public: if (!h.list().empty()) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Logic connect " << h.list().size() << " fixed"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic connect " + << h.list().size() << " fixed"; return h.list(); } if (counts_.attempts() > 0) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Logic waiting on " << counts_.attempts() << " attempts"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic waiting on " + << counts_.attempts() << " attempts"; return none; } } @@ -535,14 +539,14 @@ public: if (!h.list().empty()) { JLOG(journal.debug()) - << beast::Leftw(18) << "Logic connect " << h.list().size() << " live " + << std::left << std::setw(18) << "Logic connect " << h.list().size() << " live " << ((h.list().size() > 1) ? "endpoints" : "endpoint"); return h.list(); } if (counts_.attempts() > 0) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Logic waiting on " << counts_.attempts() << " attempts"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic waiting on " + << counts_.attempts() << " attempts"; return none; } } @@ -568,8 +572,9 @@ public: if (!h.list().empty()) { - JLOG(journal.debug()) << beast::Leftw(18) << "Logic connect " << h.list().size() - << " boot " << ((h.list().size() > 1) ? "addresses" : "address"); + JLOG(journal.debug()) << std::left << std::setw(18) << "Logic connect " + << h.list().size() << " boot " + << ((h.list().size() > 1) ? "addresses" : "address"); return h.list(); } @@ -689,8 +694,8 @@ public: // Enforce hop limit if (ep.hops > Tuning::kMaxHops) { - JLOG(journal.debug()) << beast::Leftw(18) << "Endpoints drop " << ep.address - << " for excess hops " << ep.hops; + JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop " + << ep.address << " for excess hops " << ep.hops; iter = list.erase(iter); continue; } @@ -706,18 +711,18 @@ public: } else { - JLOG(journal.debug()) - << beast::Leftw(18) << "Endpoints drop " << ep.address << " for extra self"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop " + << ep.address << " for extra self"; iter = list.erase(iter); continue; } } // Discard invalid addresses - if (config_.verifyEndpoints && !isValidAddress(ep.address)) + if (!isValidAddress(ep.address)) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Endpoints drop " << ep.address << " as invalid"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop " + << ep.address << " as invalid"; iter = list.erase(iter); continue; } @@ -727,8 +732,8 @@ public: return ep.address == other.address; })) { - JLOG(journal.debug()) - << beast::Leftw(18) << "Endpoints drop " << ep.address << " as duplicate"; + JLOG(journal.debug()) << std::left << std::setw(18) << "Endpoints drop " + << ep.address << " as duplicate"; iter = list.erase(iter); continue; } @@ -1074,13 +1079,13 @@ public: if (!results.error) { int const count(addBootcacheAddresses(results.addresses)); - JLOG(journal.info()) << beast::Leftw(18) << "Logic added " << count << " new " + JLOG(journal.info()) << std::left << std::setw(18) << "Logic added " << count << " new " << ((count == 1) ? "address" : "addresses") << " from " << source->name(); } else { - JLOG(journal.error()) << beast::Leftw(18) << "Logic failed " + JLOG(journal.error()) << std::left << std::setw(18) << "Logic failed " << "'" << source->name() << "' fetch, " << results.error.message(); } @@ -1098,8 +1103,6 @@ public: { if (isUnspecified(address)) return false; - if (isLoopback(address)) - return false; if (!isPublic(address)) return false; if (address.port() == 0) @@ -1221,8 +1224,8 @@ Logic::onRedirects( bootcache.insert(beast::IPAddressConversion::fromAsio(*first)); if (n > 0) { - JLOG(journal.trace()) << beast::Leftw(18) << "Logic add " << n << " redirect IPs from " - << remoteAddress; + JLOG(journal.trace()) << std::left << std::setw(18) << "Logic add " << n + << " redirect IPs from " << remoteAddress; } } diff --git a/src/xrpld/peerfinder/detail/SlotImp.h b/include/xrpl/peerfinder/detail/SlotImp.h similarity index 96% rename from src/xrpld/peerfinder/detail/SlotImp.h rename to include/xrpl/peerfinder/detail/SlotImp.h index 898941b157..35c61b13cf 100644 --- a/src/xrpld/peerfinder/detail/SlotImp.h +++ b/include/xrpl/peerfinder/detail/SlotImp.h @@ -1,10 +1,9 @@ #pragma once -#include -#include - #include #include +#include +#include #include #include @@ -172,7 +171,7 @@ private: std::optional localEndpoint_; std::optional publicKey_; - static constexpr std::int32_t kUnknownPort = -1; + static std::int32_t constexpr kUnknownPort = -1; std::atomic listeningPort_; public: diff --git a/src/xrpld/peerfinder/detail/Source.h b/include/xrpl/peerfinder/detail/Source.h similarity index 95% rename from src/xrpld/peerfinder/detail/Source.h rename to include/xrpl/peerfinder/detail/Source.h index b205dc8dfb..5cdb535bdd 100644 --- a/src/xrpld/peerfinder/detail/Source.h +++ b/include/xrpl/peerfinder/detail/Source.h @@ -1,8 +1,7 @@ #pragma once -#include - #include +#include #include diff --git a/src/xrpld/peerfinder/detail/SourceStrings.h b/include/xrpl/peerfinder/detail/SourceStrings.h similarity index 90% rename from src/xrpld/peerfinder/detail/SourceStrings.h rename to include/xrpl/peerfinder/detail/SourceStrings.h index b79cf0df03..325a024764 100644 --- a/src/xrpld/peerfinder/detail/SourceStrings.h +++ b/include/xrpl/peerfinder/detail/SourceStrings.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/Store.h b/include/xrpl/peerfinder/detail/Store.h similarity index 100% rename from src/xrpld/peerfinder/detail/Store.h rename to include/xrpl/peerfinder/detail/Store.h diff --git a/src/xrpld/peerfinder/detail/Tuning.h b/include/xrpl/peerfinder/detail/Tuning.h similarity index 100% rename from src/xrpld/peerfinder/detail/Tuning.h rename to include/xrpl/peerfinder/detail/Tuning.h diff --git a/include/xrpl/peerfinder/make_Manager.h b/include/xrpl/peerfinder/make_Manager.h new file mode 100644 index 0000000000..5da372e588 --- /dev/null +++ b/include/xrpl/peerfinder/make_Manager.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include + +namespace xrpl::PeerFinder { + +/** + * @brief Create a new Manager. + * + * @param ioContext The io_context used to schedule asynchronous work. + * @param clock The clock used for timekeeping. + * @param journal The journal used for logging. + * @param store The persistence backend for the bootstrap cache. The caller + * retains ownership and must keep it alive (and opened) for the lifetime of + * the returned Manager. This lets consumers supply their own Store + * implementation (e.g. the SQLite-backed StoreSqdb in xrpld). + * @param collector The collector used to report metrics. + * @return The newly created Manager. + */ +std::unique_ptr +makeManager( + boost::asio::io_context& ioContext, + clock_type& clock, + beast::Journal journal, + Store& store, + beast::insight::Collector::ptr const& collector); + +} // namespace xrpl::PeerFinder diff --git a/include/xrpl/protocol/detail/features.macro b/include/xrpl/protocol/detail/features.macro index bfe03a6303..4f1fac82da 100644 --- a/include/xrpl/protocol/detail/features.macro +++ b/include/xrpl/protocol/detail/features.macro @@ -15,6 +15,7 @@ // Add new amendments to the top of this list. // Keep it sorted in reverse chronological order. +XRPL_FIX (Cleanup3_4_0, Supported::Yes, VoteBehavior::DefaultNo) XRPL_FEATURE(Sponsor, Supported::Yes, VoteBehavior::DefaultNo) XRPL_FEATURE(BatchV1_1, Supported::Yes, VoteBehavior::DefaultNo) XRPL_FEATURE(LendingProtocolV1_1, Supported::No, VoteBehavior::DefaultNo) diff --git a/include/xrpl/shamap/Family.h b/include/xrpl/shamap/Family.h index 7624b3e600..467d41e4cc 100644 --- a/include/xrpl/shamap/Family.h +++ b/include/xrpl/shamap/Family.h @@ -26,10 +26,10 @@ public: explicit Family() = default; virtual ~Family() = default; - virtual NodeStore::Database& + virtual node_store::Database& db() = 0; - [[nodiscard]] virtual NodeStore::Database const& + [[nodiscard]] virtual node_store::Database const& db() const = 0; virtual beast::Journal const& diff --git a/include/xrpl/tx/invariants/PermissionedDEXInvariant.h b/include/xrpl/tx/invariants/PermissionedDEXInvariant.h index 2763b80a94..ae0d573385 100644 --- a/include/xrpl/tx/invariants/PermissionedDEXInvariant.h +++ b/include/xrpl/tx/invariants/PermissionedDEXInvariant.h @@ -17,7 +17,8 @@ class ValidPermissionedDEX bool regularOffers_ = false; // post-fixCleanup3_2_0: excludes deleted offers bool badHybridsOld_ = false; // pre-fixCleanup3_1_3: missing field/domain or size > 1 bool badHybrids_ = false; // post-fixCleanup3_1_3: also catches size == 0 (size != 1) - hash_set domains_; + hash_set domainsOld_; // pre-fixCleanup3_4_0: also flags deleted domains + hash_set domains_; // post-fixCleanup3_4_0: excludes deleted domains public: void diff --git a/nix/check-tools/README.md b/nix/check-tools/README.md new file mode 100644 index 0000000000..5b7538f2ca --- /dev/null +++ b/nix/check-tools/README.md @@ -0,0 +1,49 @@ +# check-tools snapshots + +These files capture the output of [`bin/check-tools.sh`](../../bin/check-tools.sh) +— the versions of the development tooling — in each Nix environment: + +| File | Environment | +| ---------------------- | ------------------------------------ | +| `nix-ubuntu-amd64.txt` | `nix-ubuntu` CI image, `linux/amd64` | +| `nix-ubuntu-arm64.txt` | `nix-ubuntu` CI image, `linux/arm64` | +| `macos.txt` | macOS, inside `nix develop` | + +The [`check-tools`](../../.github/workflows/check-tools.yml) workflow regenerates +each snapshot in its environment and fails if it differs from the committed file. +So if you change the environment (bump the image tag in +[`linux.json`](../../.github/scripts/strategy-matrix/linux.json), update +`flake.lock`, change the tool list in `check-tools.sh`, …) you must regenerate +and commit the affected snapshots. + +Each snapshot is `check-tools.sh` stdout with the git-clone connectivity check +skipped (`CHECK_TOOLS_SKIP_CLONE=1`), so it contains only deterministic version +data. On macOS the dev-shell greeting that `nix develop` prints first is dropped +with `sed -n '/^Detected OS:/,$p'`. + +## Regenerating + +The two Linux snapshots come from the `nix-ubuntu` image (Docker or a compatible +runtime such as Apple `container`). The image tag is pinned in `linux.json`: + +```bash +img="ghcr.io/xrplf/xrpld/nix-ubuntu:$(jq -r .image_tag .github/scripts/strategy-matrix/linux.json)" + +for arch in amd64 arm64; do + container run --rm -i -e CHECK_TOOLS_SKIP_CLONE=1 -a "${arch}" --entrypoint bash "${img}" -s \ + "nix/check-tools/nix-ubuntu-${arch}.txt" +done +``` + +(With Docker, replace `container run … -a "${arch}"` with +`docker run … --platform "linux/${arch}"`.) + +The macOS snapshot is generated locally. `CI=` is unset so `check-tools.sh` +checks the full dev-shell tool set (it otherwise skips some tools when `CI` is +set): + +```bash +CI= nix develop -c bash -c 'CHECK_TOOLS_SKIP_CLONE=1 bash bin/check-tools.sh' | + sed -n '/^Detected OS:/,$p' \ + >nix/check-tools/macos.txt +``` diff --git a/nix/check-tools/macos.txt b/nix/check-tools/macos.txt new file mode 100644 index 0000000000..93cc926181 --- /dev/null +++ b/nix/check-tools/macos.txt @@ -0,0 +1,47 @@ +Detected OS: macos (Darwin arm64) + +Core build tools: + [ ok ] cmake cmake version 4.1.2 + [ ok ] conan Conan version 2.28.1 + [ ok ] git git version 2.54.0 + [ ok ] python3 Python 3.13.13 + +Development tooling: + [ ok ] ccache ccache version 4.13.6 + [ ok ] clang clang version 22.1.7 + [ ok ] clang++ clang version 22.1.7 + [ ok ] ClangBuildAnalyzer ClangBuildAnalyzer 1.6.0 + [ ok ] curl curl 8.20.0 (aarch64-apple-darwin25.3.0) libcurl/8.20.0 OpenSSL/3.6.2 zlib/1.3.2 libssh2/1.11.1 nghttp2/1.69.0 mit-krb5/1.22.1 + [ ok ] file file-5.47 + [ ok ] less less 692 (PCRE2 regular expressions) + [ ok ] make GNU Make 4.4.1 + [ ok ] netstat present + [ ok ] ninja 1.13.2 + [ ok ] perl v5.42.0 + [ ok ] pkg-config 0.29.2 + [ ok ] vim VIM - Vi IMproved 9.2 (2026 Feb 14, compiled Jan 01 1980 00:00:00) + [ ok ] zip Zip 3.0 + [ ok ] clang-format clang-format version 22.1.7 + [ ok ] dot dot - graphviz version 12.2.1 (0) + [ ok ] doxygen 1.16.1 + [ ok ] gcovr gcovr 8.4 + [ ok ] gh gh version 2.94.0 (nixpkgs) + [ ok ] git-cliff git-cliff 2.13.1 + [ ok ] git-lfs git-lfs/3.7.1 (3.7.1; darwin arm64; go 1.26.3) + [ ok ] gpg gpg (GnuPG) 2.4.9 + [ ok ] pre-commit pre-commit 4.5.1 + [ ok ] run-clang-tidy usage: run-clang-tidy [-h] [-allow-enabling-alpha-checkers] + +Rust toolchain: + [ ok ] cargo cargo 1.95.0 (f2d3ce0bd 2026-03-21) + [ ok ] cargo-audit cargo-audit-audit 0.22.1 + [ ok ] cargo-llvm-cov cargo-llvm-cov 0.8.5 + [ ok ] cargo-nextest cargo-nextest 0.9.137 + [ ok ] clippy clippy 0.1.95 (59807616e1 2026-04-14) + [ ok ] rust-analyzer rust-analyzer 1.95.0 (59807616 2026-04-14) + [ ok ] rustc rustc 1.95.0 (59807616e 2026-04-14) + [ ok ] rustfmt rustfmt 1.9.0-stable (59807616e1 2026-04-14) + +Skipping git-over-HTTPS check (CHECK_TOOLS_SKIP_CLONE is set). + +All 36 checked tools are present and runnable. diff --git a/nix/check-tools/nix-ubuntu-amd64.txt b/nix/check-tools/nix-ubuntu-amd64.txt new file mode 100644 index 0000000000..b922cca4a8 --- /dev/null +++ b/nix/check-tools/nix-ubuntu-amd64.txt @@ -0,0 +1,55 @@ +Detected OS: linux (Linux x86_64) + +Core build tools: + [ ok ] cmake cmake version 4.1.2 + [ ok ] conan Conan version 2.28.1 + [ ok ] git git version 2.54.0 + [ ok ] python3 Python 3.13.13 + +Development tooling: + [ ok ] ccache ccache version 4.13.6 + [ ok ] clang clang version 22.1.7 + [ ok ] clang++ clang version 22.1.7 + [ ok ] ClangBuildAnalyzer ClangBuildAnalyzer 1.6.0 + [ ok ] curl curl 8.20.0 (x86_64-pc-linux-gnu) libcurl/8.20.0 OpenSSL/3.6.2 zlib/1.3.2 libssh2/1.11.1 nghttp2/1.69.0 mit-krb5/1.22.1 + [ ok ] file file-5.47 + [ ok ] less less 692 (PCRE2 regular expressions) + [ ok ] make GNU Make 4.4.1 + [ ok ] netstat net-tools 2.10 + [ ok ] ninja 1.13.2 + [ ok ] perl v5.42.0 + [ ok ] pkg-config 0.29.2 + [ ok ] vim VIM - Vi IMproved 9.2 (2026 Feb 14, compiled Jan 01 1980 00:00:00) + [ ok ] zip Zip 3.0 + [ ok ] clang-format clang-format version 22.1.7 + [ ok ] dot dot - graphviz version 12.2.1 (0) + [ ok ] doxygen 1.16.1 + [ ok ] gcovr gcovr 8.4 + [ ok ] gh gh version 2.94.0 (nixpkgs) + [ ok ] git-cliff git-cliff 2.13.1 + [ ok ] git-lfs git-lfs/3.7.1 (3.7.1; linux amd64; go 1.26.3) + [ ok ] gpg gpg (GnuPG) 2.4.9 + [ ok ] pre-commit pre-commit 4.5.1 + [ ok ] run-clang-tidy usage: run-clang-tidy [-h] [-allow-enabling-alpha-checkers] + +Rust toolchain: + [ ok ] cargo cargo 1.95.0 (f2d3ce0bd 2026-03-21) + [ ok ] cargo-audit cargo-audit-audit 0.22.1 + [ ok ] cargo-llvm-cov cargo-llvm-cov 0.8.5 + [ ok ] cargo-nextest cargo-nextest 0.9.137 + [ ok ] clippy clippy 0.1.95 (59807616e1 2026-04-14) + [ ok ] rust-analyzer rust-analyzer 1.95.0 (5980761 2026-04-14) + [ ok ] rustc rustc 1.95.0 (59807616e 2026-04-14) + [ ok ] rustfmt rustfmt 1.9.0-stable (59807616e1 2026-04-14) + +GCC toolchain: + [ ok ] gcc gcc (GCC) 15.2.0 + [ ok ] g++ g++ (GCC) 15.2.0 + [ ok ] gcov gcov (GCC) 15.2.0 + +Mold: + [ ok ] mold mold 2.41.0 (compatible with GNU ld) + +Skipping git-over-HTTPS check (CHECK_TOOLS_SKIP_CLONE is set). + +All 40 checked tools are present and runnable. diff --git a/nix/check-tools/nix-ubuntu-arm64.txt b/nix/check-tools/nix-ubuntu-arm64.txt new file mode 100644 index 0000000000..5267839682 --- /dev/null +++ b/nix/check-tools/nix-ubuntu-arm64.txt @@ -0,0 +1,55 @@ +Detected OS: linux (Linux aarch64) + +Core build tools: + [ ok ] cmake cmake version 4.1.2 + [ ok ] conan Conan version 2.28.1 + [ ok ] git git version 2.54.0 + [ ok ] python3 Python 3.13.13 + +Development tooling: + [ ok ] ccache ccache version 4.13.6 + [ ok ] clang clang version 22.1.7 + [ ok ] clang++ clang version 22.1.7 + [ ok ] ClangBuildAnalyzer ClangBuildAnalyzer 1.6.0 + [ ok ] curl curl 8.20.0 (aarch64-unknown-linux-gnu) libcurl/8.20.0 OpenSSL/3.6.2 zlib/1.3.2 libssh2/1.11.1 nghttp2/1.69.0 mit-krb5/1.22.1 + [ ok ] file file-5.47 + [ ok ] less less 692 (PCRE2 regular expressions) + [ ok ] make GNU Make 4.4.1 + [ ok ] netstat net-tools 2.10 + [ ok ] ninja 1.13.2 + [ ok ] perl v5.42.0 + [ ok ] pkg-config 0.29.2 + [ ok ] vim VIM - Vi IMproved 9.2 (2026 Feb 14, compiled Jan 01 1980 00:00:00) + [ ok ] zip Zip 3.0 + [ ok ] clang-format clang-format version 22.1.7 + [ ok ] dot dot - graphviz version 12.2.1 (0) + [ ok ] doxygen 1.16.1 + [ ok ] gcovr gcovr 8.4 + [ ok ] gh gh version 2.94.0 (nixpkgs) + [ ok ] git-cliff git-cliff 2.13.1 + [ ok ] git-lfs git-lfs/3.7.1 (3.7.1; linux arm64; go 1.26.3) + [ ok ] gpg gpg (GnuPG) 2.4.9 + [ ok ] pre-commit pre-commit 4.5.1 + [ ok ] run-clang-tidy usage: run-clang-tidy [-h] [-allow-enabling-alpha-checkers] + +Rust toolchain: + [ ok ] cargo cargo 1.95.0 (f2d3ce0bd 2026-03-21) + [ ok ] cargo-audit cargo-audit-audit 0.22.1 + [ ok ] cargo-llvm-cov cargo-llvm-cov 0.8.5 + [ ok ] cargo-nextest cargo-nextest 0.9.137 + [ ok ] clippy clippy 0.1.95 (59807616e1 2026-04-14) + [ ok ] rust-analyzer rust-analyzer 1.95.0 (5980761 2026-04-14) + [ ok ] rustc rustc 1.95.0 (59807616e 2026-04-14) + [ ok ] rustfmt rustfmt 1.9.0-stable (59807616e1 2026-04-14) + +GCC toolchain: + [ ok ] gcc gcc (GCC) 15.2.0 + [ ok ] g++ g++ (GCC) 15.2.0 + [ ok ] gcov gcov (GCC) 15.2.0 + +Mold: + [ ok ] mold mold 2.41.0 (compatible with GNU ld) + +Skipping git-over-HTTPS check (CHECK_TOOLS_SKIP_CLONE is set). + +All 40 checked tools are present and runnable. diff --git a/nix/ci-env.nix b/nix/ci-env.nix index 9b754af97d..787b94406e 100644 --- a/nix/ci-env.nix +++ b/nix/ci-env.nix @@ -6,106 +6,18 @@ let inherit (import ./packages.nix { inherit pkgs; }) commonPackages - gccPackage - llvmPackages + gccVersion llvmVersion + mkVersionedToolLinks ; - # Underlying compiler toolchains to wrap (versions pinned in packages.nix). - customGccPackage = gccPackage; - customLlvmPackages = llvmPackages; - - # binutils wrapped to emit binaries that reference the custom glibc - # (dynamic linker path, library search path, RPATH). - customBinutils = pkgs.wrapBintoolsWith { - bintools = pkgs.binutils-unwrapped; - libc = customGlibc; - }; - - # Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom - # glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses - # the existing gcc binary but links against the custom glibc, so the - # resulting compiler ships runtime libraries that only reference symbols - # available in that glibc. - customGccCc = customGccPackage.cc.override { - stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv ( - pkgs.wrapCCWith { - cc = customGccPackage.cc; - libc = customGlibc; - bintools = customBinutils; - } - ); - }; - - # cc-wrapper around the rebuilt compiler, pointing at the custom glibc - # headers and libraries. This is what we actually expose to users. - customGcc = pkgs.wrapCCWith { - cc = customGccCc; - libc = customGlibc; - bintools = customBinutils; - }; - - # gcov ships in gcc's `cc` output, but the cc-wrapper doesn't expose it. - # Surface the gcov from our rebuilt gcc (linked against the custom glibc, so - # it runs under the loader installed in the image) and matching the exact - # compiler version, so gcovr can produce coverage reports in the CI env. - customGcov = pkgs.runCommand "gcov-custom-for-ci-env" { } '' - mkdir -p "$out/bin" - ln -s "${customGccCc}/bin/gcov" "$out/bin/gcov" - ''; - - # stdenv built around the rebuilt gcc / custom glibc. Used to rebuild - # compiler-rt below so its sanitizer runtimes see the custom glibc - # headers. - customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc; - - # Rebuild compiler-rt against the custom glibc so the sanitizer runtimes - # don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that - # only exist in newer glibc versions. scudo is dropped because its CMake - # includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which - # makes std::atomic unfindable in our stdenv; we don't use scudo (only - # asan/ubsan/tsan etc.). - customCompilerRt = - (customLlvmPackages.compiler-rt.override { - stdenv = customStdenv; - }).overrideAttrs - (old: { - postPatch = (old.postPatch or "") + '' - substituteInPlace lib/CMakeLists.txt \ - --replace-quiet 'add_subdirectory(scudo/standalone)' \ - '# scudo/standalone disabled in xrpld ci-env' - ''; - }); - - # cc-wrapper around clang, pointing at the custom glibc headers and - # libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that - # C++ binaries produced by clang also only reference symbols available - # in the custom glibc. compiler-rt is wired into a resource-root so - # sanitizer runtimes (libclang_rt.*.a) are found at link time; this - # mirrors what nixpkgs does internally when building llvmPackages.clang. - customClang = pkgs.wrapCCWith { - cc = customLlvmPackages.clang-unwrapped; - libc = customGlibc; - bintools = customBinutils; - gccForLibs = customGccCc; - extraPackages = [ customCompilerRt ]; - extraBuildCommands = '' - rsrc="$out/resource-root" - mkdir "$rsrc" - ln -s "${customLlvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include" - ln -s "${customCompilerRt.out}/lib" "$rsrc/lib" - ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true - echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags - # compiler-rt ships the sanitizer/profile/xray interface headers (e.g. - # ) in its `dev` output. In a normal Nix - # build these reach the include path because compiler-rt is propagated - # via depsTargetTargetPropagated and stdenv's setup hooks add its - # dev/include. The CI image runs clang outside a Nix stdenv (binaries - # on PATH, no setup hooks), so that never happens; add the headers - # explicitly. gcc ships its own copy, which is why this is clang-only. - echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags - ''; - }; + # Custom-glibc toolchain, shared with the Linux dev shell (see compilers.nix). + inherit (import ./compilers.nix { inherit pkgs customGlibc; }) + customGcc + customClang + customBinutils + customGcov + ; # Strip the generic cc/c++/cpp symlinks from the clang wrapper so it can # coexist with the gcc wrapper in buildEnv. gcc remains the default @@ -127,6 +39,25 @@ in customGcov customClangForCiEnv customBinutils + (mkVersionedToolLinks { + name = "gcc"; + package = customGcc; + version = gccVersion; + tools = [ + "gcc" + "g++" + "cpp" + ]; + }) + (mkVersionedToolLinks { + name = "clang"; + package = customClang; + version = llvmVersion; + tools = [ + "clang" + "clang++" + ]; + }) # CA certificate bundle so HTTPS clients (git, curl, conan) can verify # TLS connections without ca-certificates being installed in the system. pkgs.cacert diff --git a/nix/compilers.nix b/nix/compilers.nix new file mode 100644 index 0000000000..90856afacc --- /dev/null +++ b/nix/compilers.nix @@ -0,0 +1,117 @@ +# Custom-glibc compiler toolchain shared by the CI environment (ci-env.nix) and +# the Linux dev shell (devshell.nix): gcc / clang / binutils rebuilt to target +# the pinned custom glibc. Linux only — the pinned glibc snapshot does not build +# on darwin, so callers must not evaluate this on macOS. +{ + pkgs, + customGlibc, +}: +let + inherit (import ./packages.nix { inherit pkgs; }) + gccPackage + llvmPackages + llvmVersion + mkGcov + ; + + # binutils wrapped to emit binaries that reference the custom glibc + # (dynamic linker path, library search path, RPATH). + customBinutils = pkgs.wrapBintoolsWith { + bintools = pkgs.binutils-unwrapped; + libc = customGlibc; + }; + + # Rebuild gcc (specifically libstdc++ / libgcc_s) against the custom + # glibc. The override swaps gcc.cc's bootstrap stdenv for one that uses + # the existing gcc binary but links against the custom glibc, so the + # resulting compiler ships runtime libraries that only reference symbols + # available in that glibc. + customGccCc = gccPackage.cc.override { + stdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv ( + pkgs.wrapCCWith { + cc = gccPackage.cc; + libc = customGlibc; + bintools = customBinutils; + } + ); + }; + + # cc-wrapper around the rebuilt compiler, pointing at the custom glibc + # headers and libraries. This is what we actually expose to users. + customGcc = pkgs.wrapCCWith { + cc = customGccCc; + libc = customGlibc; + bintools = customBinutils; + }; + + # gcov matching the rebuilt gcc (linked against the custom glibc), so gcovr + # can produce coverage reports both in CI and in the dev shell. + customGcov = mkGcov { + name = "custom"; + cc = customGccCc; + }; + + # stdenv built around the rebuilt gcc / custom glibc. Exported as the dev + # shell's gcc stdenv, and used below to rebuild compiler-rt so its sanitizer + # runtimes see the custom glibc headers. + customStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customGcc; + + # Rebuild compiler-rt against the custom glibc so the sanitizer runtimes + # don't use glibc symbols (or sysconf constants like _SC_SIGSTKSZ) that + # only exist in newer glibc versions. scudo is dropped because its CMake + # includes CheckAtomic with -nostdinc++ in CMAKE_REQUIRED_FLAGS, which + # makes std::atomic unfindable in our stdenv; we don't use scudo (only + # asan/ubsan/tsan etc.). + customCompilerRt = + (llvmPackages.compiler-rt.override { + stdenv = customStdenv; + }).overrideAttrs + (old: { + postPatch = (old.postPatch or "") + '' + substituteInPlace lib/CMakeLists.txt \ + --replace-quiet 'add_subdirectory(scudo/standalone)' \ + '# scudo/standalone disabled in xrpld ci-env' + ''; + }); + + # cc-wrapper around clang, pointing at the custom glibc headers and + # libraries. Reuses the rebuilt gcc for libstdc++ / libgcc_s so that + # C++ binaries produced by clang also only reference symbols available + # in the custom glibc. compiler-rt is wired into a resource-root so + # sanitizer runtimes (libclang_rt.*.a) are found at link time; this + # mirrors what nixpkgs does internally when building llvmPackages.clang. + customClang = pkgs.wrapCCWith { + cc = llvmPackages.clang-unwrapped; + libc = customGlibc; + bintools = customBinutils; + gccForLibs = customGccCc; + extraPackages = [ customCompilerRt ]; + extraBuildCommands = '' + rsrc="$out/resource-root" + mkdir "$rsrc" + ln -s "${llvmPackages.clang-unwrapped.lib}/lib/clang/${toString llvmVersion}/include" "$rsrc/include" + ln -s "${customCompilerRt.out}/lib" "$rsrc/lib" + ln -s "${customCompilerRt.out}/share" "$rsrc/share" || true + echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags + # compiler-rt ships the sanitizer/profile/xray interface headers (e.g. + # ) in its `dev` output. In a normal Nix + # build these reach the include path because compiler-rt is propagated + # via depsTargetTargetPropagated and stdenv's setup hooks add its + # dev/include. The CI image runs clang outside a Nix stdenv (binaries + # on PATH, no setup hooks), so that never happens; add the headers + # explicitly. gcc ships its own copy, which is why this is clang-only. + echo "-isystem ${customCompilerRt.dev}/include" >> $out/nix-support/cc-cflags + ''; + }; +in +{ + inherit + customGcc + customClang + customBinutils + customStdenv + customGcov + ; + + customClangStdenv = pkgs.stdenvAdapters.overrideCC pkgs.stdenv customClang; +} diff --git a/nix/devshell.nix b/nix/devshell.nix index 34f173ef08..cb4a99c76a 100644 --- a/nix/devshell.nix +++ b/nix/devshell.nix @@ -1,23 +1,69 @@ -{ pkgs, ... }: +{ pkgs, customGlibc, ... }: let inherit (import ./packages.nix { inherit pkgs; }) commonPackages + gccPackage gccVersion + llvmVersion llvmPackages + mkVersionedToolLinks + mkGcov ; - # Plain nixpkgs stdenvs — no custom glibc, unlike ci-env.nix. - gccStdenv = pkgs."gcc${toString gccVersion}Stdenv"; - clangStdenv = llvmPackages.stdenv; + # Plain nixpkgs stdenvs — no custom glibc. + plainGccStdenv = pkgs."gcc${toString gccVersion}Stdenv"; + plainClangStdenv = llvmPackages.stdenv; + + # Custom-glibc stdenvs, matching the CI environment (see compilers.nix). The + # pinned glibc snapshot only builds on Linux, so on darwin these fall back to + # the plain stdenvs; the `if isLinux` guard keeps `customGlibc` from being + # forced (and erroring) on macOS. + customCompilers = import ./compilers.nix { inherit pkgs customGlibc; }; + customGccStdenv = if pkgs.stdenv.isLinux then customCompilers.customStdenv else plainGccStdenv; + customClangStdenv = + if pkgs.stdenv.isLinux then customCompilers.customClangStdenv else plainClangStdenv; + + # gcov matching each gcc shell, so `-Dcoverage=ON` builds work in the shell. + plainGcov = mkGcov { + name = "plain"; + cc = gccPackage.cc; + }; + customGccGcov = if pkgs.stdenv.isLinux then customCompilers.customGcov else plainGcov; + + # Shown when entering a *-plain shell. These exist only on Linux (see below), + # where the stock toolchain diverges from CI. + plainWarningHook = '' + echo "⚠️ WARNING: this is the stock nixpkgs toolchain and does not match CI's glibc. Prefer 'nix develop .#gcc' / '.#clang' unless you need to skip the custom-glibc build." + ''; + + # Tools to expose under version-suffixed names (see mkVersionedToolLinks). + gccVersionedTools = [ + "gcc" + "g++" + "cpp" + ]; + clangVersionedTools = [ + "clang" + "clang++" + ]; # compilerName is the command used to print the version, or null for none. makeShell = { + shellName, stdenv, compilerName, + version ? null, + versionedTools ? [ ], + extraPackages ? [ ], + warningHook ? "", + # Opt out of PatchNixBinary.cmake retargeting binaries to the system + # loader. The plain toolchain links a newer glibc, so it must not be + # patched; the custom toolchain patches by default. + noPatchNixBinary ? false, }: let - compilerVersion = + compilerVersionHook = if compilerName == null then ''echo "No compiler specified - using system compiler"'' else @@ -25,33 +71,81 @@ let echo "Compiler: " ${compilerName} --version ''; + versionedLinks = pkgs.lib.optional (version != null) (mkVersionedToolLinks { + name = compilerName; + package = stdenv.cc; + inherit version; + tools = versionedTools; + }); in - (pkgs.mkShell.override { inherit stdenv; }) { - packages = commonPackages; - shellHook = '' - echo "Welcome to xrpld development shell"; - ${compilerVersion} - ''; - }; + (pkgs.mkShell.override { inherit stdenv; }) ( + { + packages = commonPackages ++ versionedLinks ++ extraPackages; + # Marks a managed dev shell, so the build (XrplSanity.cmake) can tell an + # intentional Nix toolchain from one leaked into a bare shell. + XRPL_DEVSHELL = shellName; + shellHook = '' + echo "Welcome to xrpld development shell"; + ${compilerVersionHook} + ${warningHook} + ''; + } + // pkgs.lib.optionalAttrs noPatchNixBinary { XRPLD_NO_PATCH_NIX_BINARY = "1"; } + ); in rec { # macOS: Nix Clang. Linux: Nix GCC. default = if pkgs.stdenv.isDarwin then clang else gcc; + # gcc/clang use the custom-glibc toolchain, matching CI. On darwin there is no + # custom glibc, so they fall back to the plain nixpkgs toolchain. gcc = makeShell { - stdenv = gccStdenv; + shellName = "gcc"; + stdenv = customGccStdenv; compilerName = "gcc"; + version = gccVersion; + versionedTools = gccVersionedTools; + extraPackages = [ customGccGcov ]; }; clang = makeShell { - stdenv = clangStdenv; + shellName = "clang"; + stdenv = customClangStdenv; compilerName = "clang"; + version = llvmVersion; + versionedTools = clangVersionedTools; }; # Nix provides no compiler; use the one from your system (e.g. Apple Clang). no-compiler = makeShell { + shellName = "no-compiler"; stdenv = pkgs.stdenvNoCC; compilerName = null; }; apple-clang = no-compiler; } +# The *-plain shells (stock nixpkgs toolchain) exist only on Linux: on darwin +# gcc/clang are already plain, so these would be redundant and are omitted, which +# makes `nix develop .#gcc-plain` fail there rather than silently aliasing gcc. +// pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { + gcc-plain = makeShell { + shellName = "gcc-plain"; + stdenv = plainGccStdenv; + compilerName = "gcc"; + version = gccVersion; + versionedTools = gccVersionedTools; + extraPackages = [ plainGcov ]; + warningHook = plainWarningHook; + noPatchNixBinary = true; + }; + + clang-plain = makeShell { + shellName = "clang-plain"; + stdenv = plainClangStdenv; + compilerName = "clang"; + version = llvmVersion; + versionedTools = clangVersionedTools; + warningHook = plainWarningHook; + noPatchNixBinary = true; + }; +} diff --git a/nix/docker/Dockerfile b/nix/docker/Dockerfile index 7222cc8fa8..74c630cb61 100644 --- a/nix/docker/Dockerfile +++ b/nix/docker/Dockerfile @@ -8,10 +8,12 @@ RUN mkdir -p ~/.config/nix && \ # Copy our source and setup our working dir. COPY nix/ci-env.nix /tmp/build/nix/ci-env.nix +COPY nix/compilers.nix /tmp/build/nix/compilers.nix COPY nix/packages.nix /tmp/build/nix/packages.nix COPY nix/utils.nix /tmp/build/nix/utils.nix COPY flake.nix /tmp/build/ COPY flake.lock /tmp/build/ +COPY rust-toolchain.toml /tmp/build/ WORKDIR /tmp/build FROM builder-source AS builder @@ -56,7 +58,7 @@ ENV GIT_SSL_CAINFO="/nix/ci-env/etc/ssl/certs/ca-bundle.crt" # Externally-built dynamically-linked ELF binaries hard-code the loader path # (e.g. /lib64/ld-linux-x86-64.so.2) in their PT_INTERP header. Install it # from the Nix store when the base image doesn't already provide one. -COPY nix/docker/loader-path.sh /tmp/loader-path.sh +COPY bin/default-loader-path.sh /tmp/loader-path.sh RUN <&2 + exit 1 + fi + + local binary="${dst_dir}/proc_macro" + cp "${built}" "${binary}" + + echo "=== Patching ${binary} to use ${loader} as PT_INTERP ===" + patchelf --set-interpreter "${loader}" --remove-rpath "${binary}" + + rm -rf "${proj}/target" +} + +compile_proc_macro + echo "=== All binaries compiled ===" ls -la "${dst_dir}" diff --git a/nix/docker/test_files/rust/proc_macro/Cargo.lock b/nix/docker/test_files/rust/proc_macro/Cargo.lock new file mode 100644 index 0000000000..acab3fa0b9 --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/Cargo.lock @@ -0,0 +1,14 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "echo_macro" +version = "0.0.0" + +[[package]] +name = "test_macro" +version = "0.0.0" +dependencies = [ + "echo_macro", +] diff --git a/nix/docker/test_files/rust/proc_macro/Cargo.toml b/nix/docker/test_files/rust/proc_macro/Cargo.toml new file mode 100644 index 0000000000..d54955de3d --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/Cargo.toml @@ -0,0 +1,3 @@ +[workspace] +resolver = "2" +members = ["echo_macro", "test_macro"] diff --git a/nix/docker/test_files/rust/proc_macro/echo_macro/Cargo.toml b/nix/docker/test_files/rust/proc_macro/echo_macro/Cargo.toml new file mode 100644 index 0000000000..b0f92ce75a --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/echo_macro/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "echo_macro" +version = "0.0.0" +edition = "2024" +publish = false + +[lib] +proc-macro = true diff --git a/nix/docker/test_files/rust/proc_macro/echo_macro/src/lib.rs b/nix/docker/test_files/rust/proc_macro/echo_macro/src/lib.rs new file mode 100644 index 0000000000..e4b90d58fe --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/echo_macro/src/lib.rs @@ -0,0 +1,6 @@ +use proc_macro::TokenStream; + +#[proc_macro] +pub fn define_echo(item: TokenStream) -> TokenStream { + format!("fn echo() -> u32 {{ {item} }}").parse().unwrap() +} diff --git a/nix/docker/test_files/rust/proc_macro/test_macro/Cargo.toml b/nix/docker/test_files/rust/proc_macro/test_macro/Cargo.toml new file mode 100644 index 0000000000..25b6ba8e45 --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/test_macro/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "test_macro" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +echo_macro = { path = "../echo_macro" } diff --git a/nix/docker/test_files/rust/proc_macro/test_macro/src/main.rs b/nix/docker/test_files/rust/proc_macro/test_macro/src/main.rs new file mode 100644 index 0000000000..77718b9d75 --- /dev/null +++ b/nix/docker/test_files/rust/proc_macro/test_macro/src/main.rs @@ -0,0 +1,9 @@ +use echo_macro::define_echo; + +define_echo!(42); + +fn main() { + let a = echo(); + println!("proc-macro answer = {a}"); + assert_eq!(a, 42, "proc-macro expansion produced the wrong value"); +} diff --git a/nix/docker/test_files/rust/run-binaries.sh b/nix/docker/test_files/rust/run-binaries.sh index b627c12609..4cafee00ec 100755 --- a/nix/docker/test_files/rust/run-binaries.sh +++ b/nix/docker/test_files/rust/run-binaries.sh @@ -1,7 +1,7 @@ #!/bin/bash # Run pre-compiled Rust binaries and confirm each emits its expected diagnostic. # Binaries must already exist in as for name in -# {hello,panic,overflow}. +# {hello,panic,overflow,proc_macro}. set -eo pipefail @@ -54,12 +54,13 @@ declare -A expect=( [hello]="Hello from main thread" [panic]="explicit panic from test" [overflow]="attempt to add with overflow" + [proc_macro]="proc-macro answer = 42" ) -for name in hello panic overflow; do +for name in hello panic overflow proc_macro; do binary="${bins_dir}/${name}" - if [ "${name}" = "hello" ]; then + if [ "${name}" = "hello" ] || [ "${name}" = "proc_macro" ]; then expected_rc=0 else expected_rc=nonzero diff --git a/nix/packages.nix b/nix/packages.nix index 41d7e97328..01ab2ecf9a 100644 --- a/nix/packages.nix +++ b/nix/packages.nix @@ -15,6 +15,82 @@ let runClangTidy = pkgs.writeShellScriptBin "run-clang-tidy" '' exec ${pkgs.python3}/bin/python3 ${llvmPackages.clang-unwrapped}/bin/run-clang-tidy "$@" ''; + + # rust-overlay's toolchain propagates the *default* stdenv.cc onto the PATH (so + # cargo has a linker). That default may be different from the clang we pin here, + # so it shadows our clang and the build can silently use a different compiler + # version. Drop that cc from every propagation channel instead of pinning a + # replacement: the toolchain then carries no compiler and cargo just uses the + # active shell's stdenv cc. Must cover all channels — rust-overlay uses both + # propagatedBuildInputs and depsHostHostPropagated. + rustToolchainBase = pkgs.rust-bin.fromRustupToolchainFile ../rust-toolchain.toml; + rustToolchain = + let + defaultCc = pkgs.stdenv.cc; # default compiler from nixpkgs stdenv + withoutDefaultCc = builtins.filter (dep: (dep.outPath or "") != defaultCc.outPath); + in + rustToolchainBase.overrideAttrs (old: { + propagatedBuildInputs = withoutDefaultCc (old.propagatedBuildInputs or [ ]); + depsHostHostPropagated = withoutDefaultCc (old.depsHostHostPropagated or [ ]); + }); + + # Nix wraps its toolchain so that binaries are exposed only under unsuffixed + # names (gcc, g++, clang-tidy, ...). Several tools probe for a + # version-suffixed name first and fall back to a system binary on the PATH + # when Nix doesn't provide it: + # - Conan's Boost recipe looks up `g++-` before plain `g++`. + # - bin/pre-commit/clang_tidy_check.py looks up `run-clang-tidy-` and + # `clang-apply-replacements-` before the unsuffixed names. + # On a host that also has the matching system binary (e.g. Ubuntu's + # `/usr/bin/g++-15` or `clang-tidy-22`) the probe escapes Nix and mixes a + # system tool into the Nix environment. Generate version-suffixed symlinks + # next to a package's tools so those probes resolve to the Nix ones. + # + # Compiler links must point at whichever compiler is active in a given + # environment (the plain stdenv compiler in the dev shell, the custom-glibc + # wrappers in ci-env.nix), so those callers pass their own `package`; the + # clang tooling is environment-independent and is linked in commonPackages. + mkVersionedToolLinks = + { + name, + package, + version, + tools, + }: + pkgs.linkFarm "${name}-${toString version}-versioned-links" ( + map (tool: { + name = "bin/${tool}-${toString version}"; + path = "${package}/bin/${tool}"; + }) tools + ); + + # The cc-wrapper doesn't re-export gcov, but coverage tooling (gcovr) needs a + # gcov that exactly matches the compiler. Surface it from a gcc `cc` output. + mkGcov = + { name, cc }: + pkgs.linkFarm "gcov-${name}" [ + { + name = "bin/gcov"; + path = "${cc}/bin/gcov"; + } + ]; + + clangToolLinks = mkVersionedToolLinks { + name = "clang-tools"; + package = clangTools; + version = llvmVersion; + tools = [ + "clang-tidy" + "clang-apply-replacements" + "clang-format" + ]; + }; + runClangTidyLink = mkVersionedToolLinks { + name = "run-clang-tidy"; + package = runClangTidy; + version = llvmVersion; + tools = [ "run-clang-tidy" ]; + }; in { inherit @@ -22,9 +98,13 @@ in llvmVersion gccPackage llvmPackages + mkVersionedToolLinks + mkGcov ; commonPackages = with pkgs; [ + clangToolLinks + runClangTidyLink ccache clangbuildanalyzer clangTools @@ -63,14 +143,10 @@ in vim zip # Rust packages - cargo cargo-audit cargo-llvm-cov cargo-nextest - clippy corrosion - rust-analyzer - rustc - rustfmt + rustToolchain ]; } diff --git a/nix/utils.nix b/nix/utils.nix index d83e612c16..0b70183ef3 100644 --- a/nix/utils.nix +++ b/nix/utils.nix @@ -1,4 +1,8 @@ -{ nixpkgs, nixpkgs-custom-glibc }: +{ + nixpkgs, + nixpkgs-custom-glibc, + rust-overlay, +}: function: nixpkgs.lib.genAttrs [ @@ -10,7 +14,12 @@ nixpkgs.lib.genAttrs ( system: function { - pkgs = import nixpkgs { inherit system; }; + # rust-overlay adds `pkgs.rust-bin`, from which we build the pinned Rust + # toolchain (see packages.nix). Consumed by both the CI image and dev shell. + pkgs = import nixpkgs { + inherit system; + overlays = [ (import rust-overlay) ]; + }; # glibc 2.31 — matches the system libc on Ubuntu 20.04 LTS. Sourced # from the nixpkgs snapshot pinned via the `nixpkgs-custom-glibc` # flake input, so the build uses the compiler from that snapshot diff --git a/rust-toolchain.toml b/rust-toolchain.toml index dbc9e74c5d..a82b4734d8 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,8 +1,4 @@ -# Rust toolchain pin for rustup-based CI runners and local development. -# rustup reads this file and installs the pinned toolchain (see the -# prepare-runner action in XRPLF/actions, which runs `rustup toolchain install`). -# NOTE: the Nix CI image and development shell ignore this file; its rustc comes from flake.lock. [toolchain] channel = "1.95" -components = ["rustfmt", "clippy"] +components = ["rustfmt", "clippy", "rust-analyzer", "llvm-tools-preview"] profile = "minimal" diff --git a/src/benchmarks/libxrpl/nodestore/Backend.cpp b/src/benchmarks/libxrpl/nodestore/Backend.cpp index 7db0185053..f854dbd3a7 100644 --- a/src/benchmarks/libxrpl/nodestore/Backend.cpp +++ b/src/benchmarks/libxrpl/nodestore/Backend.cpp @@ -17,7 +17,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { namespace { constexpr std::size_t kPoolSizes[] = {1000, 10000, 100000}; @@ -326,4 +326,4 @@ registerStoreBatch(BackendConfig const& bc) }(); } // namespace -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/benchmarks/libxrpl/nodestore/Database.cpp b/src/benchmarks/libxrpl/nodestore/Database.cpp index 2303075ab9..cd4337b603 100644 --- a/src/benchmarks/libxrpl/nodestore/Database.cpp +++ b/src/benchmarks/libxrpl/nodestore/Database.cpp @@ -18,7 +18,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { namespace { // Number of distinct objects pre-generated per run. @@ -240,4 +240,4 @@ registerWorkload(BackendConfig const& bc, Workload const& w) }(); } // namespace -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/benchmarks/libxrpl/nodestore/NodeStoreBench.h b/src/benchmarks/libxrpl/nodestore/NodeStoreBench.h index fe6c2a350e..57abf42e89 100644 --- a/src/benchmarks/libxrpl/nodestore/NodeStoreBench.h +++ b/src/benchmarks/libxrpl/nodestore/NodeStoreBench.h @@ -32,7 +32,7 @@ // Shared helpers for the NodeStore benchmarks. // -namespace xrpl::NodeStore { +namespace xrpl::node_store { // Fill `bytes` of memory at `buffer` with random bits drawn from `g`. template @@ -315,4 +315,4 @@ backendConfigs() return kConfigs; } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/xrpld/consensus/Consensus.cpp b/src/libxrpl/consensus/Consensus.cpp similarity index 98% rename from src/xrpld/consensus/Consensus.cpp rename to src/libxrpl/consensus/Consensus.cpp index d529ab2e44..6f398cf66c 100644 --- a/src/xrpld/consensus/Consensus.cpp +++ b/src/libxrpl/consensus/Consensus.cpp @@ -1,10 +1,9 @@ -#include - -#include -#include +#include #include #include +#include +#include #include #include diff --git a/src/libxrpl/nodestore/BatchWriter.cpp b/src/libxrpl/nodestore/BatchWriter.cpp index e0a1fbf20c..92dfa09e0c 100644 --- a/src/libxrpl/nodestore/BatchWriter.cpp +++ b/src/libxrpl/nodestore/BatchWriter.cpp @@ -11,7 +11,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { BatchWriter::BatchWriter(Callback& callback, Scheduler& scheduler) : callback_(callback), scheduler_(scheduler) @@ -72,7 +72,7 @@ BatchWriter::writeBatch() writeSet_.swap(set); XRPL_ASSERT( - writeSet_.empty(), "xrpl::NodeStore::BatchWriter::writeBatch : writes not set"); + writeSet_.empty(), "xrpl::node_store::BatchWriter::writeBatch : writes not set"); writeLoad_ = set.size(); if (set.empty()) @@ -107,4 +107,4 @@ BatchWriter::waitForWriting() writeCondition_.wait(sl); } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/Database.cpp b/src/libxrpl/nodestore/Database.cpp index ac51dbfb2c..f9de660042 100644 --- a/src/libxrpl/nodestore/Database.cpp +++ b/src/libxrpl/nodestore/Database.cpp @@ -30,7 +30,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { Database::Database( Scheduler& scheduler, @@ -43,7 +43,7 @@ Database::Database( , requestBundle_(get(config, Keys::kRqBundle, 4)) , readThreads_(std::max(1, readThreads)) { - XRPL_ASSERT(readThreads, "xrpl::NodeStore::Database::Database : nonzero threads input"); + XRPL_ASSERT(readThreads, "xrpl::node_store::Database::Database : nonzero threads input"); if (earliestLedgerSeq_ < 1) Throw("Invalid earliest_seq"); @@ -89,7 +89,7 @@ Database::Database( { XRPL_ASSERT( !it->second.empty(), - "xrpl::NodeStore::Database::Database : non-empty " + "xrpl::node_store::Database::Database : non-empty " "data"); auto const& hash = it->first; @@ -164,7 +164,7 @@ Database::stop() { XRPL_ASSERT( steady_clock::now() - start < 30s, - "xrpl::NodeStore::Database::stop : maximum stop duration"); + "xrpl::node_store::Database::stop : maximum stop duration"); std::this_thread::yield(); } @@ -213,7 +213,7 @@ Database::importInternal(Backend& dstBackend, Database& srcDB) }; srcDB.forEach([&](std::shared_ptr nodeObject) { - XRPL_ASSERT(nodeObject, "xrpl::NodeStore::Database::importInternal : non-null node"); + XRPL_ASSERT(nodeObject, "xrpl::node_store::Database::importInternal : non-null node"); if (!nodeObject) // This should never happen return; @@ -257,7 +257,7 @@ Database::fetchNodeObject( void Database::getCountsJson(json::Value& obj) { - XRPL_ASSERT(obj.isObject(), "xrpl::NodeStore::Database::getCountsJson : valid input type"); + XRPL_ASSERT(obj.isObject(), "xrpl::node_store::Database::getCountsJson : valid input type"); { std::unique_lock const lock(readLock_); @@ -276,4 +276,4 @@ Database::getCountsJson(json::Value& obj) obj[jss::node_reads_duration_us] = std::to_string(fetchDurationUs_); } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/DatabaseNodeImp.cpp b/src/libxrpl/nodestore/DatabaseNodeImp.cpp index 9323d69131..1b880ac658 100644 --- a/src/libxrpl/nodestore/DatabaseNodeImp.cpp +++ b/src/libxrpl/nodestore/DatabaseNodeImp.cpp @@ -15,7 +15,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { void DatabaseNodeImp::store(NodeObjectType type, Blob&& data, uint256 const& hash, std::uint32_t) @@ -125,4 +125,4 @@ DatabaseNodeImp::fetchNodeObject( return nodeObject; } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/DatabaseRotatingImp.cpp b/src/libxrpl/nodestore/DatabaseRotatingImp.cpp index 23a48a3bf3..81b1d6b297 100644 --- a/src/libxrpl/nodestore/DatabaseRotatingImp.cpp +++ b/src/libxrpl/nodestore/DatabaseRotatingImp.cpp @@ -22,7 +22,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { DatabaseRotatingImp::DatabaseRotatingImp( Scheduler& scheduler, @@ -43,7 +43,7 @@ DatabaseRotatingImp::DatabaseRotatingImp( void DatabaseRotatingImp::rotate( - std::unique_ptr&& newBackend, + std::unique_ptr&& newBackend, std::function const& f) { // Pass these two names to the callback function @@ -52,7 +52,7 @@ DatabaseRotatingImp::rotate( // Hold on to current archive backend pointer until after the // callback finishes. Only then will the archive directory be // deleted. - std::shared_ptr oldArchiveBackend; + std::shared_ptr oldArchiveBackend; std::uint64_t copyForwards = 0; { std::scoped_lock const lock(mutex_); @@ -232,4 +232,4 @@ DatabaseRotatingImp::forEach(std::function)> f) archive->forEach(f); } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/DecodedBlob.cpp b/src/libxrpl/nodestore/DecodedBlob.cpp index 9740462ae8..321089d40f 100644 --- a/src/libxrpl/nodestore/DecodedBlob.cpp +++ b/src/libxrpl/nodestore/DecodedBlob.cpp @@ -10,7 +10,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { DecodedBlob::DecodedBlob(void const* key, void const* value, int valueBytes) : key_(key) { @@ -55,7 +55,7 @@ DecodedBlob::DecodedBlob(void const* key, void const* value, int valueBytes) : k std::shared_ptr DecodedBlob::createObject() { - XRPL_ASSERT(success_, "xrpl::NodeStore::DecodedBlob::createObject : valid object type"); + XRPL_ASSERT(success_, "xrpl::node_store::DecodedBlob::createObject : valid object type"); std::shared_ptr object; @@ -69,4 +69,4 @@ DecodedBlob::createObject() return object; } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/DummyScheduler.cpp b/src/libxrpl/nodestore/DummyScheduler.cpp index 1f93ed3d0f..32cd14cbdc 100644 --- a/src/libxrpl/nodestore/DummyScheduler.cpp +++ b/src/libxrpl/nodestore/DummyScheduler.cpp @@ -3,7 +3,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { void DummyScheduler::scheduleTask(Task& task) @@ -22,4 +22,4 @@ DummyScheduler::onBatchWrite(BatchWriteReport const& report) { } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/ManagerImp.cpp b/src/libxrpl/nodestore/ManagerImp.cpp index a3db22ce74..c78ccd5761 100644 --- a/src/libxrpl/nodestore/ManagerImp.cpp +++ b/src/libxrpl/nodestore/ManagerImp.cpp @@ -22,7 +22,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { ManagerImp& ManagerImp::instance() @@ -112,7 +112,7 @@ ManagerImp::erase(Factory& factory) std::scoped_lock const _(mutex_); auto const iter = std::ranges::find_if(list_, [&factory](Factory* other) { return other == &factory; }); - XRPL_ASSERT(iter != list_.end(), "xrpl::NodeStore::ManagerImp::erase : valid input"); + XRPL_ASSERT(iter != list_.end(), "xrpl::node_store::ManagerImp::erase : valid input"); list_.erase(iter); } @@ -135,4 +135,4 @@ Manager::instance() return ManagerImp::instance(); } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/backend/MemoryFactory.cpp b/src/libxrpl/nodestore/backend/MemoryFactory.cpp index 22557d652e..39d2123bc9 100644 --- a/src/libxrpl/nodestore/backend/MemoryFactory.cpp +++ b/src/libxrpl/nodestore/backend/MemoryFactory.cpp @@ -24,7 +24,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { struct MemoryDB { @@ -132,7 +132,7 @@ public: Status fetch(uint256 const& hash, std::shared_ptr* pObject) override { - XRPL_ASSERT(db_, "xrpl::NodeStore::MemoryBackend::fetch : non-null database"); + XRPL_ASSERT(db_, "xrpl::node_store::MemoryBackend::fetch : non-null database"); std::scoped_lock const _(db_->mutex); @@ -149,7 +149,7 @@ public: void store(std::shared_ptr const& object) override { - XRPL_ASSERT(db_, "xrpl::NodeStore::MemoryBackend::store : non-null database"); + XRPL_ASSERT(db_, "xrpl::node_store::MemoryBackend::store : non-null database"); std::scoped_lock const _(db_->mutex); db_->table.emplace(object->getHash(), object); } @@ -169,7 +169,7 @@ public: void forEach(std::function)> f) override { - XRPL_ASSERT(db_, "xrpl::NodeStore::MemoryBackend::forEach : non-null database"); + XRPL_ASSERT(db_, "xrpl::node_store::MemoryBackend::forEach : non-null database"); for (auto const& e : db_->table) f(e.second); } @@ -216,4 +216,4 @@ MemoryFactory::createInstance( return std::make_unique(keyBytes, keyValues, journal); } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/backend/NuDBFactory.cpp b/src/libxrpl/nodestore/backend/NuDBFactory.cpp index 38ea34258f..bbf37f3edf 100644 --- a/src/libxrpl/nodestore/backend/NuDBFactory.cpp +++ b/src/libxrpl/nodestore/backend/NuDBFactory.cpp @@ -44,7 +44,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { class NuDBBackend : public Backend { @@ -136,7 +136,7 @@ public: { // LCOV_EXCL_START UNREACHABLE( - "xrpl::NodeStore::NuDBBackend::open : database is already " + "xrpl::node_store::NuDBBackend::open : database is already " "open"); JLOG(j.error()) << "database is already open"; return; @@ -441,4 +441,4 @@ registerNuDBFactory(Manager& manager) static NuDBFactory const kInstance{manager}; } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/backend/NullFactory.cpp b/src/libxrpl/nodestore/backend/NullFactory.cpp index 0c76cb9938..feef3d37d0 100644 --- a/src/libxrpl/nodestore/backend/NullFactory.cpp +++ b/src/libxrpl/nodestore/backend/NullFactory.cpp @@ -13,7 +13,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { class NullBackend : public Backend { @@ -125,4 +125,4 @@ registerNullFactory(Manager& manager) static NullFactory const kInstance{manager}; } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/libxrpl/nodestore/backend/RocksDBFactory.cpp b/src/libxrpl/nodestore/backend/RocksDBFactory.cpp index 673b0daae0..4b7a1171fe 100644 --- a/src/libxrpl/nodestore/backend/RocksDBFactory.cpp +++ b/src/libxrpl/nodestore/backend/RocksDBFactory.cpp @@ -42,7 +42,7 @@ #include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { class RocksDBEnv : public rocksdb::EnvWrapper { @@ -231,7 +231,7 @@ public: { // LCOV_EXCL_START UNREACHABLE( - "xrpl::NodeStore::RocksDBBackend::open : database is already " + "xrpl::node_store::RocksDBBackend::open : database is already " "open"); JLOG(journal.error()) << "database is already open"; return; @@ -279,7 +279,7 @@ public: Status fetch(uint256 const& hash, std::shared_ptr* pObject) override { - XRPL_ASSERT(db, "xrpl::NodeStore::RocksDBBackend::fetch : non-null database"); + XRPL_ASSERT(db, "xrpl::node_store::RocksDBBackend::fetch : non-null database"); pObject->reset(); Status status = Status::Ok; @@ -339,7 +339,7 @@ public: { XRPL_ASSERT( db, - "xrpl::NodeStore::RocksDBBackend::storeBatch : non-null " + "xrpl::node_store::RocksDBBackend::storeBatch : non-null " "database"); rocksdb::WriteBatch wb; @@ -369,7 +369,7 @@ public: void forEach(std::function)> f) override { - XRPL_ASSERT(db, "xrpl::NodeStore::RocksDBBackend::forEach : non-null database"); + XRPL_ASSERT(db, "xrpl::node_store::RocksDBBackend::forEach : non-null database"); rocksdb::ReadOptions const options; std::unique_ptr it(db->NewIterator(options)); @@ -468,6 +468,6 @@ registerRocksDBFactory(Manager& manager) static RocksDBFactory const kInstance{manager}; } -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store #endif diff --git a/src/xrpld/peerfinder/detail/Bootcache.cpp b/src/libxrpl/peerfinder/Bootcache.cpp similarity index 80% rename from src/xrpld/peerfinder/detail/Bootcache.cpp rename to src/libxrpl/peerfinder/Bootcache.cpp index a0a753530b..a2a56b4d01 100644 --- a/src/xrpld/peerfinder/detail/Bootcache.cpp +++ b/src/libxrpl/peerfinder/Bootcache.cpp @@ -1,19 +1,19 @@ -#include - -#include -#include -#include -#include +#include #include #include #include #include #include +#include +#include +#include #include #include #include +#include +#include #include namespace xrpl::PeerFinder { @@ -82,13 +82,14 @@ Bootcache::load() auto const result(this->map_.insert(value_type(endpoint, valence))); if (!result.second) { - JLOG(this->journal_.error()) << beast::Leftw(18) << "Bootcache discard " << endpoint; + JLOG(this->journal_.error()) + << std::left << std::setw(18) << "Bootcache discard " << endpoint; } })); if (n > 0) { - JLOG(journal_.info()) << beast::Leftw(18) << "Bootcache loaded " << n + JLOG(journal_.info()) << std::left << std::setw(18) << "Bootcache loaded " << n << ((n > 1) ? " addresses" : " address"); prune(); } @@ -100,7 +101,7 @@ Bootcache::insert(beast::IP::Endpoint const& endpoint) auto const result(map_.insert(value_type(endpoint, 0))); if (result.second) { - JLOG(journal_.trace()) << beast::Leftw(18) << "Bootcache insert " << endpoint; + JLOG(journal_.trace()) << std::left << std::setw(18) << "Bootcache insert " << endpoint; prune(); flagForUpdate(); } @@ -121,7 +122,7 @@ Bootcache::insertStatic(beast::IP::Endpoint const& endpoint) if (result.second) { - JLOG(journal_.trace()) << beast::Leftw(18) << "Bootcache insert " << endpoint; + JLOG(journal_.trace()) << std::left << std::setw(18) << "Bootcache insert " << endpoint; prune(); flagForUpdate(); } @@ -146,8 +147,9 @@ Bootcache::onSuccess(beast::IP::Endpoint const& endpoint) XRPL_ASSERT(result.second, "xrpl::PeerFinder::Bootcache::onSuccess : endpoint inserted"); } Entry const& entry(result.first->right); - JLOG(journal_.info()) << beast::Leftw(18) << "Bootcache connect " << endpoint << " with " - << entry.valence() << ((entry.valence() > 1) ? " successes" : " success"); + JLOG(journal_.info()) << std::left << std::setw(18) << "Bootcache connect " << endpoint + << " with " << entry.valence() + << ((entry.valence() > 1) ? " successes" : " success"); flagForUpdate(); } @@ -170,8 +172,8 @@ Bootcache::onFailure(beast::IP::Endpoint const& endpoint) } Entry const& entry(result.first->right); auto const n(std::abs(entry.valence())); - JLOG(journal_.debug()) << beast::Leftw(18) << "Bootcache failed " << endpoint << " with " << n - << ((n > 1) ? " attempts" : " attempt"); + JLOG(journal_.debug()) << std::left << std::setw(18) << "Bootcache failed " << endpoint + << " with " << n << ((n > 1) ? " attempts" : " attempt"); flagForUpdate(); } @@ -209,17 +211,19 @@ Bootcache::prune() // Work backwards because bimap doesn't handle // erasing using a reverse iterator very well. // - for (auto iter(map_.right.end()); count-- > 0 && iter != map_.right.begin(); ++pruned) + for (auto iter(map_.right.end()); count > 0 && iter != map_.right.begin(); ++pruned) { + --count; --iter; beast::IP::Endpoint const& endpoint(iter->get_left()); Entry const& entry(iter->get_right()); - JLOG(journal_.trace()) << beast::Leftw(18) << "Bootcache pruned" << endpoint + JLOG(journal_.trace()) << std::left << std::setw(18) << "Bootcache pruned" << endpoint << " at valence " << entry.valence(); iter = map_.right.erase(iter); } - JLOG(journal_.debug()) << beast::Leftw(18) << "Bootcache pruned " << pruned << " entries total"; + JLOG(journal_.debug()) << std::left << std::setw(18) << "Bootcache pruned " << pruned + << " entries total"; } // Updates the Store with the current set of entries if needed. diff --git a/src/libxrpl/peerfinder/Config.cpp b/src/libxrpl/peerfinder/Config.cpp new file mode 100644 index 0000000000..3e2f74f42b --- /dev/null +++ b/src/libxrpl/peerfinder/Config.cpp @@ -0,0 +1,135 @@ +#include + +#include + +#include +#include +#include +#include + +namespace xrpl::PeerFinder { + +std::size_t +Config::calcOutPeers() const +{ + return std::max( + ((maxPeers * Tuning::kOutPercent) + 50) / 100, std::size_t(Tuning::kMinOutCount)); +} + +void +Config::applyTuning() +{ + if (ipLimit == 0) + { + // Unless a limit is explicitly set, we allow between + // 2 and 5 connections from non RFC-1918 "private" + // IP addresses. + ipLimit = 2; + + if (inPeers > Tuning::kDefaultMaxPeers) + ipLimit += std::min(5, static_cast(inPeers / Tuning::kDefaultMaxPeers)); + } + + // We don't allow a single IP to consume all incoming slots, + // unless we only have one incoming slot available. + ipLimit = std::max(1, std::min(ipLimit, static_cast(inPeers / 2))); +} + +void +Config::onWrite(beast::PropertyStream::Map& map) const +{ + map["max_peers"] = maxPeers; + map["out_peers"] = outPeers; + map["want_incoming"] = wantIncoming; + map["auto_connect"] = autoConnect; + map["port"] = listeningPort; + map["features"] = features; + map["ip_limit"] = ipLimit; + map["verify_endpoints"] = verifyEndpoints; +} + +Config +Config::makeConfig( + bool peerPrivate, + bool standalone, + PeerLimitConfig const& limits, + std::uint16_t port, + bool validationPublicKey, + int ipLimit, + bool verifyEndpoints) +{ + PeerFinder::Config config; + + if (!limits.maxPeers) + { + if (limits.inPeers && !limits.outPeers) + throw std::runtime_error("Both inbound and outbound peer limits must be configured"); + + if (limits.outPeers && !limits.inPeers) + throw std::runtime_error("Both inbound and outbound peer limits must be configured"); + + if (limits.inPeers && *limits.inPeers > 1000) + throw std::runtime_error("Inbound peer limit must be less than or equal to 1000"); + + if (limits.outPeers && (*limits.outPeers < 10 || *limits.outPeers > 1000)) + throw std::runtime_error("Outbound peer limit must be in the range 10-1000"); + } + + config.peerPrivate = peerPrivate; + + // Servers with peer privacy don't want to allow incoming connections + config.wantIncoming = (!config.peerPrivate) && (port != 0); + + if (limits.maxPeers || (!limits.inPeers && !limits.outPeers)) + { + if (limits.maxPeers && *limits.maxPeers != 0) + config.maxPeers = *limits.maxPeers; + + config.maxPeers = std::max(config.maxPeers, Tuning::kMinOutCount); + config.outPeers = config.calcOutPeers(); + + // Calculate the number of outbound peers we want. If we dont want + // or can't accept incoming, this will simply be equal to maxPeers. + if (!config.wantIncoming) + config.outPeers = config.maxPeers; + + // Calculate the largest number of inbound connections we could + // take. + if (config.maxPeers >= config.outPeers) + { + config.inPeers = config.maxPeers - config.outPeers; + } + else + { + config.inPeers = 0; + } + } + else + { + config.outPeers = *limits.outPeers; + config.inPeers = *limits.inPeers; + config.maxPeers = 0; + } + + // This will cause servers configured as validators to request that + // peers they connect to never report their IP address. We set this + // after we set the 'wantIncoming' because we want a "soft" version + // of peer privacy unless the operator explicitly asks for it. + if (validationPublicKey) + config.peerPrivate = true; + + // if it's a private peer or we are running as standalone + // automatic connections would defeat the purpose. + config.autoConnect = !standalone && !peerPrivate; + config.listeningPort = port; + config.features = ""; + config.ipLimit = ipLimit; + config.verifyEndpoints = verifyEndpoints; + + // Enforce business rules + config.applyTuning(); + + return config; +} + +} // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/Endpoint.cpp b/src/libxrpl/peerfinder/Endpoint.cpp similarity index 76% rename from src/xrpld/peerfinder/detail/Endpoint.cpp rename to src/libxrpl/peerfinder/Endpoint.cpp index 15de5cd153..12f2725ea5 100644 --- a/src/xrpld/peerfinder/detail/Endpoint.cpp +++ b/src/libxrpl/peerfinder/Endpoint.cpp @@ -1,7 +1,5 @@ -#include -#include - #include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/PeerfinderManager.cpp b/src/libxrpl/peerfinder/PeerfinderManager.cpp similarity index 92% rename from src/xrpld/peerfinder/detail/PeerfinderManager.cpp rename to src/libxrpl/peerfinder/PeerfinderManager.cpp index 2727a03013..2219627f09 100644 --- a/src/xrpld/peerfinder/detail/PeerfinderManager.cpp +++ b/src/libxrpl/peerfinder/PeerfinderManager.cpp @@ -1,11 +1,4 @@ -#include - -#include -#include -#include -#include -#include -#include +#include #include #include @@ -13,7 +6,15 @@ #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -38,10 +39,9 @@ public: std::optional> work_; clock_type& clock_; beast::Journal journal_; - StoreSqdb store_; + Store& store_; Checker checker_; Logic logic_; - BasicConfig const& config_; // NOLINTEND(readability-identifier-naming) //-------------------------------------------------------------------------- @@ -50,16 +50,15 @@ public: boost::asio::io_context& ioContext, clock_type& clock, beast::Journal journal, - BasicConfig const& config, + Store& store, beast::insight::Collector::ptr const& collector) : io_context_(ioContext) , work_(std::in_place, boost::asio::make_work_guard(io_context_)) , clock_(clock) , journal_(journal) - , store_(journal) + , store_(store) , checker_(io_context_) , logic_(clock, store_, checker_, journal) - , config_(config) , stats_([this] { collectMetrics(); }, collector) { } @@ -206,7 +205,6 @@ public: void start() override { - store_.open(config_); logic_.load(); } @@ -261,10 +259,10 @@ makeManager( boost::asio::io_context& ioContext, clock_type& clock, beast::Journal journal, - BasicConfig const& config, + Store& store, beast::insight::Collector::ptr const& collector) { - return std::make_unique(ioContext, clock, journal, config, collector); + return std::make_unique(ioContext, clock, journal, store, collector); } } // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/SlotImp.cpp b/src/libxrpl/peerfinder/SlotImp.cpp similarity index 93% rename from src/xrpld/peerfinder/detail/SlotImp.cpp rename to src/libxrpl/peerfinder/SlotImp.cpp index a54ddb56e7..0a4f32fd62 100644 --- a/src/xrpld/peerfinder/detail/SlotImp.cpp +++ b/src/libxrpl/peerfinder/SlotImp.cpp @@ -1,12 +1,10 @@ -#include +#include -#include -#include -#include - -#include #include #include +#include +#include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/SourceStrings.cpp b/src/libxrpl/peerfinder/SourceStrings.cpp similarity index 93% rename from src/xrpld/peerfinder/detail/SourceStrings.cpp rename to src/libxrpl/peerfinder/SourceStrings.cpp index 7b28db4306..f47e0cd51d 100644 --- a/src/xrpld/peerfinder/detail/SourceStrings.cpp +++ b/src/libxrpl/peerfinder/SourceStrings.cpp @@ -1,9 +1,8 @@ -#include - -#include +#include #include #include +#include #include #include diff --git a/src/libxrpl/protocol/STObject.cpp b/src/libxrpl/protocol/STObject.cpp index 4b3ace2be3..c2543d2cca 100644 --- a/src/libxrpl/protocol/STObject.cpp +++ b/src/libxrpl/protocol/STObject.cpp @@ -899,6 +899,10 @@ STObject::add(Serializer& s, WhichFields whichFields) const XRPL_ASSERT( (sType != STI_OBJECT) || (field->getFName().fieldType == STI_OBJECT), "xrpl::STObject::add : valid field type"); + XRPL_ASSERT( + getStyle(field->getFName()) != SoeDefault || !field->isDefault(), + "xrpl::STObject::add : non-default value"); + field->addFieldID(s); field->add(s); if (sType == STI_ARRAY || sType == STI_OBJECT) diff --git a/src/libxrpl/tx/invariants/PermissionedDEXInvariant.cpp b/src/libxrpl/tx/invariants/PermissionedDEXInvariant.cpp index 1014642b36..44f623f284 100644 --- a/src/libxrpl/tx/invariants/PermissionedDEXInvariant.cpp +++ b/src/libxrpl/tx/invariants/PermissionedDEXInvariant.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -19,17 +20,23 @@ namespace xrpl { void ValidPermissionedDEX::visitEntry(bool isDelete, SLE::const_ref before, SLE::const_ref after) { + auto trackDomain = [this, isDelete](uint256 const& domain) { + domainsOld_.insert(domain); + if (!isDelete) + domains_.insert(domain); + }; + if (after && after->getType() == ltDIR_NODE) { if (after->isFieldPresent(sfDomainID)) - domains_.insert(after->getFieldH256(sfDomainID)); + trackDomain(after->getFieldH256(sfDomainID)); } if (after && after->getType() == ltOFFER) { if (after->isFieldPresent(sfDomainID)) { - domains_.insert(after->getFieldH256(sfDomainID)); + trackDomain(after->getFieldH256(sfDomainID)); } else { @@ -87,7 +94,8 @@ ValidPermissionedDEX::finalize( // for both payment and offercreate, there shouldn't be another domain // that's different from the domain specified - for (auto const& d : domains_) + auto const& domains = view.rules().enabled(fixCleanup3_4_0) ? domains_ : domainsOld_; + for (auto const& d : domains) { if (d != domain) { diff --git a/src/test/app/Batch_test.cpp b/src/test/app/Batch_test.cpp index 5085ad6172..ffaf26b5a7 100644 --- a/src/test/app/Batch_test.cpp +++ b/src/test/app/Batch_test.cpp @@ -498,7 +498,7 @@ class Batch_test : public beast::unit_test::Suite auto const batchFee = batch::calcBatchFee(env, 0, 2); auto tx1 = batch::Inner(pay(alice, bob, XRP(1)), seq + 1); tx1[jss::Fee] = "1.5"; - env.setParseFailureExpected(true); + auto const g = env.getParseFailureGuard(true); try { env(batch::outer(alice, seq, batchFee, tfAllOrNothing), @@ -510,7 +510,6 @@ class Batch_test : public beast::unit_test::Suite { BEAST_EXPECT(true); } - env.setParseFailureExpected(false); } // temSEQ_AND_TICKET: Batch: inner txn cannot have both Sequence diff --git a/src/test/app/Invariants_test.cpp b/src/test/app/Invariants_test.cpp index 076e39a42b..eaf1f2704c 100644 --- a/src/test/app/Invariants_test.cpp +++ b/src/test/app/Invariants_test.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -101,6 +102,12 @@ class Invariants_test : public beast::unit_test::Suite return xrpl::test::jtx::testableAmendments() | fixCleanup3_1_3 | fixCleanup3_2_0; } + test::jtx::Env + makeEnv(FeatureBitset features) + { + return {*this, test::jtx::envconfig(), features, nullptr, beast::Severity::Disabled}; + } + /** * Run a specific test case to put the ledger into a state that will be * detected by an invariant. Simulates the actions of a transaction that @@ -130,7 +137,7 @@ class Invariants_test : public beast::unit_test::Suite TxAccount setTxAccount = TxAccount::None) { doInvariantCheck( - test::jtx::Env(*this, defaultAmendments()), + makeEnv(defaultAmendments()), expectLogs, precheck, fee, @@ -1494,7 +1501,7 @@ class Invariants_test : public beast::unit_test::Suite testcase << "PermissionedDomain" + std::string(fixEnabled ? " fix" : ""); doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"permissioned domain with no rules."}}, [](Account const& a1, Account const& a2, ApplyContext& ac) { return createPermissionedDomain(ac, a1, a2, 0).get(); @@ -1507,7 +1514,7 @@ class Invariants_test : public beast::unit_test::Suite static constexpr auto kTooBig = kMaxPermissionedDomainCredentialsArraySize + 1; doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"permissioned domain bad credentials size " + std::to_string(kTooBig)}}, [](Account const& a1, Account const& a2, ApplyContext& ac) { return !!createPermissionedDomain(ac, a1, a2, kTooBig); @@ -1518,7 +1525,7 @@ class Invariants_test : public beast::unit_test::Suite testcase << "PermissionedDomain 3"; doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"permissioned domain credentials aren't sorted"}}, [](Account const& a1, Account const& a2, ApplyContext& ac) { auto slePd = createPermissionedDomain(ac, a1, a2, 0); @@ -1542,7 +1549,7 @@ class Invariants_test : public beast::unit_test::Suite testcase << "PermissionedDomain 4"; doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"permissioned domain credentials aren't unique"}}, [](Account const& a1, Account const& a2, ApplyContext& ac) { auto slePd = createPermissionedDomain(ac, a1, a2, 0); @@ -1565,7 +1572,7 @@ class Invariants_test : public beast::unit_test::Suite testcase << "PermissionedDomain Set 1"; doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"permissioned domain with no rules."}}, [&](Account const& a1, Account const& a2, ApplyContext& ac) { // create PD @@ -1586,7 +1593,7 @@ class Invariants_test : public beast::unit_test::Suite testcase << "PermissionedDomain Set 2"; doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"permissioned domain bad credentials size " + std::to_string(kTooBig)}}, [&](Account const& a1, Account const& a2, ApplyContext& ac) { // create PD @@ -1617,7 +1624,7 @@ class Invariants_test : public beast::unit_test::Suite testcase << "PermissionedDomain Set 3"; doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"permissioned domain credentials aren't sorted"}}, [&](Account const& a1, Account const& a2, ApplyContext& ac) { // create PD @@ -1647,7 +1654,7 @@ class Invariants_test : public beast::unit_test::Suite testcase << "PermissionedDomain Set 4"; doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"permissioned domain credentials aren't unique"}}, [&](Account const& a1, Account const& a2, ApplyContext& ac) { // create PD @@ -1688,7 +1695,7 @@ class Invariants_test : public beast::unit_test::Suite { testcase << "PermissionedDomain set 2 domains "; doInvariantCheck( - Env(*this, features), + makeEnv(features), fixEnabled ? badMoreThan1 : emptyV, [](Account const& a1, Account const& a2, ApplyContext& ac) { createPermissionedDomain(ac, a1, a2); @@ -1734,7 +1741,7 @@ class Invariants_test : public beast::unit_test::Suite { testcase << "PermissionedDomain set 0 domains "; doInvariantCheck( - Env(*this, features), + makeEnv(features), fixEnabled ? badNoDomains : emptyV, [](Account const&, Account const&, ApplyContext&) { return true; }, XRPAmount{}, @@ -1757,7 +1764,7 @@ class Invariants_test : public beast::unit_test::Suite env1.close(); doInvariantCheck( - Env(*this, features), + makeEnv(features), a1, a2, fixEnabled ? badNoDomains : emptyV, @@ -1798,7 +1805,7 @@ class Invariants_test : public beast::unit_test::Suite { testcase << "PermissionedDomain del, create domain "; doInvariantCheck( - Env(*this, features), + makeEnv(features), fixEnabled ? badNotDeleted : emptyV, [](Account const& a1, Account const& a2, ApplyContext& ac) { createPermissionedDomain(ac, a1, a2); @@ -1995,7 +2002,7 @@ class Invariants_test : public beast::unit_test::Suite testcase << "PermissionedDEX" + std::string(fixEnabled ? " fix" : ""); doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"domain doesn't exist"}}, [](Account const& a1, Account const&, ApplyContext& ac) { Keylet const offerKey = keylet::offer(a1.id(), 10); @@ -2022,7 +2029,7 @@ class Invariants_test : public beast::unit_test::Suite // missing domain ID in offer object doInvariantCheck( - Env(*this, features), + makeEnv(features), {{"hybrid offer is malformed"}}, [&](Account const& a1, Account const& a2, ApplyContext& ac) { Keylet const offerKey = keylet::offer(a2.id(), 10); @@ -4342,7 +4349,7 @@ class Invariants_test : public beast::unit_test::Suite }; doInvariantCheck( - Env{*this, defaultAmendments() - fixCleanup3_2_0}, + makeEnv(defaultAmendments() - fixCleanup3_2_0), {}, [](Account const&, Account const&, ApplyContext&) { return true; }, XRPAmount{}, @@ -5468,7 +5475,7 @@ class Invariants_test : public beast::unit_test::Suite // sfHighLimit issue, not the keylet currency). testcase << "overwrite: NoXRPTrustLines" + std::string(fixEnabled ? " fix" : ""); doInvariantCheck( - Env(*this, features), + makeEnv(features), fixEnabled ? std::vector{{"an XRP trust line was created"}} : std::vector{}, [&insertOrderedTrustLinePair](Account const& a1, Account const& a2, ApplyContext& ac) { @@ -5496,7 +5503,7 @@ class Invariants_test : public beast::unit_test::Suite // Regression: bad deep-freeze trust line followed by a valid one. testcase << "overwrite: NoDeepFreeze" + std::string(fixEnabled ? " fix" : ""); doInvariantCheck( - Env(*this, features), + makeEnv(features), fixEnabled ? std::vector{{"a trust line with deep freeze flag without " "normal freeze was created"}} : std::vector{}, @@ -5530,7 +5537,7 @@ class Invariants_test : public beast::unit_test::Suite // still fires ("a MPT issuance was created"). testcase << "overwrite: NoZeroEscrow MPT" + std::string(fixEnabled ? " fix" : ""); doInvariantCheck( - Env(*this, features), + makeEnv(features), fixEnabled ? std::vector{{"escrow specifies invalid amount"}} : std::vector{{"a MPT issuance was created"}}, [](Account const& a1, Account const&, ApplyContext& ac) { diff --git a/src/test/app/PermissionedDEX_test.cpp b/src/test/app/PermissionedDEX_test.cpp index 998b7b1c7f..67cb7602a0 100644 --- a/src/test/app/PermissionedDEX_test.cpp +++ b/src/test/app/PermissionedDEX_test.cpp @@ -2001,6 +2001,61 @@ class PermissionedDEX_test : public beast::unit_test::Suite } } + void + testReplaceDomainOfferWithOtherDomainOffer(FeatureBitset features) + { + bool const fixEnabled = features[fixCleanup3_4_0]; + + testcase << "Replace domain offer via OfferCreate" + << (fixEnabled ? " (fixCleanup3_4_0 enabled)" : " (fixCleanup3_4_0 disabled)"); + + Env env(*this, features); + auto const& [gw, domainOwner, alice, bob, carol, USD, domainA, credType] = + PermissionedDEX(env); + + Account const domainOwnerB("permdex-domainOwnerB"); + auto const domainB = + setupDomain(env, {alice, bob, carol, gw}, domainOwnerB, "permdex-other-domain"); + BEAST_EXPECT(domainA != domainB); + + auto const oldSeq = env.seq(alice); + env(offer(alice, USD(100), XRP(1)), Domain(domainA)); + env.close(); + + BEAST_EXPECT(checkOffer(env, alice, oldSeq, USD(100), XRP(1), 0, true)); + auto const oldOffer = env.le(keylet::offer(alice.id(), oldSeq)); + if (!BEAST_EXPECT(oldOffer)) + return; + BEAST_EXPECT(oldOffer->getFieldH256(sfDomainID) == domainA); + + auto const newSeq = env.seq(alice); + // The invariant should reject mixing active Permissioned DEX domains, + // not a domain that is only touched because its offer is being deleted. + if (fixEnabled) + { + env(offer(alice, USD(100), XRP(2)), Domain(domainB), Json(jss::OfferSequence, oldSeq)); + env.close(); + + BEAST_EXPECT(!offerExists(env, alice, oldSeq)); + BEAST_EXPECT(checkOffer(env, alice, newSeq, USD(100), XRP(2), 0, true)); + auto const newOffer = env.le(keylet::offer(alice.id(), newSeq)); + if (!BEAST_EXPECT(newOffer)) + return; + BEAST_EXPECT(newOffer->getFieldH256(sfDomainID) == domainB); + } + else + { + env(offer(alice, USD(100), XRP(2)), + Domain(domainB), + Json(jss::OfferSequence, oldSeq), + Ter(tecINVARIANT_FAILED)); + env.close(); + + BEAST_EXPECT(checkOffer(env, alice, oldSeq, USD(100), XRP(1), 0, true)); + BEAST_EXPECT(!offerExists(env, alice, newSeq)); + } + } + public: void run() override @@ -2038,6 +2093,8 @@ public: // only after fixCleanup3_2_0. testCancelRegularOfferWithDomainCreate(all); testCancelRegularOfferWithDomainCreate(all - fixCleanup3_2_0); + testReplaceDomainOfferWithOtherDomainOffer(all); + testReplaceDomainOfferWithOtherDomainOffer(all - fixCleanup3_4_0); } }; diff --git a/src/test/app/RCLValidations_test.cpp b/src/test/app/RCLValidations_test.cpp index aaf84225a7..9ace2e21af 100644 --- a/src/test/app/RCLValidations_test.cpp +++ b/src/test/app/RCLValidations_test.cpp @@ -2,12 +2,12 @@ #include #include -#include #include #include #include #include +#include #include #include #include diff --git a/src/test/app/SHAMapStore_test.cpp b/src/test/app/SHAMapStore_test.cpp index c219bd8737..537ee4c177 100644 --- a/src/test/app/SHAMapStore_test.cpp +++ b/src/test/app/SHAMapStore_test.cpp @@ -489,7 +489,7 @@ public: lastRotated = ledgerSeq - 1; } - std::unique_ptr + std::unique_ptr makeBackendRotating(jtx::Env& env, NodeStoreScheduler& scheduler, std::string path) { Section section{env.app().config().section(Sections::kNodeDatabase)}; @@ -500,7 +500,7 @@ public: newPath = path; section.set(Keys::kPath, newPath.string()); - auto backend{NodeStore::Manager::instance().makeBackend( + auto backend{node_store::Manager::instance().makeBackend( section, megabytes(env.app().config().getValueFor(SizedItem::BurstSize, std::nullopt)), scheduler, @@ -549,7 +549,7 @@ public: auto archiveBackend = makeBackendRotating(env, scheduler, archiveDb); static constexpr int kReadThreads = 4; - auto dbr = std::make_unique( + auto dbr = std::make_unique( scheduler, kReadThreads, std::move(writableBackend), diff --git a/src/test/app/Vault_test.cpp b/src/test/app/Vault_test.cpp index 617820c89c..12ad7e6782 100644 --- a/src/test/app/Vault_test.cpp +++ b/src/test/app/Vault_test.cpp @@ -5524,9 +5524,9 @@ class Vault_test : public beast::unit_test::Suite env.close(); // 2. Mantissa larger than uint64 max - env.setParseFailureExpected(true); try { + auto const g = env.getParseFailureGuard(true); tx[sfAssetsMaximum] = "18446744073709551617e5"; // uint64 max + 1 env(tx); BEAST_EXPECTS(false, "Expected parse_error for mantissa larger than uint64 max"); @@ -5537,7 +5537,6 @@ class Vault_test : public beast::unit_test::Suite BEAST_EXPECT( e.what() == "invalidParamsField 'tx_json.AssetsMaximum' has invalid data."s); } - env.setParseFailureExpected(false); } } diff --git a/src/test/consensus/ByzantineFailureSim_test.cpp b/src/test/consensus/ByzantineFailureSim_test.cpp deleted file mode 100644 index c3c51125b5..0000000000 --- a/src/test/consensus/ByzantineFailureSim_test.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include - -namespace xrpl::test { - -class ByzantineFailureSim_test : public beast::unit_test::Suite -{ - void - run() override - { - using namespace csf; - using namespace std::chrono; - - // This test simulates a specific topology with nodes generating - // different ledgers due to a simulated byzantine failure (injecting - // an extra non-consensus transaction). - - Sim sim; - ConsensusParms const parms{}; - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - PeerGroup a = sim.createGroup(1); - PeerGroup b = sim.createGroup(1); - PeerGroup c = sim.createGroup(1); - PeerGroup d = sim.createGroup(1); - PeerGroup e = sim.createGroup(1); - PeerGroup f = sim.createGroup(1); - PeerGroup g = sim.createGroup(1); - - a.trustAndConnect(a + b + c + g, delay); - b.trustAndConnect(b + a + c + d + e, delay); - c.trustAndConnect(c + a + b + d + e, delay); - d.trustAndConnect(d + b + c + e + f, delay); - e.trustAndConnect(e + b + c + d + f, delay); - f.trustAndConnect(f + d + e + g, delay); - g.trustAndConnect(g + a + f, delay); - - PeerGroup const network = a + b + c + d + e + f + g; - - StreamCollector sc{std::cout}; - - sim.collectors.add(sc); - - for (TrustGraph::ForkInfo const& fi : sim.trustGraph.forkablePairs(0.8)) - { - std::cout << "Can fork " << PeerGroup{fi.unlA} << " " - << " " << PeerGroup{fi.unlB} << " overlap " << fi.overlap << " required " - << fi.required << "\n"; - }; - - // set prior state - sim.run(1); - - PeerGroup byzantineNodes = a + b + c + g; - // All peers see some TX 0 - for (Peer* peer : network) - { - peer->submit(Tx{0}); - // Peers 0,1,2,6 will close the next ledger differently by injecting - // a non-consensus approved transaction - if (byzantineNodes.contains(peer)) - { - peer->txInjections.emplace(peer->lastClosedLedger.seq(), Tx{42}); - } - } - sim.run(4); - std::cout << "Branches: " << sim.branches() << "\n"; - std::cout << "Fully synchronized: " << std::boolalpha << sim.synchronized() << "\n"; - // Not tessting anything currently. - pass(); - } -}; - -BEAST_DEFINE_TESTSUITE_MANUAL(ByzantineFailureSim, consensus, xrpl); - -} // namespace xrpl::test diff --git a/src/test/consensus/Consensus_test.cpp b/src/test/consensus/Consensus_test.cpp deleted file mode 100644 index 45f58d16ba..0000000000 --- a/src/test/consensus/Consensus_test.cpp +++ /dev/null @@ -1,1432 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace xrpl::test { - -class Consensus_test : public beast::unit_test::Suite -{ - SuiteJournal journal_; - -public: - Consensus_test() : journal_("Consensus_test", *this) - { - } - - void - testShouldCloseLedger() - { - using namespace std::chrono_literals; - testcase("should close ledger"); - - // Use default parameters - ConsensusParms const p{}; - - // Bizarre times forcibly close - BEAST_EXPECT(shouldCloseLedger(true, 10, 10, 10, -10s, 10s, 1s, 1s, p, journal_)); - BEAST_EXPECT(shouldCloseLedger(true, 10, 10, 10, 100h, 10s, 1s, 1s, p, journal_)); - BEAST_EXPECT(shouldCloseLedger(true, 10, 10, 10, 10s, 100h, 1s, 1s, p, journal_)); - - // Rest of network has closed - BEAST_EXPECT(shouldCloseLedger(true, 10, 3, 5, 10s, 10s, 10s, 10s, p, journal_)); - - // No transactions means wait until end of internval - BEAST_EXPECT(!shouldCloseLedger(false, 10, 0, 0, 1s, 1s, 1s, 10s, p, journal_)); - BEAST_EXPECT(shouldCloseLedger(false, 10, 0, 0, 1s, 10s, 1s, 10s, p, journal_)); - - // Enforce minimum ledger open time - BEAST_EXPECT(!shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 1s, 10s, p, journal_)); - - // Don't go too much faster than last time - BEAST_EXPECT(!shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 3s, 10s, p, journal_)); - - BEAST_EXPECT(shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 10s, 10s, p, journal_)); - } - - void - testCheckConsensus() - { - using namespace std::chrono_literals; - testcase("check consensus"); - - // Use default parameters - ConsensusParms const p{}; - - /////////////// - // Disputes still in doubt - // - // Not enough time has elapsed - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 2s, false, p, true, journal_)); - - // If not enough peers have proposed, ensure - // more time for proposals - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 4s, false, p, true, journal_)); - - // Enough time has elapsed and we all agree - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 2, 2, 0, 3s, 10s, false, p, true, journal_)); - - // Enough time has elapsed and we don't yet agree - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 1, 0, 3s, 10s, false, p, true, journal_)); - - // Our peers have moved on - // Enough time has elapsed and we all agree - BEAST_EXPECT( - ConsensusState::MovedOn == - checkConsensus(10, 2, 1, 8, 3s, 10s, false, p, true, journal_)); - - // If no peers, don't agree until time has passed. - BEAST_EXPECT( - ConsensusState::No == checkConsensus(0, 0, 0, 0, 3s, 10s, false, p, true, journal_)); - - // Agree if no peers and enough time has passed. - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(0, 0, 0, 0, 3s, 16s, false, p, true, journal_)); - - // Expire if too much time has passed without agreement - BEAST_EXPECT( - ConsensusState::Expired == - checkConsensus(10, 8, 1, 0, 1s, 19s, false, p, true, journal_)); - - /////////////// - // Stalled - // - // Not enough time has elapsed - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 2s, true, p, true, journal_)); - - // If not enough peers have proposed, ensure - // more time for proposals - BEAST_EXPECT( - ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 4s, true, p, true, journal_)); - - // Enough time has elapsed and we all agree - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 2, 2, 0, 3s, 10s, true, p, true, journal_)); - - // Enough time has elapsed and we don't yet agree, but there's nothing - // left to dispute - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 2, 1, 0, 3s, 10s, true, p, true, journal_)); - - // Our peers have moved on - // Enough time has elapsed and we all agree, nothing left to dispute - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 2, 1, 8, 3s, 10s, true, p, true, journal_)); - - // If no peers, don't agree until time has passed. - BEAST_EXPECT( - ConsensusState::No == checkConsensus(0, 0, 0, 0, 3s, 10s, true, p, true, journal_)); - - // Agree if no peers and enough time has passed. - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(0, 0, 0, 0, 3s, 16s, true, p, true, journal_)); - - // We are done if there's nothing left to dispute, no matter how much - // time has passed - BEAST_EXPECT( - ConsensusState::Yes == checkConsensus(10, 8, 1, 0, 1s, 19s, true, p, true, journal_)); - } - - void - testStandalone() - { - using namespace std::chrono_literals; - using namespace csf; - testcase("standalone"); - - Sim s; - PeerGroup const peers = s.createGroup(1); - Peer* peer = peers[0]; - peer->targetLedgers = 1; - peer->start(); - peer->submit(Tx{1}); - - s.scheduler.step(); - - // Inspect that the proper ledger was created - auto const& lcl = peer->lastClosedLedger; - BEAST_EXPECT(peer->prevLedgerID() == lcl.id()); - BEAST_EXPECT(lcl.seq() == Ledger::Seq{1}); - BEAST_EXPECT(lcl.txs().size() == 1); - BEAST_EXPECT(lcl.txs().contains(Tx{1})); - BEAST_EXPECT(peer->prevProposers == 0); - } - - void - testPeersAgree() - { - using namespace csf; - using namespace std::chrono; - testcase("peers agree"); - - ConsensusParms const parms{}; - Sim sim; - PeerGroup peers = sim.createGroup(5); - - // Connected trust and network graphs with single fixed delay - peers.trustAndConnect(peers, round(0.2 * parms.ledgerGRANULARITY)); - - // everyone submits their own ID as a TX - for (Peer* p : peers) - p->submit(Tx(static_cast(p->id))); - - sim.run(1); - - // All peers are in sync - if (BEAST_EXPECT(sim.synchronized())) - { - for (Peer const* peer : peers) - { - auto const& lcl = peer->lastClosedLedger; - BEAST_EXPECT(lcl.id() == peer->prevLedgerID()); - BEAST_EXPECT(lcl.seq() == Ledger::Seq{1}); - // All peers proposed - BEAST_EXPECT(peer->prevProposers == peers.size() - 1); - // All transactions were accepted - for (std::uint32_t i = 0; i < peers.size(); ++i) - BEAST_EXPECT(lcl.txs().contains(Tx{i})); - } - } - } - - void - testSlowPeers() - { - using namespace csf; - using namespace std::chrono; - testcase("slow peers"); - - // Several tests of a complete trust graph with a subset of peers - // that have significantly longer network delays to the rest of the - // network - - // Test when a slow peer doesn't delay a consensus quorum (4/5 agree) - { - ConsensusParms const parms{}; - Sim sim; - PeerGroup slow = sim.createGroup(1); - PeerGroup fast = sim.createGroup(4); - PeerGroup network = fast + slow; - - // Fully connected trust graph - network.trust(network); - - // Fast and slow network connections - fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); - - slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); - - // All peers submit their own ID as a transaction - for (Peer* peer : network) - peer->submit(Tx{static_cast(peer->id)}); - - sim.run(1); - - // Verify all peers have same LCL but are missing transaction 0 - // All peers are in sync even with a slower peer 0 - if (BEAST_EXPECT(sim.synchronized())) - { - for (Peer const* peer : network) - { - auto const& lcl = peer->lastClosedLedger; - BEAST_EXPECT(lcl.id() == peer->prevLedgerID()); - BEAST_EXPECT(lcl.seq() == Ledger::Seq{1}); - - BEAST_EXPECT(peer->prevProposers == network.size() - 1); - BEAST_EXPECT(peer->prevRoundTime == network[0]->prevRoundTime); - - BEAST_EXPECT(not lcl.txs().contains(Tx{0})); - for (std::uint32_t i = 2; i < network.size(); ++i) - BEAST_EXPECT(lcl.txs().contains(Tx{i})); - - // Tx 0 didn't make it - BEAST_EXPECT(peer->openTxs.contains(Tx{0})); - } - } - } - - // Test when the slow peers delay a consensus quorum (4/6 agree) - { - // Run two tests - // 1. The slow peers are participating in consensus - // 2. The slow peers are just observing - - for (auto isParticipant : {true, false}) - { - ConsensusParms const parms{}; - - Sim sim; - PeerGroup slow = sim.createGroup(2); - PeerGroup fast = sim.createGroup(4); - PeerGroup network = fast + slow; - - // Connected trust graph - network.trust(network); - - // Fast and slow network connections - fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); - - slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); - - for (Peer* peer : slow) - peer->runAsValidator = isParticipant; - - // All peers submit their own ID as a transaction and relay it - // to peers - for (Peer* peer : network) - peer->submit(Tx{static_cast(peer->id)}); - - sim.run(1); - - if (BEAST_EXPECT(sim.synchronized())) - { - // Verify all peers have same LCL but are missing - // transaction 0,1 which was not received by all peers - // before the ledger closed - for (Peer const* peer : network) - { - // Closed ledger has all but transaction 0,1 - auto const& lcl = peer->lastClosedLedger; - BEAST_EXPECT(lcl.seq() == Ledger::Seq{1}); - BEAST_EXPECT(not lcl.txs().contains(Tx{0})); - BEAST_EXPECT(not lcl.txs().contains(Tx{1})); - for (std::uint32_t i = slow.size(); i < network.size(); ++i) - BEAST_EXPECT(lcl.txs().contains(Tx{i})); - - // Tx 0-1 didn't make it - BEAST_EXPECT(peer->openTxs.contains(Tx{0})); - BEAST_EXPECT(peer->openTxs.contains(Tx{1})); - } - - Peer const* slowPeer = slow[0]; - if (isParticipant) - { - BEAST_EXPECT(slowPeer->prevProposers == network.size() - 1); - } - else - { - BEAST_EXPECT(slowPeer->prevProposers == fast.size()); - } - - for (Peer const* peer : fast) - { - // Due to the network link delay settings - // Peer 0 initially proposes {0} - // Peer 1 initially proposes {1} - // Peers 2-5 initially propose {2,3,4,5} - // Since peers 2-5 agree, 4/6 > the initial 50% needed - // to include a disputed transaction, so Peer 0/1 switch - // to agree with those peers. Peer 0/1 then closes with - // an 80% quorum of agreeing positions (5/6) match. - // - // Peers 2-5 do not change position, since tx 0 or tx 1 - // have less than the 50% initial threshold. They also - // cannot declare consensus, since 4/6 agreeing - // positions are < 80% threshold. They therefore need an - // additional timerEntry call to see the updated - // positions from Peer 0 & 1. - - if (isParticipant) - { - BEAST_EXPECT(peer->prevProposers == network.size() - 1); - BEAST_EXPECT(peer->prevRoundTime > slowPeer->prevRoundTime); - } - else - { - BEAST_EXPECT(peer->prevProposers == fast.size() - 1); - // so all peers should have closed together - BEAST_EXPECT(peer->prevRoundTime == slowPeer->prevRoundTime); - } - } - } - } - } - } - - void - testCloseTimeDisagree() - { - using namespace csf; - using namespace std::chrono; - testcase("close time disagree"); - - // This is a very specialized test to get ledgers to disagree on - // the close time. It unfortunately assumes knowledge about current - // timing constants. This is a necessary evil to get coverage up - // pending more extensive refactorings of timing constants. - - // In order to agree-to-disagree on the close time, there must be no - // clear majority of nodes agreeing on a close time. This test - // sets a relative offset to the peers internal clocks so that they - // send proposals with differing times. - - // However, agreement is on the effective close time, not the - // exact close time. The minimum closeTimeResolution is given by - // ledgerPossibleTimeResolutions[0], which is currently 10s. This means - // the skews need to be at least 10 seconds to have different effective - // close times. - - // Complicating this matter is that nodes will ignore proposals - // with times more than proposeFRESHNESS =20s in the past. So at - // the minimum granularity, we have at most 3 types of skews - // (0s,10s,20s). - - // This test therefore has 6 nodes, with 2 nodes having each type of - // skew. Then no majority (1/3 < 1/2) of nodes will agree on an - // actual close time. - - ConsensusParms const parms{}; - Sim sim; - - PeerGroup groupA = sim.createGroup(2); - PeerGroup const groupB = sim.createGroup(2); - PeerGroup const groupC = sim.createGroup(2); - PeerGroup network = groupA + groupB + groupC; - - network.trust(network); - network.connect(network, round(0.2 * parms.ledgerGRANULARITY)); - - // Run consensus without skew until we have a short close time - // resolution - Peer const* firstPeer = *groupA.begin(); - while (firstPeer->lastClosedLedger.closeTimeResolution() >= parms.proposeFRESHNESS) - sim.run(1); - - // Introduce a shift on the time of 2/3 of peers - for (Peer* peer : groupA) - peer->clockSkew = parms.proposeFRESHNESS / 2; - for (Peer* peer : groupB) - peer->clockSkew = parms.proposeFRESHNESS; - - sim.run(1); - - // All nodes agreed to disagree on the close time - if (BEAST_EXPECT(sim.synchronized())) - { - for (Peer const* peer : network) - BEAST_EXPECT(!peer->lastClosedLedger.closeAgree()); - } - } - - void - testWrongLCL() - { - using namespace csf; - using namespace std::chrono; - testcase("wrong LCL"); - - // Specialized test to exercise a temporary fork in which some peers - // are working on an incorrect prior ledger. - - ConsensusParms const parms{}; - - // Vary the time it takes to process validations to exercise detecting - // the wrong LCL at different phases of consensus - for (auto validationDelay : {0ms, parms.ledgerMinClose}) - { - // Consider 10 peers: - // 0 1 2 3 4 5 6 7 8 9 - // minority majorityA majorityB - // - // Nodes 0-1 trust nodes 0-4 - // Nodes 2-9 trust nodes 2-9 - // - // By submitting tx 0 to nodes 0-4 and tx 1 to nodes 5-9, - // nodes 0-1 will generate the wrong LCL (with tx 0). The remaining - // nodes will instead accept the ledger with tx 1. - - // Nodes 0-1 will detect this mismatch during a subsequent round - // since nodes 2-4 will validate a different ledger. - - // Nodes 0-1 will acquire the proper ledger from the network and - // resume consensus and eventually generate the dominant network - // ledger. - - // This topology can potentially fork with the above trust relations - // but that is intended for this test. - - Sim sim; - - PeerGroup minority = sim.createGroup(2); - PeerGroup const majorityA = sim.createGroup(3); - PeerGroup const majorityB = sim.createGroup(5); - - PeerGroup majority = majorityA + majorityB; - PeerGroup const network = minority + majority; - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - minority.trustAndConnect(minority + majorityA, delay); - majority.trustAndConnect(majority, delay); - - CollectByNode jumps; - sim.collectors.add(jumps); - - BEAST_EXPECT(sim.trustGraph.canFork(parms.minConsensusPct / 100.)); - - // initial round to set prior state - sim.run(1); - - // Nodes in smaller UNL have seen tx 0, nodes in other unl have seen - // tx 1 - for (Peer* peer : network) - peer->delays.recvValidation = validationDelay; - for (Peer* peer : (minority + majorityA)) - peer->openTxs.insert(Tx{0}); - for (Peer* peer : majorityB) - peer->openTxs.insert(Tx{1}); - - // Run for additional rounds - // With no validation delay, only 2 more rounds are needed. - // 1. Round to generate different ledgers - // 2. Round to detect different prior ledgers (but still generate - // wrong ones) and recover within that round since wrong LCL - // is detected before we close - // - // With a validation delay of ledgerMIN_CLOSE, we need 3 more - // rounds. - // 1. Round to generate different ledgers - // 2. Round to detect different prior ledgers (but still generate - // wrong ones) but end up declaring consensus on wrong LCL (but - // with the right transaction set!). This is because we detect - // the wrong LCL after we have closed the ledger, so we declare - // consensus based solely on our peer proposals. But we haven't - // had time to acquire the right ledger. - // 3. Round to correct - sim.run(3); - - // The network never actually forks, since node 0-1 never see a - // quorum of validations to fully validate the incorrect chain. - - // However, for a non zero-validation delay, the network is not - // synchronized because nodes 0 and 1 are running one ledger behind - if (BEAST_EXPECT(sim.branches() == 1)) - { - for (Peer const* peer : majority) - { - // No jumps for majority nodes - BEAST_EXPECT(jumps[peer->id].closeJumps.empty()); - BEAST_EXPECT(jumps[peer->id].fullyValidatedJumps.empty()); - } - for (Peer const* peer : minority) - { - auto& peerJumps = jumps[peer->id]; - // last closed ledger jump between chains - { - if (BEAST_EXPECT(peerJumps.closeJumps.size() == 1)) - { - JumpCollector::Jump const& jump = peerJumps.closeJumps.front(); - // Jump is to a different chain - BEAST_EXPECT(jump.from.seq() <= jump.to.seq()); - BEAST_EXPECT(!jump.to.isAncestor(jump.from)); - } - } - // fully validated jump forward in same chain - { - if (BEAST_EXPECT(peerJumps.fullyValidatedJumps.size() == 1)) - { - JumpCollector::Jump const& jump = peerJumps.fullyValidatedJumps.front(); - // Jump is to a different chain with same seq - BEAST_EXPECT(jump.from.seq() < jump.to.seq()); - BEAST_EXPECT(jump.to.isAncestor(jump.from)); - } - } - } - } - } - - { - // Additional test engineered to switch LCL during the establish - // phase. This was added to trigger a scenario that previously - // crashed, in which switchLCL switched from establish to open - // phase, but still processed the establish phase logic. - - // Loner node will accept an initial ledger A, but all other nodes - // accept ledger B a bit later. By delaying the time it takes - // to process a validation, loner node will detect the wrongLCL - // after it is already in the establish phase of the next round. - - Sim sim; - PeerGroup loner = sim.createGroup(1); - PeerGroup const friends = sim.createGroup(3); - loner.trust(loner + friends); - - PeerGroup const others = sim.createGroup(6); - PeerGroup clique = friends + others; - clique.trust(clique); - - PeerGroup network = loner + clique; - network.connect(network, round(0.2 * parms.ledgerGRANULARITY)); - - // initial round to set prior state - sim.run(1); - for (Peer* peer : (loner + friends)) - peer->openTxs.insert(Tx(0)); - for (Peer* peer : others) - peer->openTxs.insert(Tx(1)); - - // Delay validation processing - for (Peer* peer : network) - peer->delays.recvValidation = parms.ledgerGRANULARITY; - - // additional rounds to generate wrongLCL and recover - sim.run(2); - - // Check all peers recovered - for (Peer const* p : network) - BEAST_EXPECT(p->prevLedgerID() == network[0]->prevLedgerID()); - } - } - - void - testConsensusCloseTimeRounding() - { - using namespace csf; - using namespace std::chrono; - testcase("consensus close time rounding"); - - // This is a specialized test engineered to yield ledgers with different - // close times even though the peers believe they had close time - // consensus on the ledger. - ConsensusParms const parms; - - Sim sim; - - // This requires a group of 4 fast and 2 slow peers to create a - // situation in which a subset of peers requires seeing additional - // proposals to declare consensus. - PeerGroup slow = sim.createGroup(2); - PeerGroup fast = sim.createGroup(4); - PeerGroup network = fast + slow; - - // Connected trust graph - network.trust(network); - - // Fast and slow network connections - fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); - slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); - - // Run to the ledger *prior* to decreasing the resolution - sim.run(kIncreaseLedgerTimeResolutionEvery - 2); - - // In order to create the discrepancy, we want a case where if - // X = effCloseTime(closeTime, resolution, parentCloseTime) - // X != effCloseTime(X, resolution, parentCloseTime) - // - // That is, the effective close time is not a fixed point. This can - // happen if X = parentCloseTime + 1, but a subsequent rounding goes - // to the next highest multiple of resolution. - - // So we want to find an offset (now + offset) % 30s = 15 - // (now + offset) % 20s = 15 - // This way, the next ledger will close and round up Due to the - // network delay settings, the round of consensus will take 5s, so - // the next ledger's close time will - - NetClock::duration when = network[0]->now().time_since_epoch(); - - // Check we are before the 30s to 20s transition - NetClock::duration const resolution = network[0]->lastClosedLedger.closeTimeResolution(); - BEAST_EXPECT(resolution == NetClock::duration{30s}); - - while (((when % NetClock::duration{30s}) != NetClock::duration{15s}) || - ((when % NetClock::duration{20s}) != NetClock::duration{15s})) - when += 1s; - // Advance the clock without consensus running (IS THIS WHAT - // PREVENTS IT IN PRACTICE?) - sim.scheduler.stepFor(NetClock::time_point{when} - network[0]->now()); - - // Run one more ledger with 30s resolution - sim.run(1); - if (BEAST_EXPECT(sim.synchronized())) - { - // close time should be ahead of clock time since we engineered - // the close time to round up - for (Peer const* peer : network) - { - BEAST_EXPECT(peer->lastClosedLedger.closeTime() > peer->now()); - BEAST_EXPECT(peer->lastClosedLedger.closeAgree()); - } - } - - // All peers submit their own ID as a transaction - for (Peer* peer : network) - peer->submit(Tx{static_cast(peer->id)}); - - // Run 1 more round, this time it will have a decreased - // resolution of 20 seconds. - - // The network delays are engineered so that the slow peers - // initially have the wrong tx hash, but they see a majority - // of agreement from their peers and declare consensus - // - // The trick is that everyone starts with a raw close time of - // 84681s - // Which has - // effCloseTime(86481s, 20s, 86490s) = 86491s - // However, when the slow peers update their position, they change - // the close time to 86451s. The fast peers declare consensus with - // the 86481s as their position still. - // - // When accepted the ledger - // - fast peers use eff(86481s) -> 86491s as the close time - // - slow peers use eff(eff(86481s)) -> eff(86491s) -> 86500s! - - sim.run(1); - - BEAST_EXPECT(sim.synchronized()); - } - - void - testFork() - { - using namespace csf; - using namespace std::chrono; - testcase("fork"); - - std::uint32_t const numPeers = 10; - // Vary overlap between two UNLs - for (std::uint32_t overlap = 0; overlap <= numPeers; ++overlap) - { - ConsensusParms const parms{}; - Sim sim; - - std::uint32_t const numA = (numPeers - overlap) / 2; - std::uint32_t const numB = numPeers - numA - overlap; - - PeerGroup const aOnly = sim.createGroup(numA); - PeerGroup const bOnly = sim.createGroup(numB); - PeerGroup const commonOnly = sim.createGroup(overlap); - - PeerGroup a = aOnly + commonOnly; - PeerGroup b = bOnly + commonOnly; - - PeerGroup const network = a + b; - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - a.trustAndConnect(a, delay); - b.trustAndConnect(b, delay); - - // Initial round to set prior state - sim.run(1); - for (Peer* peer : network) - { - // Nodes have only seen transactions from their neighbors - peer->openTxs.insert(Tx{static_cast(peer->id)}); - for (Peer const* to : sim.trustGraph.trustedPeers(peer)) - peer->openTxs.insert(Tx{static_cast(to->id)}); - } - sim.run(1); - - // Fork should not happen for 40% or greater overlap - // Since the overlapped nodes have a UNL that is the union of the - // two cliques, the maximum sized UNL list is the number of peers - if (overlap > 0.4 * numPeers) - { - BEAST_EXPECT(sim.synchronized()); - } - else - { - // Even if we do fork, there shouldn't be more than 3 ledgers - // One for cliqueA, one for cliqueB and one for nodes in both - BEAST_EXPECT(sim.branches() <= 3); - } - } - } - - void - testHubNetwork() - { - using namespace csf; - using namespace std::chrono; - testcase("hub network"); - - // Simulate a set of 5 validators that aren't directly connected but - // rely on a single hub node for communication - - ConsensusParms const parms{}; - Sim sim; - PeerGroup validators = sim.createGroup(5); - PeerGroup center = sim.createGroup(1); - validators.trust(validators); - center.trust(validators); - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - validators.connect(center, delay); - - center[0]->runAsValidator = false; - - // prep round to set initial state. - sim.run(1); - - // everyone submits their own ID as a TX and relay it to peers - for (Peer* p : validators) - p->submit(Tx(static_cast(p->id))); - - sim.run(1); - - // All peers are in sync - BEAST_EXPECT(sim.synchronized()); - } - - // Helper collector for testPreferredByBranch - // Invasively disconnects network at bad times to cause splits - struct Disruptor - { - csf::PeerGroup& network; - csf::PeerGroup& groupCfast; - csf::PeerGroup& groupCsplit; - csf::SimDuration delay; - bool reconnected = false; - - Disruptor(csf::PeerGroup& net, csf::PeerGroup& c, csf::PeerGroup& split, csf::SimDuration d) - : network(net), groupCfast(c), groupCsplit(split), delay(d) - { - } - - template - void - on(csf::PeerID, csf::SimTime, E const&) - { - } - - void - on(csf::PeerID who, csf::SimTime, csf::FullyValidateLedger const& e) - { - using namespace std::chrono; - // As soon as the fastC node fully validates C, disconnect - // ALL c nodes from the network. The fast C node needs to disconnect - // as well to prevent it from relaying the validations it did see - if (who == groupCfast[0]->id && e.ledger.seq() == csf::Ledger::Seq{2}) - { - network.disconnect(groupCsplit); - network.disconnect(groupCfast); - } - } - - void - on(csf::PeerID who, csf::SimTime, csf::AcceptLedger const& e) - { - // As soon as anyone generates a child of B or C, reconnect the - // network so those validations make it through - if (!reconnected && e.ledger.seq() == csf::Ledger::Seq{3}) - { - reconnected = true; - network.connect(groupCsplit, delay); - } - } - }; - - void - testPreferredByBranch() - { - using namespace csf; - using namespace std::chrono; - testcase("preferred by branch"); - - // Simulate network splits that are prevented from forking when using - // preferred ledger by trie. This is a contrived example that involves - // excessive network splits, but demonstrates the safety improvement - // from the preferred ledger by trie approach. - - // Consider 10 validating nodes that comprise a single common UNL - // Ledger history: - // 1: A - // _/ \_ - // 2: B C - // _/ _/ \_ - // 3: D C' |||||||| (8 different ledgers) - - // - All nodes generate the common ledger A - // - 2 nodes generate B and 8 nodes generate C - // - Only 1 of the C nodes sees all the C validations and fully - // validates C. The rest of the C nodes split at just the right time - // such that they never see any C validations but their own. - // - The C nodes continue and generate 8 different child ledgers. - // - Meanwhile, the D nodes only saw 1 validation for C and 2 - // validations - // for B. - // - The network reconnects and the validations for generation 3 ledgers - // are observed (D and the 8 C's) - // - In the old approach, 2 votes for D outweighs 1 vote for each C' - // so the network would avalanche towards D and fully validate it - // EVEN though C was fully validated by one node - // - In the new approach, 2 votes for D are not enough to outweight the - // 8 implicit votes for C, so nodes will avalanche to C instead - - ConsensusParms const parms{}; - Sim sim; - - // Goes A->B->D - PeerGroup const groupABD = sim.createGroup(2); - // Single node that initially fully validates C before the split - PeerGroup groupCfast = sim.createGroup(1); - // Generates C, but fails to fully validate before the split - PeerGroup groupCsplit = sim.createGroup(7); - - PeerGroup groupNotFastC = groupABD + groupCsplit; - PeerGroup network = groupABD + groupCsplit + groupCfast; - - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - SimDuration const fDelay = round(0.1 * parms.ledgerGRANULARITY); - - network.trust(network); - // C must have a shorter delay to see all the validations before the - // other nodes - network.connect(groupCfast, fDelay); - // The rest of the network is connected at the same speed - groupNotFastC.connect(groupNotFastC, delay); - - Disruptor dc(network, groupCfast, groupCsplit, delay); - sim.collectors.add(dc); - - // Consensus round to generate ledger A - sim.run(1); - BEAST_EXPECT(sim.synchronized()); - - // Next round generates B and C - // To force B, we inject an extra transaction in to those nodes - for (Peer* peer : groupABD) - { - peer->txInjections.emplace(peer->lastClosedLedger.seq(), Tx{42}); - } - // The Disruptor will ensure that nodes disconnect before the C - // validations make it to all but the fastC node - sim.run(1); - - // We are no longer in sync, but have not yet forked: - // 9 nodes consider A the last fully validated ledger and fastC sees C - BEAST_EXPECT(!sim.synchronized()); - BEAST_EXPECT(sim.branches() == 1); - - // Run another round to generate the 8 different C' ledgers - for (Peer* p : network) - p->submit(Tx(static_cast(p->id))); - sim.run(1); - - // Still not forked - BEAST_EXPECT(!sim.synchronized()); - BEAST_EXPECT(sim.branches() == 1); - - // Disruptor will reconnect all but the fastC node - sim.run(1); - - if (BEAST_EXPECT(sim.branches() == 1)) - { - BEAST_EXPECT(sim.synchronized()); - } - else // old approach caused a fork - { - BEAST_EXPECT(sim.branches(groupNotFastC) == 1); - BEAST_EXPECT(sim.synchronized(groupNotFastC) == 1); - } - } - - // Helper collector for testPauseForLaggards - // This will remove the ledgerAccept delay used to - // initially create the slow vs. fast validator groups. - struct UndoDelay - { - csf::PeerGroup& g; - - UndoDelay(csf::PeerGroup& a) : g(a) - { - } - - template - void - on(csf::PeerID, csf::SimTime, E const&) - { - } - - void - on(csf::PeerID who, csf::SimTime, csf::AcceptLedger const& e) - { - for (csf::Peer* p : g) - { - if (p->id == who) - p->delays.ledgerAccept = std::chrono::seconds{0}; - } - } - }; - - void - testPauseForLaggards() - { - using namespace csf; - using namespace std::chrono; - testcase("pause for laggards"); - - // Test that validators that jump ahead of the network slow - // down. - - // We engineer the following validated ledger history scenario: - // - // / --> B1 --> C1 --> ... -> G1 "ahead" - // A - // \ --> B2 --> C2 "behind" - // - // After validating a common ledger A, a set of "behind" validators - // briefly run slower and validate the lower chain of ledgers. - // The "ahead" validators run normal speed and run ahead validating the - // upper chain of ledgers. - // - // Due to the uncommitted support definition of the preferred branch - // protocol, even if the "behind" validators are a majority, the "ahead" - // validators cannot jump to the proper branch until the "behind" - // validators catch up to the same sequence number. For this test to - // succeed, the ahead validators need to briefly slow down consensus. - - ConsensusParms const parms{}; - Sim sim; - SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); - - PeerGroup behind = sim.createGroup(3); - PeerGroup const ahead = sim.createGroup(2); - PeerGroup network = ahead + behind; - - hash_set trustedKeys; - for (Peer const* p : network) - trustedKeys.insert(p->key); - for (Peer* p : network) - p->trustedKeys = trustedKeys; - - network.trustAndConnect(network, delay); - - // Initial seed round to set prior state - sim.run(1); - - // Have the "behind" group initially take a really long time to - // accept a ledger after ending deliberation - for (Peer* p : behind) - p->delays.ledgerAccept = 20s; - - // Use the collector to revert the delay after the single - // slow ledger is generated - UndoDelay undoDelay{behind}; - sim.collectors.add(undoDelay); - // Run the simulation for 100 seconds of simulation time with - std::chrono::nanoseconds const simDuration = 100s; - - // Simulate clients submitting 1 tx every 5 seconds to a random - // validator - Rate const rate{.count = 1, .duration = 5s}; - auto peerSelector = makeSelector( - network.begin(), network.end(), std::vector(network.size(), 1.), sim.rng); - auto txSubmitter = makeSubmitter( - ConstantDistribution{rate.inv()}, - sim.scheduler.now(), - sim.scheduler.now() + simDuration, - peerSelector, - sim.scheduler, - sim.rng); - - // Run simulation - sim.run(simDuration); - - // Verify that the network recovered - BEAST_EXPECT(sim.synchronized()); - } - - void - testDisputes() - { - testcase("disputes"); - - using namespace csf; - - // Test dispute objects directly - using Dispute = DisputedTx; - - Tx const txTrue{99}; - Tx const txFalse{98}; - Tx const txFollowingTrue{97}; - Tx const txFollowingFalse{96}; - int const numPeers = 100; - ConsensusParms const p; - std::size_t peersUnchanged = 0; - - auto logs = std::make_unique(beast::Severity::Error); - auto j = logs->journal("Test"); - auto clog = std::make_unique(); - - // Three cases: - // 1 proposing, initial vote yes - // 2 proposing, initial vote no - // 3 not proposing, initial vote doesn't matter after the first update, - // use yes - { - Dispute proposingTrue{txTrue.id(), true, numPeers, journal_}; - Dispute proposingFalse{txFalse.id(), false, numPeers, journal_}; - Dispute followingTrue{txFollowingTrue.id(), true, numPeers, journal_}; - Dispute followingFalse{txFollowingFalse.id(), false, numPeers, journal_}; - BEAST_EXPECT(proposingTrue.id() == 99); - BEAST_EXPECT(proposingFalse.id() == 98); - BEAST_EXPECT(followingTrue.id() == 97); - BEAST_EXPECT(followingFalse.id() == 96); - - // Create an even split in the peer votes - for (int i = 0; i < numPeers; ++i) - { - BEAST_EXPECT(proposingTrue.setVote(PeerID(i), i < 50)); - BEAST_EXPECT(proposingFalse.setVote(PeerID(i), i < 50)); - BEAST_EXPECT(followingTrue.setVote(PeerID(i), i < 50)); - BEAST_EXPECT(followingFalse.setVote(PeerID(i), i < 50)); - } - // Switch the middle vote to match mine - BEAST_EXPECT(proposingTrue.setVote(PeerID(50), true)); - BEAST_EXPECT(proposingFalse.setVote(PeerID(49), false)); - BEAST_EXPECT(followingTrue.setVote(PeerID(50), true)); - BEAST_EXPECT(followingFalse.setVote(PeerID(49), false)); - - // no changes yet - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(clog->str().empty()); - - // I'm in the majority, my vote should not change - BEAST_EXPECT(!proposingTrue.updateVote(5, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(5, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(5, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(5, false, p)); - - BEAST_EXPECT(!proposingTrue.updateVote(10, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(10, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(10, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(10, false, p)); - - peersUnchanged = 2; - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(clog->str().empty()); - - // Right now, the vote is 51%. The requirement is about to jump to - // 65% - BEAST_EXPECT(proposingTrue.updateVote(55, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(55, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(55, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(55, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == false); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - // 16 validators change their vote to match my original vote - for (int i = 0; i < 16; ++i) - { - auto pTrue = PeerID(numPeers - i - 1); - auto pFalse = PeerID(i); - BEAST_EXPECT(proposingTrue.setVote(pTrue, true)); - BEAST_EXPECT(proposingFalse.setVote(pFalse, false)); - BEAST_EXPECT(followingTrue.setVote(pTrue, true)); - BEAST_EXPECT(followingFalse.setVote(pFalse, false)); - } - // The vote should now be 66%, threshold is 65% - BEAST_EXPECT(proposingTrue.updateVote(60, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(60, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(60, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(60, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // Threshold jumps to 70% - BEAST_EXPECT(proposingTrue.updateVote(86, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(86, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(86, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(86, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == false); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // 5 more validators change their vote to match my original vote - for (int i = 16; i < 21; ++i) - { - auto pTrue = PeerID(numPeers - i - 1); - auto pFalse = PeerID(i); - BEAST_EXPECT(proposingTrue.setVote(pTrue, true)); - BEAST_EXPECT(proposingFalse.setVote(pFalse, false)); - BEAST_EXPECT(followingTrue.setVote(pTrue, true)); - BEAST_EXPECT(followingFalse.setVote(pFalse, false)); - } - - // The vote should now be 71%, threshold is 70% - BEAST_EXPECT(proposingTrue.updateVote(90, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(90, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(90, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(90, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // The vote should now be 71%, threshold is 70% - BEAST_EXPECT(!proposingTrue.updateVote(150, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(150, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(150, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(150, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // The vote should now be 71%, threshold is 70% - BEAST_EXPECT(!proposingTrue.updateVote(190, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(190, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(190, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(190, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - peersUnchanged = 3; - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(clog->str().empty()); - - // Threshold jumps to 95% - BEAST_EXPECT(proposingTrue.updateVote(220, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(220, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(220, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(220, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == false); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // 25 more validators change their vote to match my original vote - for (int i = 21; i < 46; ++i) - { - auto pTrue = PeerID(numPeers - i - 1); - auto pFalse = PeerID(i); - BEAST_EXPECT(proposingTrue.setVote(pTrue, true)); - BEAST_EXPECT(proposingFalse.setVote(pFalse, false)); - BEAST_EXPECT(followingTrue.setVote(pTrue, true)); - BEAST_EXPECT(followingFalse.setVote(pFalse, false)); - } - - // The vote should now be 96%, threshold is 95% - BEAST_EXPECT(proposingTrue.updateVote(250, true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(250, true, p)); - BEAST_EXPECT(!followingTrue.updateVote(250, false, p)); - BEAST_EXPECT(!followingFalse.updateVote(250, false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - for (peersUnchanged = 0; peersUnchanged < 6; ++peersUnchanged) - { - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingTrue.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(!followingFalse.stalled(p, false, peersUnchanged, j, clog)); - BEAST_EXPECT(clog->str().empty()); - } - - auto expectStalled = [this, &clog]( - int txid, - bool ourVote, - int ourTime, - int peerTime, - int support, - std::uint32_t line) { - using namespace std::string_literals; - - auto const s = clog->str(); - expect(s.find("stalled"), s, __FILE__, line); - expect(s.starts_with("Transaction "s + std::to_string(txid)), s, __FILE__, line); - expect(s.contains("voting "s + (ourVote ? "YES" : "NO")), s, __FILE__, line); - expect( - s.contains("for "s + std::to_string(ourTime) + " rounds."s), s, __FILE__, line); - expect( - s.contains("votes in "s + std::to_string(peerTime) + " rounds."), - s, - __FILE__, - line); - expect( - s.ends_with("has "s + std::to_string(support) + "% support. "s), - s, - __FILE__, - line); - clog = std::make_unique(); - }; - - for (int i = 0; i < 1; ++i) - { - BEAST_EXPECT(!proposingTrue.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!followingTrue.updateVote(250 + (10 * i), false, p)); - BEAST_EXPECT(!followingFalse.updateVote(250 + (10 * i), false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // true vote has changed recently, so not stalled - BEAST_EXPECT(!proposingTrue.stalled(p, true, 0, j, clog)); - BEAST_EXPECT(clog->str().empty()); - // remaining votes have been unchanged in so long that we only - // need to hit the second round at 95% to be stalled, regardless - // of peers - BEAST_EXPECT(proposingFalse.stalled(p, true, 0, j, clog)); - expectStalled(98, false, 11, 0, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, 0, j, clog)); - expectStalled(97, true, 11, 0, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, 0, j, clog)); - expectStalled(96, false, 11, 0, 3, __LINE__); - - // true vote has changed recently, so not stalled - BEAST_EXPECT(!proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - BEAST_EXPECTS(clog->str().empty(), clog->str()); - // remaining votes have been unchanged in so long that we only - // need to hit the second round at 95% to be stalled, regardless - // of peers - BEAST_EXPECT(proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(98, false, 11, 6, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(97, true, 11, 6, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(96, false, 11, 6, 3, __LINE__); - } - for (int i = 1; i < 3; ++i) - { - BEAST_EXPECT(!proposingTrue.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!followingTrue.updateVote(250 + (10 * i), false, p)); - BEAST_EXPECT(!followingFalse.updateVote(250 + (10 * i), false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - // true vote changed 2 rounds ago, and peers are changing, so - // not stalled - BEAST_EXPECT(!proposingTrue.stalled(p, true, 0, j, clog)); - BEAST_EXPECTS(clog->str().empty(), clog->str()); - // still stalled - BEAST_EXPECT(proposingFalse.stalled(p, true, 0, j, clog)); - expectStalled(98, false, 11 + i, 0, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, 0, j, clog)); - expectStalled(97, true, 11 + i, 0, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, 0, j, clog)); - expectStalled(96, false, 11 + i, 0, 3, __LINE__); - - // true vote changed 2 rounds ago, and peers are NOT changing, - // so stalled - BEAST_EXPECT(proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(99, true, 1 + i, 6, 97, __LINE__); - // still stalled - BEAST_EXPECT(proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(98, false, 11 + i, 6, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(97, true, 11 + i, 6, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(96, false, 11 + i, 6, 3, __LINE__); - } - for (int i = 3; i < 5; ++i) - { - BEAST_EXPECT(!proposingTrue.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!proposingFalse.updateVote(250 + (10 * i), true, p)); - BEAST_EXPECT(!followingTrue.updateVote(250 + (10 * i), false, p)); - BEAST_EXPECT(!followingFalse.updateVote(250 + (10 * i), false, p)); - - BEAST_EXPECT(proposingTrue.getOurVote() == true); - BEAST_EXPECT(proposingFalse.getOurVote() == false); - BEAST_EXPECT(followingTrue.getOurVote() == true); - BEAST_EXPECT(followingFalse.getOurVote() == false); - - BEAST_EXPECT(proposingTrue.stalled(p, true, 0, j, clog)); - expectStalled(99, true, 1 + i, 0, 97, __LINE__); - BEAST_EXPECT(proposingFalse.stalled(p, true, 0, j, clog)); - expectStalled(98, false, 11 + i, 0, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, 0, j, clog)); - expectStalled(97, true, 11 + i, 0, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, 0, j, clog)); - expectStalled(96, false, 11 + i, 0, 3, __LINE__); - - BEAST_EXPECT(proposingTrue.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(99, true, 1 + i, 6, 97, __LINE__); - BEAST_EXPECT(proposingFalse.stalled(p, true, peersUnchanged, j, clog)); - expectStalled(98, false, 11 + i, 6, 2, __LINE__); - BEAST_EXPECT(followingTrue.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(97, true, 11 + i, 6, 97, __LINE__); - BEAST_EXPECT(followingFalse.stalled(p, false, peersUnchanged, j, clog)); - expectStalled(96, false, 11 + i, 6, 3, __LINE__); - } - } - } - - void - run() override - { - testShouldCloseLedger(); - testCheckConsensus(); - - testStandalone(); - testPeersAgree(); - testSlowPeers(); - testCloseTimeDisagree(); - testWrongLCL(); - testConsensusCloseTimeRounding(); - testFork(); - testHubNetwork(); - testPreferredByBranch(); - testPauseForLaggards(); - testDisputes(); - } -}; - -BEAST_DEFINE_TESTSUITE(Consensus, consensus, xrpl); -} // namespace xrpl::test diff --git a/src/test/consensus/DistributedValidatorsSim_test.cpp b/src/test/consensus/DistributedValidatorsSim_test.cpp deleted file mode 100644 index 437ad81ee0..0000000000 --- a/src/test/consensus/DistributedValidatorsSim_test.cpp +++ /dev/null @@ -1,253 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace xrpl::test { - -/** - * In progress simulations for diversifying and distributing validators - */ -class DistributedValidators_test : public beast::unit_test::Suite -{ - void - completeTrustCompleteConnectFixedDelay( - std::size_t numPeers, - std::chrono::milliseconds delay = std::chrono::milliseconds(200), - bool printHeaders = false) - { - using namespace csf; - using namespace std::chrono; - - // Initialize persistent collector logs specific to this method - std::string const prefix = - "DistributedValidators_" - "completeTrustCompleteConnectFixedDelay"; - std::fstream txLog(prefix + "_tx.csv", std::ofstream::app), - ledgerLog(prefix + "_ledger.csv", std::ofstream::app); - - // title - log << prefix << "(" << numPeers << "," << delay.count() << ")" << std::endl; - - // number of peers, UNLs, connections - BEAST_EXPECT(numPeers >= 1); - - Sim sim; - PeerGroup peers = sim.createGroup(numPeers); - - // complete trust graph - peers.trust(peers); - - // complete connect graph with fixed delay - peers.connect(peers, delay); - - // Initialize collectors to track statistics to report - TxCollector txCollector; - LedgerCollector ledgerCollector; - auto colls = makeCollectors(txCollector, ledgerCollector); - sim.collectors.add(colls); - - // Initial round to set prior state - sim.run(1); - - // Run for 10 minutes, submitting 100 tx/second - std::chrono::nanoseconds const simDuration = 10min; - std::chrono::nanoseconds const quiet = 10s; - Rate const rate{.count = 100, .duration = 1000ms}; - - // Initialize timers - HeartbeatTimer heart(sim.scheduler); - - // txs, start/stop/step, target - auto peerSelector = - makeSelector(peers.begin(), peers.end(), std::vector(numPeers, 1.), sim.rng); - auto txSubmitter = makeSubmitter( - ConstantDistribution{rate.inv()}, - sim.scheduler.now() + quiet, - sim.scheduler.now() + simDuration - quiet, - peerSelector, - sim.scheduler, - sim.rng); - - // run simulation for given duration - heart.start(); - sim.run(simDuration); - - // BEAST_EXPECT(sim.branches() == 1); - // BEAST_EXPECT(sim.synchronized()); - - log << std::right; - log << "| Peers: " << std::setw(2) << peers.size(); - log << " | Duration: " << std::setw(6) << duration_cast(simDuration).count() - << " ms"; - log << " | Branches: " << std::setw(1) << sim.branches(); - log << " | Synchronized: " << std::setw(1) << (sim.synchronized() ? "Y" : "N"); - log << " |" << std::endl; - - txCollector.report(simDuration, log, true); - ledgerCollector.report(simDuration, log, false); - - std::string const tag = std::to_string(numPeers); - txCollector.csv(simDuration, txLog, tag, printHeaders); - ledgerCollector.csv(simDuration, ledgerLog, tag, printHeaders); - - log << std::endl; - } - - void - completeTrustScaleFreeConnectFixedDelay( - std::size_t numPeers, - std::chrono::milliseconds delay = std::chrono::milliseconds(200), - bool printHeaders = false) - { - using namespace csf; - using namespace std::chrono; - - // Initialize persistent collector logs specific to this method - std::string const prefix = - "DistributedValidators__" - "completeTrustScaleFreeConnectFixedDelay"; - std::fstream txLog(prefix + "_tx.csv", std::ofstream::app), - ledgerLog(prefix + "_ledger.csv", std::ofstream::app); - - // title - log << prefix << "(" << numPeers << "," << delay.count() << ")" << std::endl; - - // number of peers, UNLs, connections - int const numCNLs = std::max(int(1.00 * numPeers), 1); - int const minCNLSize = std::max(int(0.25 * numCNLs), 1); - int const maxCNLSize = std::max(int(0.50 * numCNLs), 1); - BEAST_EXPECT(numPeers >= 1); - BEAST_EXPECT(numCNLs >= 1); - BEAST_EXPECT(1 <= minCNLSize && minCNLSize <= maxCNLSize && maxCNLSize <= numPeers); - - Sim sim; - PeerGroup peers = sim.createGroup(numPeers); - - // complete trust graph - peers.trust(peers); - - // scale-free connect graph with fixed delay - std::vector const ranks = sample(peers.size(), PowerLawDistribution{1, 3}, sim.rng); - randomRankedConnect( - peers, - ranks, - numCNLs, - std::uniform_int_distribution<>{minCNLSize, maxCNLSize}, - sim.rng, - delay); - - // Initialize collectors to track statistics to report - TxCollector txCollector; - LedgerCollector ledgerCollector; - auto colls = makeCollectors(txCollector, ledgerCollector); - sim.collectors.add(colls); - - // Initial round to set prior state - sim.run(1); - - // Run for 10 minutes, submitting 100 tx/second - std::chrono::nanoseconds const simDuration = 10min; - std::chrono::nanoseconds const quiet = 10s; - Rate const rate{.count = 100, .duration = 1000ms}; - - // Initialize timers - HeartbeatTimer heart(sim.scheduler); - - // txs, start/stop/step, target - auto peerSelector = - makeSelector(peers.begin(), peers.end(), std::vector(numPeers, 1.), sim.rng); - auto txSubmitter = makeSubmitter( - ConstantDistribution{rate.inv()}, - sim.scheduler.now() + quiet, - sim.scheduler.now() + simDuration - quiet, - peerSelector, - sim.scheduler, - sim.rng); - - // run simulation for given duration - heart.start(); - sim.run(simDuration); - - // BEAST_EXPECT(sim.branches() == 1); - // BEAST_EXPECT(sim.synchronized()); - - log << std::right; - log << "| Peers: " << std::setw(2) << peers.size(); - log << " | Duration: " << std::setw(6) << duration_cast(simDuration).count() - << " ms"; - log << " | Branches: " << std::setw(1) << sim.branches(); - log << " | Synchronized: " << std::setw(1) << (sim.synchronized() ? "Y" : "N"); - log << " |" << std::endl; - - txCollector.report(simDuration, log, true); - ledgerCollector.report(simDuration, log, false); - - std::string const tag = std::to_string(numPeers); - txCollector.csv(simDuration, txLog, tag, printHeaders); - ledgerCollector.csv(simDuration, ledgerLog, tag, printHeaders); - - log << std::endl; - } - - void - run() override - { - std::string const defaultArgs = "5 200"; - std::string const args = arg().empty() ? defaultArgs : arg(); - std::stringstream argStream(args); - - int maxNumValidators = 0; - int delayCount(200); - argStream >> maxNumValidators; - argStream >> delayCount; - - std::chrono::milliseconds const delay(delayCount); - - log << "DistributedValidators: 1 to " << maxNumValidators << " Peers" << std::endl; - - /** - * Simulate with N = 1 to N - * - complete trust graph is complete - * - complete network connectivity - * - fixed delay for network links - */ - completeTrustCompleteConnectFixedDelay(1, delay, true); - for (int i = 2; i <= maxNumValidators; i++) - { - completeTrustCompleteConnectFixedDelay(i, delay); - } - - /** - * Simulate with N = 1 to N - * - complete trust graph is complete - * - scale-free network connectivity - * - fixed delay for network links - */ - completeTrustScaleFreeConnectFixedDelay(1, delay, true); - for (int i = 2; i <= maxNumValidators; i++) - { - completeTrustScaleFreeConnectFixedDelay(i, delay); - } - } -}; - -BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(DistributedValidators, consensus, xrpl, 2); - -} // namespace xrpl::test diff --git a/src/test/consensus/LedgerTiming_test.cpp b/src/test/consensus/LedgerTiming_test.cpp deleted file mode 100644 index 632361c799..0000000000 --- a/src/test/consensus/LedgerTiming_test.cpp +++ /dev/null @@ -1,118 +0,0 @@ -#include -#include -#include - -#include -#include -#include - -namespace xrpl::test { - -class LedgerTiming_test : public beast::unit_test::Suite -{ - void - testGetNextLedgerTimeResolution() - { - // helper to iteratively call into getNextLedgerTimeResolution - struct TestRes - { - std::uint32_t decrease = 0; - std::uint32_t equal = 0; - std::uint32_t increase = 0; - - static TestRes - run(bool previousAgree, std::uint32_t rounds) - { - TestRes res; - auto closeResolution = kLedgerDefaultTimeResolution; - auto nextCloseResolution = closeResolution; - std::uint32_t round = 0; - do - { - nextCloseResolution = - getNextLedgerTimeResolution(closeResolution, previousAgree, ++round); - if (nextCloseResolution < closeResolution) - { - ++res.decrease; - } - else if (nextCloseResolution > closeResolution) - { - ++res.increase; - } - else - { - ++res.equal; - } - std::swap(nextCloseResolution, closeResolution); - } while (round < rounds); - return res; - } - }; - - // If we never agree on close time, only can increase resolution - // until hit the max - auto decreases = TestRes::run(false, 10); - BEAST_EXPECT(decreases.increase == 3); - BEAST_EXPECT(decreases.decrease == 0); - BEAST_EXPECT(decreases.equal == 7); - - // If we always agree on close time, only can decrease resolution - // until hit the min - auto increases = TestRes::run(false, 100); - BEAST_EXPECT(increases.increase == 3); - BEAST_EXPECT(increases.decrease == 0); - BEAST_EXPECT(increases.equal == 97); - } - - void - testRoundCloseTime() - { - using namespace std::chrono_literals; - // A closeTime equal to the epoch is not modified - using tp = NetClock::time_point; - tp const def; - BEAST_EXPECT(def == roundCloseTime(def, 30s)); - - // Otherwise, the closeTime is rounded to the nearest - // rounding up on ties - BEAST_EXPECT(tp{0s} == roundCloseTime(tp{29s}, 60s)); - BEAST_EXPECT(tp{30s} == roundCloseTime(tp{30s}, 1s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{31s}, 60s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{30s}, 60s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{59s}, 60s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{60s}, 60s)); - BEAST_EXPECT(tp{60s} == roundCloseTime(tp{61s}, 60s)); - } - - void - testEffCloseTime() - { - using namespace std::chrono_literals; - using tp = NetClock::time_point; - tp close = effCloseTime(tp{10s}, 30s, tp{0s}); - BEAST_EXPECT(close == tp{1s}); - - close = effCloseTime(tp{16s}, 30s, tp{0s}); - BEAST_EXPECT(close == tp{30s}); - - close = effCloseTime(tp{16s}, 30s, tp{30s}); - BEAST_EXPECT(close == tp{31s}); - - close = effCloseTime(tp{16s}, 30s, tp{60s}); - BEAST_EXPECT(close == tp{61s}); - - close = effCloseTime(tp{31s}, 30s, tp{0s}); - BEAST_EXPECT(close == tp{30s}); - } - - void - run() override - { - testGetNextLedgerTimeResolution(); - testRoundCloseTime(); - testEffCloseTime(); - } -}; - -BEAST_DEFINE_TESTSUITE(LedgerTiming, consensus, xrpl); -} // namespace xrpl::test diff --git a/src/test/consensus/LedgerTrie_test.cpp b/src/test/consensus/LedgerTrie_test.cpp deleted file mode 100644 index a4eb7bc087..0000000000 --- a/src/test/consensus/LedgerTrie_test.cpp +++ /dev/null @@ -1,716 +0,0 @@ -#include - -#include - -#include - -#include -#include -#include - -namespace xrpl::test { - -class LedgerTrie_test : public beast::unit_test::Suite -{ - void - testInsert() - { - using namespace csf; - // Single entry by itself - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - - t.insert(h["abc"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - } - // Suffix of existing (extending tree) - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - BEAST_EXPECT(t.checkInvariants()); - // extend with no siblings - t.insert(h["abcd"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - - // extend with existing sibling - t.insert(h["abce"]); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abce"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abce"]) == 1); - } - // uncommitted of existing node - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"]); - BEAST_EXPECT(t.checkInvariants()); - // uncommitted with no siblings - t.insert(h["abcdf"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcdf"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcdf"]) == 1); - - // uncommitted with existing child - t.insert(h["abc"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcdf"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcdf"]) == 1); - } - // Suffix + uncommitted of existing node - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"]); - BEAST_EXPECT(t.checkInvariants()); - t.insert(h["abce"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abce"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abce"]) == 1); - } - // Suffix + uncommitted with existing child - { - // abcd : abcde, abcf - - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"]); - BEAST_EXPECT(t.checkInvariants()); - t.insert(h["abcde"]); - BEAST_EXPECT(t.checkInvariants()); - t.insert(h["abcf"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcf"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcf"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abcde"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcde"]) == 1); - } - - // Multiple counts - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"], 4); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 4); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 4); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.branchSupport(h["a"]) == 4); - - t.insert(h["abc"], 2); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 4); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 6); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.branchSupport(h["a"]) == 6); - } - } - - void - testRemove() - { - using namespace csf; - // Not in trie - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - - BEAST_EXPECT(!t.remove(h["ab"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(!t.remove(h["a"])); - BEAST_EXPECT(t.checkInvariants()); - } - // In trie but with 0 tip support - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"]); - t.insert(h["abce"]); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(!t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - } - // In trie with > 1 tip support - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"], 2); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); - BEAST_EXPECT(t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - - t.insert(h["abc"], 1); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 2); - BEAST_EXPECT(t.remove(h["abc"], 2)); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - - t.insert(h["abc"], 3); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 3); - BEAST_EXPECT(t.remove(h["abc"], 300)); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - } - // In trie with = 1 tip support, no children - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["abc"]); - - BEAST_EXPECT(t.tipSupport(h["ab"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - - BEAST_EXPECT(t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 0); - } - // In trie with = 1 tip support, 1 child - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["abc"]); - t.insert(h["abcd"]); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - - BEAST_EXPECT(t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 1); - } - // In trie with = 1 tip support, > 1 children - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["abc"]); - t.insert(h["abcd"]); - t.insert(h["abce"]); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 3); - - BEAST_EXPECT(t.remove(h["abc"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 2); - } - - // In trie with = 1 tip support, parent compaction - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["abc"]); - t.insert(h["abd"]); - BEAST_EXPECT(t.checkInvariants()); - t.remove(h["ab"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abd"]) == 1); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 2); - - t.remove(h["abd"]); - BEAST_EXPECT(t.checkInvariants()); - - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); - } - } - - void - testEmpty() - { - using namespace csf; - LedgerTrie t; - LedgerHistoryHelper h; - BEAST_EXPECT(t.empty()); - - Ledger const genesis = h[""]; - t.insert(genesis); - BEAST_EXPECT(!t.empty()); - t.remove(genesis); - BEAST_EXPECT(t.empty()); - - t.insert(h["abc"]); - BEAST_EXPECT(!t.empty()); - t.remove(h["abc"]); - BEAST_EXPECT(t.empty()); - } - - void - testSupport() - { - using namespace csf; - - LedgerTrie t; - LedgerHistoryHelper h; - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.tipSupport(h["axy"]) == 0); - - BEAST_EXPECT(t.branchSupport(h["a"]) == 0); - BEAST_EXPECT(t.branchSupport(h["axy"]) == 0); - - t.insert(h["abc"]); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abcd"]) == 0); - - BEAST_EXPECT(t.branchSupport(h["a"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abcd"]) == 0); - - t.insert(h["abe"]); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 1); - BEAST_EXPECT(t.tipSupport(h["abe"]) == 1); - - BEAST_EXPECT(t.branchSupport(h["a"]) == 2); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 2); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abe"]) == 1); - - t.remove(h["abc"]); - BEAST_EXPECT(t.tipSupport(h["a"]) == 0); - BEAST_EXPECT(t.tipSupport(h["ab"]) == 0); - BEAST_EXPECT(t.tipSupport(h["abc"]) == 0); - BEAST_EXPECT(t.tipSupport(h["abe"]) == 1); - - BEAST_EXPECT(t.branchSupport(h["a"]) == 1); - BEAST_EXPECT(t.branchSupport(h["ab"]) == 1); - BEAST_EXPECT(t.branchSupport(h["abc"]) == 0); - BEAST_EXPECT(t.branchSupport(h["abe"]) == 1); - } - - void - testGetPreferred() - { - using namespace csf; - using Seq = Ledger::Seq; - // Empty - { - LedgerTrie const t; - BEAST_EXPECT(t.getPreferred(Seq{0}) == std::nullopt); - BEAST_EXPECT(t.getPreferred(Seq{2}) == std::nullopt); - } - // Genesis support is NOT empty - { - LedgerTrie t; - LedgerHistoryHelper h; - Ledger const genesis = h[""]; - t.insert(genesis); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{0})->id == genesis.id()); - BEAST_EXPECT(t.remove(genesis)); - BEAST_EXPECT(t.getPreferred(Seq{0}) == std::nullopt); - BEAST_EXPECT(!t.remove(genesis)); - } - // Single node no children - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - } - // Single node smaller child support - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"]); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - } - // Single node larger child - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"], 2); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abcd"].id()); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcd"].id()); - } - // Single node smaller children support - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"]); - t.insert(h["abce"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - - t.insert(h["abc"]); - - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - // Single node larger children - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"], 2); - t.insert(h["abce"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - - t.insert(h["abcd"]); - - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abcd"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcd"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - // Tie-breaker by id - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abcd"], 2); - t.insert(h["abce"], 2); - - BEAST_EXPECT(h["abce"].id() > h["abcd"].id()); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abce"].id()); - - t.insert(h["abcd"]); - BEAST_EXPECT(h["abce"].id() > h["abcd"].id()); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcd"].id()); - } - - // Tie-breaker not needed - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"]); - t.insert(h["abce"], 2); - // abce only has a margin of 1, but it owns the tie-breaker - BEAST_EXPECT(h["abce"].id() > h["abcd"].id()); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abce"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abce"].id()); - - // Switch support from abce to abcd, tie-breaker now needed - t.remove(h["abce"]); - t.insert(h["abcd"]); - - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - - // Single node larger grand child - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcd"], 2); - t.insert(h["abcde"], 4); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abcde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["abcde"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - - // Too much uncommitted support from competing branches - { - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["abc"]); - t.insert(h["abcde"], 2); - t.insert(h["abcfg"], 2); - // 'de' and 'fg' are tied without 'abc' vote - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abc"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["abc"].id()); - - t.remove(h["abc"]); - t.insert(h["abcd"]); - - // 'de' branch has 3 votes to 2, so earlier sequences see it as preferred - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abcde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["abcde"].id()); - - // However, if you validated a ledger with Seq 5, potentially on - // a different branch, you do not yet know if they chose abcd - // or abcf because of you, so abc remains preferred - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["abc"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - - // Changing largestSeq perspective changes preferred branch - { - /** - * Build the tree below with initial tip support annotated - * A - * / \ - * B(1) C(1) - * / | | - * H D F(1) - * | - * E(2) - * | - * G - */ - LedgerTrie t; - LedgerHistoryHelper h; - t.insert(h["ab"]); - t.insert(h["ac"]); - t.insert(h["acf"]); - t.insert(h["abde"], 2); - - // B has more branch support - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{1})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{2})->id == h["ab"].id()); - - // But if you last validated D,F or E, you do not yet know - // if someone used that validation to commit to B or C - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["a"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["a"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - - /** - * One of E advancing to G doesn't change anything - * A - * / \ - * B(1) C(1) - * / | | - * H D F(1) - * | - * E(1) - * | - * G(1) - */ - t.remove(h["abde"]); - t.insert(h["abdeg"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{1})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{2})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["a"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["a"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["a"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - - /** - * C advancing to H does advance the seq 3 preferred ledger - * A - * / \ - * B(1) C - * / | | - * H(1)D F(1) - * | - * E(1) - * | - * G(1) - */ - t.remove(h["ac"]); - t.insert(h["abh"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{1})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{2})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["a"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["a"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - - /** - * F advancing to E also moves the preferred ledger forward - * A - * / \ - * B(1) C - * / | | - * H(1)D F - * | - * E(2) - * | - * G(1) - */ - t.remove(h["acf"]); - t.insert(h["abde"]); - - // NOLINTBEGIN(bugprone-unchecked-optional-access) - BEAST_EXPECT(t.getPreferred(Seq{1})->id == h["abde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{2})->id == h["abde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{3})->id == h["abde"].id()); - BEAST_EXPECT(t.getPreferred(Seq{4})->id == h["ab"].id()); - BEAST_EXPECT(t.getPreferred(Seq{5})->id == h["ab"].id()); - // NOLINTEND(bugprone-unchecked-optional-access) - } - } - - void - testRootRelated() - { - using namespace csf; - // Since the root is a special node that breaks the no-single child - // invariant, do some tests that exercise it. - - LedgerTrie t; - LedgerHistoryHelper h; - BEAST_EXPECT(!t.remove(h[""])); - BEAST_EXPECT(t.branchSupport(h[""]) == 0); - BEAST_EXPECT(t.tipSupport(h[""]) == 0); - - t.insert(h["a"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.branchSupport(h[""]) == 1); - BEAST_EXPECT(t.tipSupport(h[""]) == 0); - - t.insert(h["e"]); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.branchSupport(h[""]) == 2); - BEAST_EXPECT(t.tipSupport(h[""]) == 0); - - BEAST_EXPECT(t.remove(h["e"])); - BEAST_EXPECT(t.checkInvariants()); - BEAST_EXPECT(t.branchSupport(h[""]) == 1); - BEAST_EXPECT(t.tipSupport(h[""]) == 0); - } - - void - testStress() - { - using namespace csf; - LedgerTrie t; - LedgerHistoryHelper h; - - // Test quasi-randomly add/remove supporting for different ledgers - // from a branching history. - - // Ledgers have sequence 1,2,3,4 - std::uint32_t const depthConst = 4; - // Each ledger has 4 possible children - std::uint32_t const width = 4; - - std::uint32_t const iterations = 10000; - - // Use explicit seed to have same results for CI - // NOLINTNEXTLINE(bugprone-random-generator-seed): fixed seed for reproducible test - std::mt19937 gen{42}; - std::uniform_int_distribution<> depthDist(0, depthConst - 1); - std::uniform_int_distribution<> widthDist(0, width - 1); - std::uniform_int_distribution<> flip(0, 1); - for (std::uint32_t i = 0; i < iterations; ++i) - { - // pick a random ledger history - std::string curr; - char const depth = depthDist(gen); - char offset = 0; - for (char d = 0; d < depth; ++d) - { - char const a = offset + widthDist(gen); - curr += a; - offset = (a + 1) * width; - } - - // 50-50 to add remove - if (flip(gen) == 0) - { - t.insert(h[curr]); - } - else - { - t.remove(h[curr]); - } - if (!BEAST_EXPECT(t.checkInvariants())) - return; - } - } - - void - run() override - { - testInsert(); - testRemove(); - testEmpty(); - testSupport(); - testGetPreferred(); - testRootRelated(); - testStress(); - } -}; - -BEAST_DEFINE_TESTSUITE(LedgerTrie, consensus, xrpl); -} // namespace xrpl::test diff --git a/src/test/consensus/RCLCensorshipDetector_test.cpp b/src/test/consensus/RCLCensorshipDetector_test.cpp deleted file mode 100644 index 722a34f937..0000000000 --- a/src/test/consensus/RCLCensorshipDetector_test.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include - -#include - -#include -#include -#include - -namespace xrpl::test { - -class RCLCensorshipDetector_test : public beast::unit_test::Suite -{ - void - test( - RCLCensorshipDetector& cdet, - int round, - std::vector proposed, - std::vector accepted, - std::vector remain, - std::vector remove) - { - // Begin tracking what we're proposing this round - RCLCensorshipDetector::TxIDSeqVec proposal; - for (auto const& i : proposed) - proposal.emplace_back(i, round); - cdet.propose(std::move(proposal)); - - // Finalize the round, by processing what we accepted; then - // remove anything that needs to be removed and ensure that - // what remains is correct. - cdet.check(std::move(accepted), [&remove, &remain](auto id, auto seq) { - // If the item is supposed to be removed from the censorship - // detector internal tracker manually, do it now: - if (std::ranges::find(remove, id) != remove.end()) - return true; - - // If the item is supposed to still remain in the censorship - // detector internal tracker; remove it from the vector. - auto it = std::ranges::find(remain, id); - if (it != remain.end()) - remain.erase(it); - return false; - }); - - // On entry, this set contained all the elements that should be tracked - // by the detector after we process this round. We removed all the items - // that actually were in the tracker, so this should now be empty: - BEAST_EXPECT(remain.empty()); - } - -public: - void - run() override - { - testcase("Censorship Detector"); - - RCLCensorshipDetector cdet; - int round = 0; - // proposed accepted remain remove - test(cdet, ++round, {}, {}, {}, {}); - test(cdet, ++round, {10, 11, 12, 13}, {11, 2}, {10, 13}, {}); - test(cdet, ++round, {10, 13, 14, 15}, {14}, {10, 13, 15}, {}); - test(cdet, ++round, {10, 13, 15, 16}, {15, 16}, {10, 13}, {}); - test(cdet, ++round, {10, 13}, {17, 18}, {10, 13}, {}); - test(cdet, ++round, {10, 19}, {}, {10, 19}, {}); - test(cdet, ++round, {10, 19, 20}, {20}, {10}, {19}); - test(cdet, ++round, {21}, {21}, {}, {}); - test(cdet, ++round, {}, {22}, {}, {}); - test(cdet, ++round, {23, 24, 25, 26}, {25, 27}, {23, 26}, {24}); - test(cdet, ++round, {23, 26, 28}, {26, 28}, {23}, {}); - - for (int i = 0; i != 10; ++i) - test(cdet, ++round, {23}, {}, {23}, {}); - - test(cdet, ++round, {23, 29}, {29}, {23}, {}); - test(cdet, ++round, {30, 31}, {31}, {30}, {}); - test(cdet, ++round, {30}, {30}, {}, {}); - test(cdet, ++round, {}, {}, {}, {}); - } -}; - -BEAST_DEFINE_TESTSUITE(RCLCensorshipDetector, consensus, xrpl); -} // namespace xrpl::test diff --git a/src/test/consensus/ScaleFreeSim_test.cpp b/src/test/consensus/ScaleFreeSim_test.cpp deleted file mode 100644 index e533e09eb0..0000000000 --- a/src/test/consensus/ScaleFreeSim_test.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include - -namespace xrpl::test { - -class ScaleFreeSim_test : public beast::unit_test::Suite -{ - void - run() override - { - using namespace std::chrono; - using namespace csf; - - // Generate a quasi-random scale free network and simulate consensus - // as we vary transaction submission rates - - int const n = 100; // Peers - - int const numUNLs = 15; // UNL lists - int const minUNLSize = n / 4, maxUNLSize = n / 2; - - ConsensusParms const parms{}; - Sim sim; - PeerGroup network = sim.createGroup(n); - - // generate trust ranks - std::vector const ranks = - sample(network.size(), PowerLawDistribution{1, 3}, sim.rng); - - // generate scale-free trust graph - randomRankedTrust( - network, - ranks, - numUNLs, - std::uniform_int_distribution<>{minUNLSize, maxUNLSize}, - sim.rng); - - // nodes with a trust line in either direction are network-connected - network.connectFromTrust(round(0.2 * parms.ledgerGRANULARITY)); - - // Initialize collectors to track statistics to report - TxCollector txCollector; - LedgerCollector ledgerCollector; - auto colls = makeCollectors(txCollector, ledgerCollector); - sim.collectors.add(colls); - - // Initial round to set prior state - sim.run(1); - - // Initialize timers - HeartbeatTimer heart(sim.scheduler, seconds(10s)); - - // Run for 10 minutes, submitting 100 tx/second - std::chrono::nanoseconds const simDuration = 10min; - std::chrono::nanoseconds const quiet = 10s; - Rate const rate{.count = 100, .duration = 1000ms}; - - // txs, start/stop/step, target - auto peerSelector = makeSelector(network.begin(), network.end(), ranks, sim.rng); - auto txSubmitter = makeSubmitter( - ConstantDistribution{rate.inv()}, - sim.scheduler.now() + quiet, - sim.scheduler.now() + (simDuration - quiet), - peerSelector, - sim.scheduler, - sim.rng); - - // run simulation for given duration - heart.start(); - sim.run(simDuration); - - BEAST_EXPECT(sim.branches() == 1); - BEAST_EXPECT(sim.synchronized()); - - // TODO: Clean up this formatting mess!! - - log << "Peers: " << network.size() << std::endl; - log << "Simulated Duration: " << duration_cast(simDuration).count() << " ms" - << std::endl; - log << "Branches: " << sim.branches() << std::endl; - log << "Synchronized: " << (sim.synchronized() ? "Y" : "N") << std::endl; - log << std::endl; - - txCollector.report(simDuration, log); - ledgerCollector.report(simDuration, log); - // Print summary? - // # forks? # of LCLs? - // # peers - // # tx submitted - // # ledgers/sec etc.? - } -}; - -BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(ScaleFreeSim, consensus, xrpl, 80); - -} // namespace xrpl::test diff --git a/src/test/consensus/Validations_test.cpp b/src/test/consensus/Validations_test.cpp deleted file mode 100644 index 606c6f2824..0000000000 --- a/src/test/consensus/Validations_test.cpp +++ /dev/null @@ -1,1059 +0,0 @@ -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace xrpl::test::csf { -class Validations_test : public beast::unit_test::Suite -{ - using clock_type = beast::AbstractClock const; - - // Helper to convert steady_clock to a reasonable NetClock - // This allows a single manual clock in the unit tests - static NetClock::time_point - toNetClock(clock_type const& c) - { - // We don't care about the actual epochs, but do want the - // generated NetClock time to be well past its epoch to ensure - // any subtractions are positive - using namespace std::chrono; - return NetClock::time_point( - duration_cast(c.now().time_since_epoch() + 86400s)); - } - - // Represents a node that can issue validations - class Node - { - clock_type const& c_; - PeerID nodeID_; - bool trusted_ = true; - std::size_t signIdx_{1}; - std::optional loadFee_; - - public: - Node(PeerID nodeID, clock_type const& c) : c_(c), nodeID_(nodeID) - { - } - - void - untrust() - { - trusted_ = false; - } - - void - trust() - { - trusted_ = true; - } - - void - setLoadFee(std::uint32_t fee) - { - loadFee_ = fee; - } - - [[nodiscard]] PeerID - nodeID() const - { - return nodeID_; - } - - void - advanceKey() - { - signIdx_++; - } - - [[nodiscard]] PeerKey - currKey() const - { - return std::make_pair(nodeID_, signIdx_); - } - - [[nodiscard]] PeerKey - masterKey() const - { - return std::make_pair(nodeID_, 0); - } - [[nodiscard]] NetClock::time_point - now() const - { - return toNetClock(c_); - } - - // Issue a new validation with given sequence number and id and - // with signing and seen times offset from the common clock - [[nodiscard]] Validation - validate( - Ledger::ID id, - Ledger::Seq seq, - NetClock::duration signOffset, - NetClock::duration seenOffset, - bool full) const - { - Validation v{ - id, - seq, - now() + signOffset, - now() + seenOffset, - currKey(), - nodeID_, - full, - loadFee_}; - if (trusted_) - v.setTrusted(); - return v; - } - - [[nodiscard]] Validation - validate(Ledger ledger, NetClock::duration signOffset, NetClock::duration seenOffset) const - { - return validate(ledger.id(), ledger.seq(), signOffset, seenOffset, true); - } - - [[nodiscard]] Validation - validate(Ledger ledger) const - { - return validate( - ledger.id(), ledger.seq(), NetClock::duration{0}, NetClock::duration{0}, true); - } - - [[nodiscard]] Validation - partial(Ledger ledger) const - { - return validate( - ledger.id(), ledger.seq(), NetClock::duration{0}, NetClock::duration{0}, false); - } - }; - - // Generic Validations adaptor - class Adaptor - { - clock_type& c_; - LedgerOracle& oracle_; - - public: - // Non-locking mutex to avoid locks in generic Validations - struct Mutex - { - void - lock() - { - } - - void - unlock() - { - } - }; - - using Validation = csf::Validation; - using Ledger = csf::Ledger; - - Adaptor(clock_type& c, LedgerOracle& o) : c_{c}, oracle_{o} - { - } - - [[nodiscard]] NetClock::time_point - now() const - { - return toNetClock(c_); - } - - std::optional - acquire(Ledger::ID const& id) - { - return oracle_.lookup(id); - } - }; - - // Specialize generic Validations using the above types - using TestValidations = Validations; - - // Gather the dependencies of TestValidations in a single class and provide - // accessors for simplifying test logic - class TestHarness - { - ValidationParms p_; - beast::ManualClock clock_; - TestValidations tv_; - PeerID nextNodeId_{0}; - - public: - explicit TestHarness(LedgerOracle& o) : tv_(p_, clock_, clock_, o) - { - } - - ValStatus - add(Validation const& v) - { - return tv_.add(v.nodeID(), v); - } - - TestValidations& - vals() - { - return tv_; - } - - Node - makeNode() - { - return Node(nextNodeId_++, clock_); - } - - ValidationParms - parms() const - { - return p_; - } - - auto& - clock() - { - return clock_; - } - }; - - Ledger const genesisLedger_{Ledger::MakeGenesis{}}; - - void - testAddValidation() - { - using namespace std::chrono_literals; - - testcase("Add validation"); - LedgerHistoryHelper h; - Ledger const ledgerA = h["a"]; - Ledger ledgerAB = h["ab"]; - Ledger ledgerAZ = h["az"]; - Ledger ledgerABC = h["abc"]; - Ledger const ledgerABCD = h["abcd"]; - Ledger const ledgerABCDE = h["abcde"]; - - { - TestHarness harness(h.oracle); - Node n = harness.makeNode(); - - auto const v = n.validate(ledgerA); - - // Add a current validation - BEAST_EXPECT(ValStatus::Current == harness.add(v)); - - // Re-adding violates the increasing seq requirement for full - // validations - BEAST_EXPECT(ValStatus::BadSeq == harness.add(v)); - - harness.clock().advance(1s); - - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerAB))); - - // Test the node changing signing key - - // Confirm old ledger on hand, but not new ledger - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerAB.id()) == 1); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerABC.id()) == 0); - - // Rotate signing keys - n.advanceKey(); - - harness.clock().advance(1s); - - // Cannot re-do the same full validation sequence - BEAST_EXPECT(ValStatus::Conflicting == harness.add(n.validate(ledgerAB))); - // Cannot send the same partial validation sequence - BEAST_EXPECT(ValStatus::Conflicting == harness.add(n.partial(ledgerAB))); - - // Now trusts the newest ledger too - harness.clock().advance(1s); - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerABC))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerAB.id()) == 1); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerABC.id()) == 1); - - // Processing validations out of order should ignore the older - // validation - harness.clock().advance(2s); - auto const valABCDE = n.validate(ledgerABCDE); - - harness.clock().advance(4s); - auto const valABCD = n.validate(ledgerABCD); - - BEAST_EXPECT(ValStatus::Current == harness.add(valABCD)); - - BEAST_EXPECT(ValStatus::Stale == harness.add(valABCDE)); - } - - { - // Process validations out of order with shifted times - - TestHarness harness(h.oracle); - Node const n = harness.makeNode(); - - // Establish a new current validation - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerA))); - - // Process a validation that has "later" seq but early sign time - BEAST_EXPECT(ValStatus::Stale == harness.add(n.validate(ledgerAB, -1s, -1s))); - - // Process a validation that has a later seq and later sign - // time - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerABC, 1s, 1s))); - } - - { - // Test stale on arrival validations - TestHarness harness(h.oracle); - Node const n = harness.makeNode(); - - BEAST_EXPECT( - ValStatus::Stale == - harness.add(n.validate(ledgerA, -harness.parms().validationCurrentEarly, 0s))); - - BEAST_EXPECT( - ValStatus::Stale == - harness.add(n.validate(ledgerA, harness.parms().validationCurrentWall, 0s))); - - BEAST_EXPECT( - ValStatus::Stale == - harness.add(n.validate(ledgerA, 0s, harness.parms().validationCurrentLocal))); - } - - { - // Test that full or partials cannot be sent for older sequence - // numbers, unless time-out has happened - for (bool doFull : {true, false}) - { - TestHarness harness(h.oracle); - Node n = harness.makeNode(); - - auto process = [&](Ledger& lgr) { - if (doFull) - return harness.add(n.validate(lgr)); - return harness.add(n.partial(lgr)); - }; - - BEAST_EXPECT(ValStatus::Current == process(ledgerABC)); - harness.clock().advance(1s); - BEAST_EXPECT(ledgerAB.seq() < ledgerABC.seq()); - BEAST_EXPECT(ValStatus::BadSeq == process(ledgerAB)); - - // If we advance far enough for AB to expire, we can fully - // validate or partially validate that sequence number again - BEAST_EXPECT(ValStatus::Conflicting == process(ledgerAZ)); - harness.clock().advance(harness.parms().validationSetExpires + 1ms); - BEAST_EXPECT(ValStatus::Current == process(ledgerAZ)); - } - } - } - - void - testOnStale() - { - testcase("Stale validation"); - // Verify validation becomes stale based solely on time passing, but - // use different functions to trigger the check for staleness - - LedgerHistoryHelper h; - Ledger ledgerA = h["a"]; - Ledger const ledgerAB = h["ab"]; - - using Trigger = std::function; - - std::vector const triggers = { - [&](TestValidations& vals) { vals.currentTrusted(); }, - [&](TestValidations& vals) { vals.getCurrentNodeIDs(); }, - [&](TestValidations& vals) { vals.getPreferred(genesisLedger_); }, - [&](TestValidations& vals) { vals.getNodesAfter(ledgerA, ledgerA.id()); }}; - for (Trigger const& trigger : triggers) - { - TestHarness harness(h.oracle); - Node const n = harness.makeNode(); - - BEAST_EXPECT(ValStatus::Current == harness.add(n.validate(ledgerAB))); - trigger(harness.vals()); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 1); - BEAST_EXPECT( - harness.vals().getPreferred(genesisLedger_) == - std::make_pair(ledgerAB.seq(), ledgerAB.id())); - harness.clock().advance(harness.parms().validationCurrentLocal); - - // trigger check for stale - trigger(harness.vals()); - - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 0); - BEAST_EXPECT(harness.vals().getPreferred(genesisLedger_) == std::nullopt); - } - } - - void - testGetNodesAfter() - { - // Test getting number of nodes working on a validation descending - // a prescribed one. This count should only be for trusted nodes, but - // includes partial and full validations - - using namespace std::chrono_literals; - testcase("Get nodes after"); - - LedgerHistoryHelper h; - Ledger const ledgerA = h["a"]; - Ledger const ledgerAB = h["ab"]; - Ledger const ledgerABC = h["abc"]; - Ledger const ledgerAD = h["ad"]; - - TestHarness harness(h.oracle); - Node const trustedNode1 = harness.makeNode(); - Node const trustedNode2 = harness.makeNode(); - Node const trustedNode3 = harness.makeNode(); - - Node notTrustedNode = harness.makeNode(); - notTrustedNode.untrust(); - - // first round a,b,c agree, d has is partial - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode1.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode2.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode3.partial(ledgerA))); - - for (Ledger const& ledger : {ledgerA, ledgerAB, ledgerABC, ledgerAD}) - BEAST_EXPECT(harness.vals().getNodesAfter(ledger, ledger.id()) == 0); - - harness.clock().advance(5s); - - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode1.validate(ledgerAB))); - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode2.validate(ledgerABC))); - BEAST_EXPECT(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerAB))); - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode3.partial(ledgerABC))); - - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 3); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerAB, ledgerAB.id()) == 2); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerABC, ledgerABC.id()) == 0); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerAD, ledgerAD.id()) == 0); - - // If given a ledger inconsistent with the id, is still able to check using slower method - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerAD, ledgerA.id()) == 1); - BEAST_EXPECT(harness.vals().getNodesAfter(ledgerAD, ledgerAB.id()) == 2); - } - - void - testCurrentTrusted() - { - using namespace std::chrono_literals; - testcase("Current trusted validations"); - - LedgerHistoryHelper h; - Ledger const ledgerA = h["a"]; - Ledger const ledgerB = h["b"]; - Ledger const ledgerAC = h["ac"]; - - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Node b = harness.makeNode(); - b.untrust(); - - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(b.validate(ledgerB))); - - // Only a is trusted - BEAST_EXPECT(harness.vals().currentTrusted().size() == 1); - BEAST_EXPECT(harness.vals().currentTrusted()[0].ledgerID() == ledgerA.id()); - BEAST_EXPECT(harness.vals().currentTrusted()[0].seq() == ledgerA.seq()); - - harness.clock().advance(3s); - - for (auto const& node : {a, b}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.validate(ledgerAC))); - - // New validation for a - BEAST_EXPECT(harness.vals().currentTrusted().size() == 1); - BEAST_EXPECT(harness.vals().currentTrusted()[0].ledgerID() == ledgerAC.id()); - BEAST_EXPECT(harness.vals().currentTrusted()[0].seq() == ledgerAC.seq()); - - // Pass enough time for it to go stale - harness.clock().advance(harness.parms().validationCurrentLocal); - BEAST_EXPECT(harness.vals().currentTrusted().empty()); - } - - void - testGetCurrentPublicKeys() - { - using namespace std::chrono_literals; - testcase("Current public keys"); - - LedgerHistoryHelper h; - Ledger const ledgerA = h["a"]; - Ledger const ledgerAC = h["ac"]; - - TestHarness harness(h.oracle); - Node a = harness.makeNode(), b = harness.makeNode(); - b.untrust(); - - for (auto const& node : {a, b}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.validate(ledgerA))); - - { - hash_set const expectedKeys = {a.nodeID(), b.nodeID()}; - BEAST_EXPECT(harness.vals().getCurrentNodeIDs() == expectedKeys); - } - - harness.clock().advance(3s); - - // Change keys and issue partials - a.advanceKey(); - b.advanceKey(); - - for (auto const& node : {a, b}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.partial(ledgerAC))); - - { - hash_set const expectedKeys = {a.nodeID(), b.nodeID()}; - BEAST_EXPECT(harness.vals().getCurrentNodeIDs() == expectedKeys); - } - - // Pass enough time for them to go stale - harness.clock().advance(harness.parms().validationCurrentLocal); - BEAST_EXPECT(harness.vals().getCurrentNodeIDs().empty()); - } - - void - testTrustedByLedgerFunctions() - { - // Test the Validations functions that calculate a value by ledger ID - using namespace std::chrono_literals; - testcase("By ledger functions"); - - // Several Validations functions return a set of values associated - // with trusted ledgers sharing the same ledger ID. The tests below - // exercise this logic by saving the set of trusted Validations, and - // verifying that the Validations member functions all calculate the - // proper transformation of the available ledgers. - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - - Node a = harness.makeNode(), b = harness.makeNode(), c = harness.makeNode(), - d = harness.makeNode(), e = harness.makeNode(); - - c.untrust(); - // Mix of load fees - a.setLoadFee(12); - b.setLoadFee(1); - c.setLoadFee(12); - e.setLoadFee(12); - - hash_map, std::vector> trustedValidations; - - //---------------------------------------------------------------------- - // checkers - auto sorted = [](auto vec) { - std::sort(vec.begin(), vec.end()); - return vec; - }; - auto compare = [&]() { - for (auto& it : trustedValidations) - { - auto const& id = it.first.first; - auto const& seq = it.first.second; - auto const& expectedValidations = it.second; - - BEAST_EXPECT(harness.vals().numTrustedForLedger(id) == expectedValidations.size()); - BEAST_EXPECT( - sorted(harness.vals().getTrustedForLedger(id, seq)) == - sorted(expectedValidations)); - - std::uint32_t const baseFee = 0; - std::vector expectedFees; - expectedFees.reserve(expectedValidations.size()); - for (auto const& val : expectedValidations) - { - expectedFees.push_back(val.loadFee().value_or(baseFee)); - } - - BEAST_EXPECT(sorted(harness.vals().fees(id, baseFee)) == sorted(expectedFees)); - } - }; - - //---------------------------------------------------------------------- - Ledger const ledgerA = h["a"]; - Ledger const ledgerB = h["b"]; - Ledger const ledgerAC = h["ac"]; - - // Add a dummy ID to cover unknown ledger identifiers - trustedValidations[{Ledger::ID{100}, Ledger::Seq{100}}] = {}; - - // first round a,b,c agree - for (auto const& node : {a, b, c}) - { - auto const val = node.validate(ledgerA); - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - if (val.trusted()) - trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); - } - // d disagrees - { - auto const val = d.validate(ledgerB); - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); - } - // e only issues partials - { - BEAST_EXPECT(ValStatus::Current == harness.add(e.partial(ledgerA))); - } - - harness.clock().advance(5s); - // second round, a,b,c move to ledger 2 - for (auto const& node : {a, b, c}) - { - auto const val = node.validate(ledgerAC); - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - if (val.trusted()) - trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); - } - // d now thinks ledger 1, but cannot re-issue a previously used seq - // and attempting it should generate a conflict. - { - BEAST_EXPECT(ValStatus::Conflicting == harness.add(d.partial(ledgerA))); - } - // e only issues partials - { - BEAST_EXPECT(ValStatus::Current == harness.add(e.partial(ledgerAC))); - } - - compare(); - } - - void - testExpire() - { - // Verify expiring clears out validations stored by ledger - testcase("Expire validations"); - SuiteJournal j("Validations_test", *this); - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - constexpr Ledger::Seq kOne(1); - constexpr Ledger::Seq kTwo(2); - - // simple cases - Ledger const ledgerA = h["a"]; - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerA))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); - harness.vals().expire(j); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); - harness.clock().advance(harness.parms().validationSetExpires); - harness.vals().expire(j); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 0); - - // use setSeqToKeep to keep the validation from expire - Ledger const ledgerB = h["ab"]; - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerB))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); - harness.vals().setSeqToKeep(ledgerB.seq(), ledgerB.seq() + kOne); - harness.clock().advance(harness.parms().validationSetExpires); - harness.vals().expire(j); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); - // change toKeep - harness.vals().setSeqToKeep(ledgerB.seq() + kOne, ledgerB.seq() + kTwo); - // advance clock slowly - int const loops = - harness.parms().validationSetExpires / harness.parms().validationFRESHNESS + 1; - for (int i = 0; i < loops; ++i) - { - harness.clock().advance(harness.parms().validationFRESHNESS); - harness.vals().expire(j); - } - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerB.id()) == 0); - - // Allow the validation with high seq to expire - Ledger const ledgerC = h["abc"]; - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerC))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerC.id()) == 1); - harness.vals().setSeqToKeep(ledgerC.seq() - kOne, ledgerC.seq()); - harness.clock().advance(harness.parms().validationSetExpires); - harness.vals().expire(j); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerC.id()) == 0); - } - - void - testFlush() - { - // Test final flush of validations - using namespace std::chrono_literals; - testcase("Flush validations"); - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const trustedNode1 = harness.makeNode(); - Node const trustedNode2 = harness.makeNode(); - Node notTrustedNode = harness.makeNode(); - notTrustedNode.untrust(); - - Ledger const ledgerA = h["a"]; - Ledger const ledgerAB = h["ab"]; - - hash_map expected; - for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode}) - { - auto const val = node.validate(ledgerA); - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - expected.emplace(node.nodeID(), val); - } - - // Send in a new validation for a, saving the new one into the expected - // map after setting the proper prior ledger ID it replaced - harness.clock().advance(1s); - auto newVal = trustedNode1.validate(ledgerAB); - BEAST_EXPECT(ValStatus::Current == harness.add(newVal)); - expected.find(trustedNode1.nodeID())->second = newVal; - } - - void - testGetPreferredLedger() - { - using namespace std::chrono_literals; - testcase("Preferred Ledger"); - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const trustedNode1 = harness.makeNode(); - Node const trustedNode2 = harness.makeNode(); - Node const trustedNode3 = harness.makeNode(); - - Node notTrustedNode = harness.makeNode(); - notTrustedNode.untrust(); - - Ledger const ledgerA = h["a"]; - Ledger const ledgerB = h["b"]; - Ledger const ledgerAC = h["ac"]; - Ledger const ledgerACD = h["acd"]; - - using Seq = Ledger::Seq; - - auto pref = [](Ledger ledger) { return std::make_pair(ledger.seq(), ledger.id()); }; - - // Empty (no ledgers) - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == std::nullopt); - - // Single ledger - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode1.validate(ledgerB))); - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == pref(ledgerB)); - BEAST_EXPECT(harness.vals().getPreferred(ledgerB) == pref(ledgerB)); - - // Minimum valid sequence - BEAST_EXPECT(harness.vals().getPreferred(ledgerA, Seq{10}) == ledgerA.id()); - - // Untrusted doesn't impact preferred ledger - // (ledgerB has tie-break over ledgerA) - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode2.validate(ledgerA))); - BEAST_EXPECT(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerA))); - BEAST_EXPECT(ledgerB.id() > ledgerA.id()); - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == pref(ledgerB)); - BEAST_EXPECT(harness.vals().getPreferred(ledgerB) == pref(ledgerB)); - - // Partial does break ties - BEAST_EXPECT(ValStatus::Current == harness.add(trustedNode3.partial(ledgerA))); - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == pref(ledgerA)); - BEAST_EXPECT(harness.vals().getPreferred(ledgerB) == pref(ledgerA)); - - harness.clock().advance(5s); - - // Parent of preferred-> stick with ledger - for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode, trustedNode3}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.validate(ledgerAC))); - // Parent of preferred stays put - BEAST_EXPECT(harness.vals().getPreferred(ledgerA) == pref(ledgerA)); - // Earlier different chain, switch - BEAST_EXPECT(harness.vals().getPreferred(ledgerB) == pref(ledgerAC)); - // Later on chain, stays where it is - BEAST_EXPECT(harness.vals().getPreferred(ledgerACD) == pref(ledgerACD)); - - // Any later grandchild or different chain is preferred - harness.clock().advance(5s); - for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode, trustedNode3}) - BEAST_EXPECT(ValStatus::Current == harness.add(node.validate(ledgerACD))); - for (auto const& ledger : {ledgerA, ledgerB, ledgerACD}) - BEAST_EXPECT(harness.vals().getPreferred(ledger) == pref(ledgerACD)); - } - - void - testGetPreferredLCL() - { - using namespace std::chrono_literals; - testcase("Get preferred LCL"); - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - - Ledger const ledgerA = h["a"]; - Ledger const ledgerB = h["b"]; - Ledger const ledgerC = h["c"]; - - using ID = Ledger::ID; - using Seq = Ledger::Seq; - - hash_map peerCounts; - - // No trusted validations or counts sticks with current ledger - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerA.id()); - - ++peerCounts[ledgerB.id()]; - - // No trusted validations, rely on peer counts - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerB.id()); - - ++peerCounts[ledgerC.id()]; - // No trusted validations, tied peers goes with larger ID - BEAST_EXPECT(ledgerC.id() > ledgerB.id()); - - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerC.id()); - - peerCounts[ledgerC.id()] += 1000; - - // Single trusted always wins over peer counts - BEAST_EXPECT(ValStatus::Current == harness.add(a.validate(ledgerA))); - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerA.id()); - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerB, Seq{0}, peerCounts) == ledgerA.id()); - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerC, Seq{0}, peerCounts) == ledgerA.id()); - - // Stick with current ledger if trusted validation ledger has too old - // of a sequence - BEAST_EXPECT(harness.vals().getPreferredLCL(ledgerB, Seq{2}, peerCounts) == ledgerB.id()); - } - - void - testAcquireValidatedLedger() - { - using namespace std::chrono_literals; - testcase("Acquire validated ledger"); - - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Node const b = harness.makeNode(); - - using ID = Ledger::ID; - using Seq = Ledger::Seq; - - // Validate the ledger before it is actually available - Validation const val = a.validate(ID{2}, Seq{2}, 0s, 0s, true); - - BEAST_EXPECT(ValStatus::Current == harness.add(val)); - // Validation is available - BEAST_EXPECT(harness.vals().numTrustedForLedger(ID{2}) == 1); - // but ledger based data is not - BEAST_EXPECT(harness.vals().getNodesAfter(genesisLedger_, ID{0}) == 0); - // Initial preferred branch falls back to the ledger we are trying to - // acquire - BEAST_EXPECT(harness.vals().getPreferred(genesisLedger_) == std::make_pair(Seq{2}, ID{2})); - - // After adding another unavailable validation, the preferred ledger - // breaks ties via higher ID - BEAST_EXPECT(ValStatus::Current == harness.add(b.validate(ID{3}, Seq{2}, 0s, 0s, true))); - BEAST_EXPECT(harness.vals().getPreferred(genesisLedger_) == std::make_pair(Seq{2}, ID{3})); - - // Create the ledger - Ledger const ledgerAB = h["ab"]; - // Now it should be available - BEAST_EXPECT(harness.vals().getNodesAfter(genesisLedger_, ID{0}) == 1); - - // Create a validation that is not available - harness.clock().advance(5s); - Validation const val2 = a.validate(ID{4}, Seq{4}, 0s, 0s, true); - BEAST_EXPECT(ValStatus::Current == harness.add(val2)); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ID{4}) == 1); - BEAST_EXPECT( - harness.vals().getPreferred(genesisLedger_) == - std::make_pair(ledgerAB.seq(), ledgerAB.id())); - - // Another node requesting that ledger still doesn't change things - Validation const val3 = b.validate(ID{4}, Seq{4}, 0s, 0s, true); - BEAST_EXPECT(ValStatus::Current == harness.add(val3)); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ID{4}) == 2); - BEAST_EXPECT( - harness.vals().getPreferred(genesisLedger_) == - std::make_pair(ledgerAB.seq(), ledgerAB.id())); - - // Switch to validation that is available - harness.clock().advance(5s); - Ledger const ledgerABCDE = h["abcde"]; - BEAST_EXPECT(ValStatus::Current == harness.add(a.partial(ledgerABCDE))); - BEAST_EXPECT(ValStatus::Current == harness.add(b.partial(ledgerABCDE))); - BEAST_EXPECT( - harness.vals().getPreferred(genesisLedger_) == - std::make_pair(ledgerABCDE.seq(), ledgerABCDE.id())); - } - - void - testNumTrustedForLedger() - { - testcase("NumTrustedForLedger"); - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Node const b = harness.makeNode(); - Ledger const ledgerA = h["a"]; - - BEAST_EXPECT(ValStatus::Current == harness.add(a.partial(ledgerA))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 0); - - BEAST_EXPECT(ValStatus::Current == harness.add(b.validate(ledgerA))); - BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); - } - - void - testSeqEnforcer() - { - testcase("SeqEnforcer"); - using Seq = Ledger::Seq; - using namespace std::chrono; - - beast::ManualClock clock; - SeqEnforcer enforcer; - - ValidationParms const p; - - BEAST_EXPECT(enforcer(clock.now(), Seq{1}, p)); - BEAST_EXPECT(enforcer(clock.now(), Seq{10}, p)); - BEAST_EXPECT(!enforcer(clock.now(), Seq{5}, p)); - BEAST_EXPECT(!enforcer(clock.now(), Seq{9}, p)); - clock.advance(p.validationSetExpires - 1ms); - BEAST_EXPECT(!enforcer(clock.now(), Seq{1}, p)); - clock.advance(2ms); - BEAST_EXPECT(enforcer(clock.now(), Seq{1}, p)); - } - - void - testTrustChanged() - { - testcase("TrustChanged"); - using namespace std::chrono; - - auto checker = [this]( - TestValidations& vals, - hash_set const& listed, - std::vector const& trustedVals) { - Ledger::ID const testID = - trustedVals.empty() ? this->genesisLedger_.id() : trustedVals[0].ledgerID(); - Ledger::Seq const testSeq = - trustedVals.empty() ? this->genesisLedger_.seq() : trustedVals[0].seq(); - BEAST_EXPECT(vals.currentTrusted() == trustedVals); - BEAST_EXPECT(vals.getCurrentNodeIDs() == listed); - BEAST_EXPECT( - vals.getNodesAfter(this->genesisLedger_, genesisLedger_.id()) == - trustedVals.size()); - if (trustedVals.empty()) - { - BEAST_EXPECT(vals.getPreferred(this->genesisLedger_) == std::nullopt); - } - else - { - BEAST_EXPECT(vals.getPreferred(this->genesisLedger_)->second == testID); - } - BEAST_EXPECT(vals.getTrustedForLedger(testID, testSeq) == trustedVals); - BEAST_EXPECT(vals.numTrustedForLedger(testID) == trustedVals.size()); - }; - - { - // Trusted to untrusted - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Ledger const ledgerAB = h["ab"]; - Validation const v = a.validate(ledgerAB); - BEAST_EXPECT(ValStatus::Current == harness.add(v)); - - hash_set const listed({a.nodeID()}); - std::vector trustedVals({v}); - checker(harness.vals(), listed, trustedVals); - - trustedVals.clear(); - harness.vals().trustChanged({}, {a.nodeID()}); - checker(harness.vals(), listed, trustedVals); - } - - { - // Untrusted to trusted - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node a = harness.makeNode(); - a.untrust(); - Ledger const ledgerAB = h["ab"]; - Validation const v = a.validate(ledgerAB); - BEAST_EXPECT(ValStatus::Current == harness.add(v)); - - hash_set const listed({a.nodeID()}); - std::vector trustedVals; - checker(harness.vals(), listed, trustedVals); - - trustedVals.push_back(v); - harness.vals().trustChanged({a.nodeID()}, {}); - checker(harness.vals(), listed, trustedVals); - } - - { - // Trusted but not acquired -> untrusted - LedgerHistoryHelper h; - TestHarness harness(h.oracle); - Node const a = harness.makeNode(); - Validation const v = a.validate(Ledger::ID{2}, Ledger::Seq{2}, 0s, 0s, true); - BEAST_EXPECT(ValStatus::Current == harness.add(v)); - - hash_set const listed({a.nodeID()}); - std::vector trustedVals({v}); - auto& vals = harness.vals(); - BEAST_EXPECT(vals.currentTrusted() == trustedVals); - - // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - BEAST_EXPECT(vals.getPreferred(genesisLedger_)->second == v.ledgerID()); - BEAST_EXPECT(vals.getNodesAfter(genesisLedger_, genesisLedger_.id()) == 0); - - trustedVals.clear(); - harness.vals().trustChanged({}, {a.nodeID()}); - // make acquiring ledger available - h["ab"]; - BEAST_EXPECT(vals.currentTrusted() == trustedVals); - BEAST_EXPECT(vals.getPreferred(genesisLedger_) == std::nullopt); - BEAST_EXPECT(vals.getNodesAfter(genesisLedger_, genesisLedger_.id()) == 0); - } - } - - void - run() override - { - testAddValidation(); - testOnStale(); - testGetNodesAfter(); - testCurrentTrusted(); - testGetCurrentPublicKeys(); - testTrustedByLedgerFunctions(); - testExpire(); - testFlush(); - testGetPreferredLedger(); - testGetPreferredLCL(); - testAcquireValidatedLedger(); - testNumTrustedForLedger(); - testSeqEnforcer(); - testTrustChanged(); - } -}; - -BEAST_DEFINE_TESTSUITE(Validations, consensus, xrpl); -} // namespace xrpl::test::csf diff --git a/src/test/csf/BasicNetwork_test.cpp b/src/test/csf/BasicNetwork_test.cpp deleted file mode 100644 index 22f52a1b63..0000000000 --- a/src/test/csf/BasicNetwork_test.cpp +++ /dev/null @@ -1,134 +0,0 @@ -#include -#include - -#include - -#include -#include - -namespace xrpl::test { - -class BasicNetwork_test : public beast::unit_test::Suite -{ -public: - struct Peer - { - int id; - std::set set; - - Peer(Peer const&) = default; - Peer(Peer&&) = default; - - explicit Peer(int id) : id(id) - { - } - - template - void - start(csf::Scheduler& scheduler, Net& net) - { - using namespace std::chrono_literals; - auto t = scheduler.in(1s, [&] { set.insert(0); }); - if (id == 0) - { - for (auto const link : net.links(this)) - { - net.send( - this, link.target, [&, to = link.target] { to->receive(net, this, 1); }); - } - } - else - { - scheduler.cancel(t); - } - } - - template - void - receive(Net& net, Peer* from, int m) - { - set.insert(m); - ++m; - if (m < 5) - { - for (auto const link : net.links(this)) - { - net.send(this, link.target, [&, mm = m, to = link.target] { - to->receive(net, this, mm); - }); - } - } - } - }; - - void - testNetwork() - { - using namespace std::chrono_literals; - std::vector pv; - pv.emplace_back(0); - pv.emplace_back(1); - pv.emplace_back(2); - csf::Scheduler scheduler; - csf::BasicNetwork net(scheduler); - BEAST_EXPECT(!net.connect(&pv[0], &pv[0])); - BEAST_EXPECT(net.connect(&pv[0], &pv[1], 1s)); - BEAST_EXPECT(net.connect(&pv[1], &pv[2], 1s)); - BEAST_EXPECT(!net.connect(&pv[0], &pv[1])); - for (auto& peer : pv) - peer.start(scheduler, net); - BEAST_EXPECT(scheduler.stepFor(0s)); - BEAST_EXPECT(scheduler.stepFor(1s)); - BEAST_EXPECT(scheduler.step()); - BEAST_EXPECT(!scheduler.step()); - BEAST_EXPECT(!scheduler.stepFor(1s)); - net.send(&pv[0], &pv[1], [] {}); - net.send(&pv[1], &pv[0], [] {}); - BEAST_EXPECT(net.disconnect(&pv[0], &pv[1])); - BEAST_EXPECT(!net.disconnect(&pv[0], &pv[1])); - for (;;) - { - auto const links = net.links(&pv[1]); - if (links.empty()) - break; - BEAST_EXPECT(net.disconnect(&pv[1], links[0].target)); - } - BEAST_EXPECT(pv[0].set == std::set({0, 2, 4})); - BEAST_EXPECT(pv[1].set == std::set({1, 3})); - BEAST_EXPECT(pv[2].set == std::set({2, 4})); - } - - void - testDisconnect() - { - using namespace std::chrono_literals; - csf::Scheduler scheduler; - csf::BasicNetwork net(scheduler); - BEAST_EXPECT(net.connect(0, 1, 1s)); - BEAST_EXPECT(net.connect(0, 2, 2s)); - - std::set delivered; - net.send(0, 1, [&]() { delivered.insert(1); }); - net.send(0, 2, [&]() { delivered.insert(2); }); - - scheduler.in(1000ms, [&]() { BEAST_EXPECT(net.disconnect(0, 2)); }); - scheduler.in(1100ms, [&]() { BEAST_EXPECT(net.connect(0, 2)); }); - - scheduler.step(); - - // only the first message is delivered because the disconnect at 1 s - // purges all pending messages from 0 to 2 - BEAST_EXPECT(delivered == std::set({1})); - } - - void - run() override - { - testNetwork(); - testDisconnect(); - } -}; - -BEAST_DEFINE_TESTSUITE(BasicNetwork, csf, xrpl); - -} // namespace xrpl::test diff --git a/src/test/csf/Digraph_test.cpp b/src/test/csf/Digraph_test.cpp deleted file mode 100644 index 40bfafde9c..0000000000 --- a/src/test/csf/Digraph_test.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#include - -#include - -#include -#include -#include -#include - -namespace xrpl::test { - -class Digraph_test : public beast::unit_test::Suite -{ -public: - void - run() override - { - using namespace csf; - using Graph = Digraph; - Graph graph; - - BEAST_EXPECT(!graph.connected('a', 'b')); - BEAST_EXPECT(!graph.edge('a', 'b')); - BEAST_EXPECT(!graph.disconnect('a', 'b')); - - BEAST_EXPECT(graph.connect('a', 'b', "foobar")); - BEAST_EXPECT(graph.connected('a', 'b')); - BEAST_EXPECT( - *graph.edge('a', 'b') == "foobar"); // NOLINT(bugprone-unchecked-optional-access) - - BEAST_EXPECT(!graph.connect('a', 'b', "repeat")); - BEAST_EXPECT(graph.disconnect('a', 'b')); - BEAST_EXPECT(graph.connect('a', 'b', "repeat")); - BEAST_EXPECT(graph.connected('a', 'b')); - BEAST_EXPECT( - *graph.edge('a', 'b') == "repeat"); // NOLINT(bugprone-unchecked-optional-access) - - BEAST_EXPECT(graph.connect('a', 'c', "tree")); - - { - std::vector> edges; - - for (auto const& edge : graph.outEdges('a')) - { - edges.emplace_back(edge.source, edge.target, edge.data); - } - - std::vector> expected; - expected.emplace_back('a', 'b', "repeat"); - expected.emplace_back('a', 'c', "tree"); - BEAST_EXPECT(edges == expected); - BEAST_EXPECT(graph.outDegree('a') == expected.size()); - } - - BEAST_EXPECT(graph.outEdges('r').size() == 0); - BEAST_EXPECT(graph.outDegree('r') == 0); - BEAST_EXPECT(graph.outDegree('c') == 0); - - // only 'a' has out edges - BEAST_EXPECT(graph.outVertices().size() == 1); - std::vector const expected = {'b', 'c'}; - - BEAST_EXPECT((graph.outVertices('a') == expected)); - BEAST_EXPECT(graph.outVertices('b').size() == 0); - BEAST_EXPECT(graph.outVertices('c').size() == 0); - BEAST_EXPECT(graph.outVertices('r').size() == 0); - - std::stringstream ss; - graph.saveDot(ss, [](char v) { return v; }); - std::string const expectedDot = - "digraph {\n" - "a -> b;\n" - "a -> c;\n" - "}\n"; - BEAST_EXPECT(ss.str() == expectedDot); - } -}; - -BEAST_DEFINE_TESTSUITE(Digraph, csf, xrpl); - -} // namespace xrpl::test diff --git a/src/test/csf/Histogram_test.cpp b/src/test/csf/Histogram_test.cpp deleted file mode 100644 index 65edb14e5d..0000000000 --- a/src/test/csf/Histogram_test.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include - -#include - -namespace xrpl::test { - -class Histogram_test : public beast::unit_test::Suite -{ -public: - void - run() override - { - using namespace csf; - Histogram hist; - - BEAST_EXPECT(hist.size() == 0); - BEAST_EXPECT(hist.numBins() == 0); - BEAST_EXPECT(hist.minValue() == 0); - BEAST_EXPECT(hist.maxValue() == 0); - BEAST_EXPECT(hist.avg() == 0); - BEAST_EXPECT(hist.percentile(0.0f) == hist.minValue()); - BEAST_EXPECT(hist.percentile(0.5f) == 0); - BEAST_EXPECT(hist.percentile(0.9f) == 0); - BEAST_EXPECT(hist.percentile(1.0f) == hist.maxValue()); - - hist.insert(1); - - BEAST_EXPECT(hist.size() == 1); - BEAST_EXPECT(hist.numBins() == 1); - BEAST_EXPECT(hist.minValue() == 1); - BEAST_EXPECT(hist.maxValue() == 1); - BEAST_EXPECT(hist.avg() == 1); - BEAST_EXPECT(hist.percentile(0.0f) == hist.minValue()); - BEAST_EXPECT(hist.percentile(0.5f) == 1); - BEAST_EXPECT(hist.percentile(0.9f) == 1); - BEAST_EXPECT(hist.percentile(1.0f) == hist.maxValue()); - - hist.insert(9); - - BEAST_EXPECT(hist.size() == 2); - BEAST_EXPECT(hist.numBins() == 2); - BEAST_EXPECT(hist.minValue() == 1); - BEAST_EXPECT(hist.maxValue() == 9); - BEAST_EXPECT(hist.avg() == 5); - BEAST_EXPECT(hist.percentile(0.0f) == hist.minValue()); - BEAST_EXPECT(hist.percentile(0.5f) == 1); - BEAST_EXPECT(hist.percentile(0.9f) == 9); - BEAST_EXPECT(hist.percentile(1.0f) == hist.maxValue()); - - hist.insert(1); - - BEAST_EXPECT(hist.size() == 3); - BEAST_EXPECT(hist.numBins() == 2); - BEAST_EXPECT(hist.minValue() == 1); - BEAST_EXPECT(hist.maxValue() == 9); - BEAST_EXPECT(hist.avg() == 11 / 3); - BEAST_EXPECT(hist.percentile(0.0f) == hist.minValue()); - BEAST_EXPECT(hist.percentile(0.5f) == 1); - BEAST_EXPECT(hist.percentile(0.9f) == 9); - BEAST_EXPECT(hist.percentile(1.0f) == hist.maxValue()); - } -}; - -BEAST_DEFINE_TESTSUITE(Histogram, csf, xrpl); - -} // namespace xrpl::test diff --git a/src/test/csf/Scheduler_test.cpp b/src/test/csf/Scheduler_test.cpp deleted file mode 100644 index 6f4ac051c4..0000000000 --- a/src/test/csf/Scheduler_test.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include - -#include - -#include - -namespace xrpl::test { - -class Scheduler_test : public beast::unit_test::Suite -{ -public: - void - run() override - { - using namespace std::chrono_literals; - csf::Scheduler scheduler; - std::set seen; - - scheduler.in(1s, [&] { seen.insert(1); }); - scheduler.in(2s, [&] { seen.insert(2); }); - auto token = scheduler.in(3s, [&] { seen.insert(3); }); - scheduler.at(scheduler.now() + 4s, [&] { seen.insert(4); }); - scheduler.at(scheduler.now() + 8s, [&] { seen.insert(8); }); - - auto start = scheduler.now(); - - // Process first event - BEAST_EXPECT(seen.empty()); - BEAST_EXPECT(scheduler.stepOne()); - BEAST_EXPECT(seen == std::set({1})); - BEAST_EXPECT(scheduler.now() == (start + 1s)); - - // No processing if stepping until current time - BEAST_EXPECT(scheduler.stepUntil(scheduler.now())); - BEAST_EXPECT(seen == std::set({1})); - BEAST_EXPECT(scheduler.now() == (start + 1s)); - - // Process next event - BEAST_EXPECT(scheduler.stepFor(1s)); - BEAST_EXPECT(seen == std::set({1, 2})); - BEAST_EXPECT(scheduler.now() == (start + 2s)); - - // Don't process cancelled event, but advance clock - scheduler.cancel(token); - BEAST_EXPECT(scheduler.stepFor(1s)); - BEAST_EXPECT(seen == std::set({1, 2})); - BEAST_EXPECT(scheduler.now() == (start + 3s)); - - // Process until 3 seen ints - BEAST_EXPECT(scheduler.stepWhile([&]() { return seen.size() < 3; })); - BEAST_EXPECT(seen == std::set({1, 2, 4})); - BEAST_EXPECT(scheduler.now() == (start + 4s)); - - // Process the rest - BEAST_EXPECT(scheduler.step()); - BEAST_EXPECT(seen == std::set({1, 2, 4, 8})); - BEAST_EXPECT(scheduler.now() == (start + 8s)); - - // Process the rest again doesn't advance - BEAST_EXPECT(!scheduler.step()); - BEAST_EXPECT(seen == std::set({1, 2, 4, 8})); - BEAST_EXPECT(scheduler.now() == (start + 8s)); - } -}; - -BEAST_DEFINE_TESTSUITE(Scheduler, csf, xrpl); - -} // namespace xrpl::test diff --git a/src/test/jtx/AbstractClient.h b/src/test/jtx/AbstractClient.h index f9d8de0768..58f57f67a5 100644 --- a/src/test/jtx/AbstractClient.h +++ b/src/test/jtx/AbstractClient.h @@ -40,6 +40,17 @@ public: */ [[nodiscard]] virtual unsigned version() const = 0; + + /** + * Close the client's connection to the server. + * + * Releases the connection the client holds against the server's per-port + * connection limit. After this call the client must not be used to + * invoke() again. Tests use this to deterministically free the slot + * rather than waiting for the server's idle timeout to drop it. + */ + virtual void + disconnect() = 0; }; } // namespace xrpl::test diff --git a/src/test/jtx/Env.h b/src/test/jtx/Env.h index 7e22cdd571..a175fd5006 100644 --- a/src/test/jtx/Env.h +++ b/src/test/jtx/Env.h @@ -482,11 +482,52 @@ public: app().getNumberOfThreads() == 1, "syncClose() is only useful on an application with a single thread"); auto const result = close(); - auto serverBarrier = std::make_shared>(); - auto future = serverBarrier->get_future(); - boost::asio::post(app().getIOContext(), [serverBarrier]() { serverBarrier->set_value(); }); - auto const status = future.wait_for(timeout); - return result && status == std::future_status::ready; + return result && drainServerIo(timeout); + } + + /** + * Disconnect the Env's built-in client and wait for the server to + * register the dropped connection. + * + * Env holds one persistent client connection to the server's RPC port for + * its whole lifetime (see client()), and that connection counts against + * the port's connection limit. Tests that need a known starting occupancy + * can call this to deterministically release that slot instead of waiting + * out the server's localhost idle timeout. + * + * The server decrements its per-port connection count in the peer's + * destructor, which runs when the io_context processes the end-of-stream + * on the closed socket. After closing the client this drains the server's + * io_context twice: the first barrier guarantees the reactor has reaped + * the closed socket and queued the peer's teardown, and the second + * guarantees that teardown (and therefore the count decrement) has run. + * + * This is only sound when the server uses a single io_context thread, so + * that draining establishes ordering against the teardown - configure the + * Env with singleThreadIo() (as syncClose() also requires). Like + * syncClose(), it relies on loopback teardown latency being negligible. + * + * @param timeout Maximum time to wait for each barrier task to execute + * @return true if both barriers executed within timeout, false otherwise + */ + [[nodiscard]] bool + disconnectClient(std::chrono::steady_clock::duration timeout = std::chrono::seconds{1}) + { + XRPL_ASSERT( + app().getNumberOfThreads() == 1, + "disconnectClient() is only useful on an application with a single " + "thread"); + + bundle_.client->disconnect(); + + // Drain the server's single io thread twice: the first barrier flushes + // the reactor's reap of the closed socket (queuing the peer teardown), + // the second flushes that teardown - and therefore the connection-count + // decrement. Both run unconditionally so a timed-out first drain does + // not short-circuit the second. + bool const reaped = drainServerIo(timeout); + bool const toreDown = drainServerIo(timeout); + return reaped && toreDown; } /** @@ -514,6 +555,49 @@ public: parseFailureExpected_ = b; } + /** + * RAII class to set and restore the parse failure flag (setParseFailureExpected). + * + * Can be created directly, or through the `getParseFailureGuard(bool)` function. + */ + class ParseFailureGuard final + { + Env& self_; + bool const oldExpected_; + + public: + ParseFailureGuard(Env& self, bool b) + : self_(self), oldExpected_(self_.parseFailureExpected_) + { + self_.setParseFailureExpected(b); + } + + ~ParseFailureGuard() + { + self_.setParseFailureExpected(oldExpected_); + } + + // No copy, no move + ParseFailureGuard(ParseFailureGuard const&) = delete; + ParseFailureGuard& + operator=(ParseFailureGuard const&) = delete; + ParseFailureGuard(ParseFailureGuard&& other) = delete; + ParseFailureGuard& + operator=(ParseFailureGuard&&) = delete; + }; + + /** + * Gets an RAII guard to set and restore the parse failure flag + * + * Usage: + * auto const guard = env.getParseFailureGuard(true/false); + */ + [[nodiscard]] ParseFailureGuard + getParseFailureGuard(bool b) + { + return ParseFailureGuard{*this, b}; + } + /** * Turn off signature checks. */ @@ -803,6 +887,25 @@ public: } private: + /** + * Drain the (single) server io_context thread once. + * + * Posts a barrier task to the server's io_context and blocks until it + * runs, so every task queued before it has been processed. Only meaningful + * with a single io thread (see syncClose()/disconnectClient()). + * + * @param timeout Maximum time to wait for the barrier task to execute + * @return true if the barrier ran within timeout, false otherwise + */ + [[nodiscard]] bool + drainServerIo(std::chrono::steady_clock::duration timeout) + { + auto barrier = std::make_shared>(); + auto future = barrier->get_future(); + boost::asio::post(app().getIOContext(), [barrier]() { barrier->set_value(); }); + return future.wait_for(timeout) == std::future_status::ready; + } + void fund(bool setDefaultRipple, STAmount const& amount, Account const& account); diff --git a/src/test/jtx/WSClient_test.cpp b/src/test/jtx/WSClient_test.cpp index d77e0f948b..801ca50504 100644 --- a/src/test/jtx/WSClient_test.cpp +++ b/src/test/jtx/WSClient_test.cpp @@ -13,8 +13,9 @@ class WSClient_test : public beast::unit_test::Suite { public: void - run() override + testSmoke() { + testcase("smoke"); using namespace jtx; Env env(*this); auto wsc = makeWSClient(env.app().config()); @@ -28,6 +29,47 @@ public: auto jv = wsc->getMsg(std::chrono::seconds(1)); pass(); } + + void + testGracefulDisconnect() + { + testcase("graceful disconnect"); + using namespace jtx; + using namespace std::chrono; + + Env env(*this); + auto wsc = makeWSClient(env.app().config()); + + // Put real traffic on the connection before closing it. + json::Value stream; + stream["streams"] = json::ValueType::Array; + stream["streams"].append("ledger"); + auto const sub = wsc->invoke("subscribe", stream); + BEAST_EXPECT(sub.isMember("result") || sub.isMember("status")); + + // disconnect() performs a graceful WebSocket closing handshake and + // blocks until the server acknowledges. On loopback that completes in + // well under its internal 1s timeout; only a broken async_close/ack + // coordination would fall through to the force-close path at ~1s. A + // generous bound keeps this from flaking under load while still + // catching that regression. + auto const start = steady_clock::now(); + wsc->disconnect(); + auto const elapsed = duration_cast(steady_clock::now() - start); + BEAST_EXPECT(elapsed < milliseconds{750}); + + // disconnect() must be idempotent: a second call (and the subsequent + // destructor) must not hang, double-close, or crash. + wsc->disconnect(); + pass(); + } + + void + run() override + { + testSmoke(); + testGracefulDisconnect(); + } }; BEAST_DEFINE_TESTSUITE(WSClient, jtx, xrpl); diff --git a/src/test/jtx/impl/Env.cpp b/src/test/jtx/impl/Env.cpp index 3f6aca9fcb..4da2e2b521 100644 --- a/src/test/jtx/impl/Env.cpp +++ b/src/test/jtx/impl/Env.cpp @@ -629,7 +629,7 @@ Env::autofill(JTx& jt) catch (ParseError const&) { if (!parseFailureExpected_) - test.log << "parse failed:\n" << pretty(jv) << std::endl; + test.log << "parse failure:\n" << pretty(jv) << std::endl; rethrow(); } } diff --git a/src/test/jtx/impl/JSONRPCClient.cpp b/src/test/jtx/impl/JSONRPCClient.cpp index 495fc5a657..06474c3616 100644 --- a/src/test/jtx/impl/JSONRPCClient.cpp +++ b/src/test/jtx/impl/JSONRPCClient.cpp @@ -14,18 +14,23 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include #include #include #include +#include +#include +#include #include #include #include @@ -84,6 +89,40 @@ class JSONRPCClient : public AbstractClient boost::beast::multi_buffer bout_; unsigned rpcVersion_; + bool disconnected_ = false; + + // Errors that mean the persistent keep-alive connection was dropped by the + // server (rather than a genuine protocol failure), so the request can be + // safely retried on a fresh connection. + static bool + droppedConnection(boost::system::error_code const& ec) + { + namespace error = boost::asio::error; + static auto const kDroppedConnectionErrors = std::to_array({ + boost::beast::http::error::end_of_stream, + error::eof, + error::connection_reset, + error::connection_aborted, + error::broken_pipe, + error::not_connected, + }); + + return std::ranges::any_of( + kDroppedConnectionErrors, + [&ec](boost::system::error_code const& e) { return ec == e; }); + } + + // Tear down and re-establish the socket to ep_, discarding any buffered + // bytes left over from the dropped connection. + void + reconnect() + { + boost::system::error_code ec; + stream_.close(ec); + bin_.clear(); + stream_.connect(ep_); + } + public: explicit JSONRPCClient(Config const& cfg, unsigned rpcVersion) : ep_(getEndpoint(cfg)), stream_(ios_), rpcVersion_(rpcVersion) @@ -91,12 +130,10 @@ public: stream_.connect(ep_); } - /* - Return value is an Object type with up to three keys: - status - error - result - */ + // Return value is an Object type with up to three keys: + // status + // error + // result json::Value invoke(std::string const& cmd, json::Value const& params) override { @@ -104,6 +141,13 @@ public: using namespace boost::asio; using namespace std::string_literals; + // Once disconnect() has released the slot, the client must not be + // reused (see AbstractClient::disconnect). Refuse rather than let the + // failed write/read below trip the reconnect path and silently + // re-consume a connection slot, which would defeat disconnectClient(). + if (disconnected_) + Throw("JSONRPCClient::invoke called after disconnect()"); + request req; req.method(boost::beast::http::verb::post); req.target("/"); @@ -131,10 +175,29 @@ public: req.body() = to_string(jr); } req.prepare_payload(); - write(stream_, req); + // The client keeps a single keep-alive connection for its whole + // lifetime, but the server drops idle localhost connections after a few + // seconds (BaseHTTPPeer::kTimeoutSecondsLocal). If a slow gap between + // requests let the server close the socket, the write/read here fails + // with end_of_stream; reconnect and retry the request exactly once. response res; - read(stream_, bin_, res); + auto writeAndRead = [&] { + write(stream_, req); + read(stream_, bin_, res); + }; + try + { + writeAndRead(); + } + catch (boost::system::system_error const& e) + { + if (!droppedConnection(e.code())) + throw; + reconnect(); + res = {}; + writeAndRead(); + } json::Reader jr; json::Value jv; @@ -151,6 +214,19 @@ public: { return rpcVersion_; } + + void + disconnect() override + { + if (disconnected_) + return; + + disconnected_ = true; + + boost::system::error_code ec; + stream_.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); + stream_.close(ec); + } }; std::unique_ptr diff --git a/src/test/jtx/impl/WSClient.cpp b/src/test/jtx/impl/WSClient.cpp index ca322415fb..a8702c12d9 100644 --- a/src/test/jtx/impl/WSClient.cpp +++ b/src/test/jtx/impl/WSClient.cpp @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -112,10 +113,11 @@ class WSClientImpl : public WSClient bool peerClosed_ = false; - // synchronize destructor - bool b0_ = false; - std::mutex m0_; - std::condition_variable cv0_; + // disconnect() waits on this until the read loop ends (for any reason: + // the server acknowledged our close, or a timeout force-closed the socket). + static constexpr auto kDisconnectTimeout = std::chrono::seconds{1}; + xrpl::Mutex readEnded_; + std::condition_variable readEndCv_; // synchronize message queue std::mutex m_; @@ -127,23 +129,26 @@ class WSClientImpl : public WSClient void cleanup() { - boost::asio::post(ios_, boost::asio::bind_executor(strand_, [this] { - if (!peerClosed_) - { - ws_.async_close( - {}, boost::asio::bind_executor(strand_, [&](error_code) { - try - { - stream_.cancel(); - } - // NOLINTNEXTLINE(bugprone-empty-catch) - catch (boost::system::system_error const&) - { - // ignored - } - })); - } - })); + boost::asio::post( + ios_, // + boost::asio::bind_executor(strand_, [this] { + if (!peerClosed_) + { + ws_.async_close( + {}, // + boost::asio::bind_executor(strand_, [&](error_code) { + try + { + stream_.cancel(); + } + // NOLINTNEXTLINE(bugprone-empty-catch) + catch (boost::system::system_error const&) + { + // ignored + } + })); + } + })); work_ = std::nullopt; thread_.join(); } @@ -289,6 +294,44 @@ public: return rpcVersion_; } + void + disconnect() override + { + // Perform a graceful WebSocket closing handshake and block until the + // read loop ends, so the server observes a clean close (not a RST) and + // has finished tearing the connection down by the time we return. + // If the server already closed, the wait below returns immediately. + boost::asio::post( + ios_, + boost::asio::bind_executor( + strand_, // + [this] { + if (!peerClosed_) + { + ws_.async_close( + boost::beast::websocket::close_code::normal, + boost::asio::bind_executor(strand_, [](error_code) {})); + } + })); + + auto lock = readEnded_.lock(); + readEndCv_.wait_for(lock, kDisconnectTimeout, [&lock] { return *lock; }); + + // On timeout (server gone or not replying) force the socket closed so + // the outstanding read ends and the worker thread can later be joined. + if (!*lock) + { + boost::asio::post( + ios_, + boost::asio::bind_executor( + strand_, // + [this] { + boost::system::error_code ec; + stream_.close(ec); + })); + } + } + private: void onReadMsg(error_code const& ec) @@ -297,33 +340,31 @@ private: { if (ec == boost::beast::websocket::error::closed) peerClosed_ = true; + + *readEnded_.lock() = true; + readEndCv_.notify_all(); + return; } json::Value jv; json::Reader jr; + jr.parse(bufferString(rb_.data()), jv); rb_.consume(rb_.size()); + auto m = std::make_shared(std::move(jv)); { std::scoped_lock const lock(m_); msgs_.push_front(m); cv_.notify_all(); } + ws_.async_read( rb_, boost::asio::bind_executor(strand_, [this](error_code const& ec, std::size_t) { onReadMsg(ec); })); } - - // Called when the read op terminates - void - onReadDone() - { - std::scoped_lock const lock(m0_); - b0_ = true; - cv0_.notify_all(); - } }; std::unique_ptr diff --git a/src/test/nodestore/Backend_test.cpp b/src/test/nodestore/Backend_test.cpp deleted file mode 100644 index 65601b0cf5..0000000000 --- a/src/test/nodestore/Backend_test.cpp +++ /dev/null @@ -1,110 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace xrpl::NodeStore { - -// Tests the Backend interface -// -class Backend_test : public TestBase -{ -public: - void - testBackend(std::string const& type, std::uint64_t const seedValue, int numObjsToTest = 2000) - { - DummyScheduler scheduler; - - testcase("Backend type=" + type); - - Section params; - beast::TempDir const tempDir; - params.set(Keys::kType, type); - params.set(Keys::kPath, tempDir.path()); - - beast::xor_shift_engine rng(seedValue); - - // Create a batch - auto batch = createPredictableBatch(numObjsToTest, rng()); - - using beast::Severity; - test::SuiteJournal journal("Backend_test", *this); - - { - // Open the backend - std::unique_ptr backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - backend->open(); - - // Write the batch - storeBatch(*backend, batch); - - { - // Read it back in - Batch copy; - fetchCopyOfBatch(*backend, ©, batch); - BEAST_EXPECT(areBatchesEqual(batch, copy)); - } - - { - // Reorder and read the copy again - std::shuffle(batch.begin(), batch.end(), rng); - Batch copy; - fetchCopyOfBatch(*backend, ©, batch); - BEAST_EXPECT(areBatchesEqual(batch, copy)); - } - } - - { - // Re-open the backend - std::unique_ptr backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - backend->open(); - - // Read it back in - Batch copy; - fetchCopyOfBatch(*backend, ©, batch); - // Canonicalize the source and destination batches - std::ranges::sort(batch, LessThan{}); - std::ranges::sort(copy, LessThan{}); - BEAST_EXPECT(areBatchesEqual(batch, copy)); - } - } - - //-------------------------------------------------------------------------- - - void - run() override - { - std::uint64_t const seedValue = 50; - - testBackend("nudb", seedValue); - -#if XRPL_ROCKSDB_AVAILABLE - testBackend("rocksdb", seedValue); -#endif - -#ifdef XRPL_ENABLE_SQLITE_BACKEND_TESTS - testBackend("sqlite", seedValue); -#endif - } -}; - -BEAST_DEFINE_TESTSUITE(Backend, nodestore, xrpl); - -} // namespace xrpl::NodeStore diff --git a/src/test/nodestore/Basics_test.cpp b/src/test/nodestore/Basics_test.cpp deleted file mode 100644 index 7d77b18630..0000000000 --- a/src/test/nodestore/Basics_test.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include - -#include -#include -#include -#include - -#include -#include - -namespace xrpl::NodeStore { - -// Tests predictable batches, and NodeObject blob encoding -// -class NodeStoreBasic_test : public TestBase -{ -public: - // Make sure predictable object generation works! - void - testBatches(std::uint64_t const seedValue) - { - testcase("batch"); - - auto batch1 = createPredictableBatch(kNumObjectsToTest, seedValue); - - auto batch2 = createPredictableBatch(kNumObjectsToTest, seedValue); - - BEAST_EXPECT(areBatchesEqual(batch1, batch2)); - - auto batch3 = createPredictableBatch(kNumObjectsToTest, seedValue + 1); - - BEAST_EXPECT(!areBatchesEqual(batch1, batch3)); - } - - // Checks encoding/decoding blobs - void - testBlobs(std::uint64_t const seedValue) - { - testcase("encoding"); - - auto batch = createPredictableBatch(kNumObjectsToTest, seedValue); - - for (auto const& expected : batch) - { - EncodedBlob const encoded(expected); - - DecodedBlob decoded(encoded.getKey(), encoded.getData(), encoded.getSize()); - - BEAST_EXPECT(decoded.wasOk()); - - if (decoded.wasOk()) - { - std::shared_ptr const object(decoded.createObject()); - - BEAST_EXPECT(isSame(expected, object)); - } - } - } - - void - run() override - { - std::uint64_t const seedValue = 50; - - testBatches(seedValue); - - testBlobs(seedValue); - } -}; - -BEAST_DEFINE_TESTSUITE(NodeStoreBasic, nodestore, xrpl); - -} // namespace xrpl::NodeStore diff --git a/src/test/nodestore/Database_test.cpp b/src/test/nodestore/DatabaseConfig_test.cpp similarity index 59% rename from src/test/nodestore/Database_test.cpp rename to src/test/nodestore/DatabaseConfig_test.cpp index bb8ec7d4fd..1f7f0f67bd 100644 --- a/src/test/nodestore/Database_test.cpp +++ b/src/test/nodestore/DatabaseConfig_test.cpp @@ -1,44 +1,21 @@ #include #include #include -#include -#include #include -#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include #include -#include -#include #include -namespace xrpl::NodeStore { +namespace xrpl::node_store { -class Database_test : public TestBase +class DatabaseConfig_test : public beast::unit_test::Suite { - test::SuiteJournal journal_; - public: - Database_test() : journal_("Database_test", *this) - { - } - void testConfig() { @@ -73,8 +50,8 @@ public: Env env = [&]() { auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kSafetyLevel, "high"); + auto& section = p->section("sqlite"); + section.set("safety_level", "high"); } p->ledgerHistory = 100'000'000; @@ -102,8 +79,8 @@ public: Env env = [&]() { auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kSafetyLevel, "low"); + auto& section = p->section("sqlite"); + section.set("safety_level", "low"); } p->ledgerHistory = 100'000'000; @@ -131,10 +108,10 @@ public: Env env = [&]() { auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kJournalMode, "off"); - section.set(Keys::kSynchronous, "extra"); - section.set(Keys::kTempStore, "default"); + auto& section = p->section("sqlite"); + section.set("journal_mode", "off"); + section.set("synchronous", "extra"); + section.set("temp_store", "default"); } return Env( @@ -145,7 +122,7 @@ public: }(); // No warning, even though higher risk settings were used because - // LEDGER_HISTORY is small + // ledgerHistory is small BEAST_EXPECT(!found); auto const s = setupDatabaseCon(env.app().config()); if (BEAST_EXPECT(s.globalPragma->size() == 3)) @@ -163,10 +140,10 @@ public: Env env = [&]() { auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kJournalMode, "off"); - section.set(Keys::kSynchronous, "extra"); - section.set(Keys::kTempStore, "default"); + auto& section = p->section("sqlite"); + section.set("journal_mode", "off"); + section.set("synchronous", "extra"); + section.set("temp_store", "default"); } p->ledgerHistory = 50'000'000; @@ -178,7 +155,7 @@ public: }(); // No warning, even though higher risk settings were used because - // LEDGER_HISTORY is small + // ledgerHistory is small BEAST_EXPECT(found); auto const s = setupDatabaseCon(env.app().config()); if (BEAST_EXPECT(s.globalPragma->size() == 3)) @@ -199,11 +176,11 @@ public: auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kSafetyLevel, "low"); - section.set(Keys::kJournalMode, "off"); - section.set(Keys::kSynchronous, "extra"); - section.set(Keys::kTempStore, "default"); + auto& section = p->section("sqlite"); + section.set("safety_level", "low"); + section.set("journal_mode", "off"); + section.set("synchronous", "extra"); + section.set("temp_store", "default"); } try @@ -230,9 +207,9 @@ public: auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kSafetyLevel, "high"); - section.set(Keys::kJournalMode, "off"); + auto& section = p->section("sqlite"); + section.set("safety_level", "high"); + section.set("journal_mode", "off"); } try @@ -259,9 +236,9 @@ public: auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kSafetyLevel, "low"); - section.set(Keys::kSynchronous, "extra"); + auto& section = p->section("sqlite"); + section.set("safety_level", "low"); + section.set("synchronous", "extra"); } try @@ -288,9 +265,9 @@ public: auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kSafetyLevel, "high"); - section.set(Keys::kTempStore, "default"); + auto& section = p->section("sqlite"); + section.set("safety_level", "high"); + section.set("temp_store", "default"); } try @@ -317,8 +294,8 @@ public: auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kSafetyLevel, "slow"); + auto& section = p->section("sqlite"); + section.set("safety_level", "slow"); } try @@ -345,8 +322,8 @@ public: auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kJournalMode, "fast"); + auto& section = p->section("sqlite"); + section.set("journal_mode", "fast"); } try @@ -373,8 +350,8 @@ public: auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kSynchronous, "instant"); + auto& section = p->section("sqlite"); + section.set("synchronous", "instant"); } try @@ -401,8 +378,8 @@ public: auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kTempStore, "network"); + auto& section = p->section("sqlite"); + section.set("temp_store", "network"); } try @@ -436,9 +413,9 @@ public: Env env = [&]() { auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kPageSize, "512"); - section.set(Keys::kJournalSizeLimit, "2582080"); + auto& section = p->section("sqlite"); + section.set("page_size", "512"); + section.set("journal_size_limit", "2582080"); } return Env(*this, std::move(p)); }(); @@ -457,8 +434,8 @@ public: bool found = false; auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kPageSize, "256"); + auto& section = p->section("sqlite"); + section.set("page_size", "256"); } try { @@ -480,8 +457,8 @@ public: bool found = false; auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kPageSize, "131072"); + auto& section = p->section("sqlite"); + section.set("page_size", "131072"); } try { @@ -503,8 +480,8 @@ public: bool found = false; auto p = test::jtx::envconfig(); { - auto& section = p->section(Sections::kSqlite); - section.set(Keys::kPageSize, "513"); + auto& section = p->section("sqlite"); + section.set("page_size", "513"); } try { @@ -522,208 +499,13 @@ public: } } - //-------------------------------------------------------------------------- - - void - testImport( - std::string const& destBackendType, - std::string const& srcBackendType, - std::int64_t seedValue) - { - DummyScheduler scheduler; - - beast::TempDir const nodeDb; - Section srcParams; - srcParams.set(Keys::kType, srcBackendType); - srcParams.set(Keys::kPath, nodeDb.path()); - - // Create a batch - auto batch = createPredictableBatch(kNumObjectsToTest, seedValue); - - // Write to source db - { - std::unique_ptr src = - Manager::instance().makeDatabase(megabytes(4), scheduler, 2, srcParams, journal_); - storeBatch(*src, batch); - } - - Batch copy; - - { - // Re-open the db - std::unique_ptr src = - Manager::instance().makeDatabase(megabytes(4), scheduler, 2, srcParams, journal_); - - // Set up the destination database - beast::TempDir const destDb; - Section destParams; - destParams.set(Keys::kType, destBackendType); - destParams.set(Keys::kPath, destDb.path()); - - std::unique_ptr dest = - Manager::instance().makeDatabase(megabytes(4), scheduler, 2, destParams, journal_); - - testcase("import into '" + destBackendType + "' from '" + srcBackendType + "'"); - - // Do the import - dest->importDatabase(*src); - - // Get the results of the import - fetchCopyOfBatch(*dest, ©, batch); - } - - // Canonicalize the source and destination batches - std::ranges::sort(batch, LessThan{}); - std::ranges::sort(copy, LessThan{}); - BEAST_EXPECT(areBatchesEqual(batch, copy)); - } - - //-------------------------------------------------------------------------- - - void - testNodeStore( - std::string const& type, - bool const testPersistence, - std::int64_t const seedValue, - int numObjsToTest = 2000) - { - DummyScheduler scheduler; - - std::string const s = "NodeStore backend '" + type + "'"; - - testcase(s); - - beast::TempDir const nodeDb; - Section nodeParams; - nodeParams.set(Keys::kType, type); - nodeParams.set(Keys::kPath, nodeDb.path()); - - beast::xor_shift_engine rng(seedValue); - - // Create a batch - auto batch = createPredictableBatch(numObjsToTest, rng()); - - { - // Open the database - std::unique_ptr db = - Manager::instance().makeDatabase(megabytes(4), scheduler, 2, nodeParams, journal_); - - // Write the batch - storeBatch(*db, batch); - - { - // Read it back in - Batch copy; - fetchCopyOfBatch(*db, ©, batch); - BEAST_EXPECT(areBatchesEqual(batch, copy)); - } - - { - // Reorder and read the copy again - std::shuffle(batch.begin(), batch.end(), rng); - Batch copy; - fetchCopyOfBatch(*db, ©, batch); - BEAST_EXPECT(areBatchesEqual(batch, copy)); - } - } - - if (testPersistence) - { - // Re-open the database without the ephemeral DB - std::unique_ptr db = - Manager::instance().makeDatabase(megabytes(4), scheduler, 2, nodeParams, journal_); - - // Read it back in - Batch copy; - fetchCopyOfBatch(*db, ©, batch); - - // Canonicalize the source and destination batches - std::ranges::sort(batch, LessThan{}); - std::ranges::sort(copy, LessThan{}); - BEAST_EXPECT(areBatchesEqual(batch, copy)); - } - - if (type == "memory") - { - // Verify default earliest ledger sequence - { - std::unique_ptr db = Manager::instance().makeDatabase( - megabytes(4), scheduler, 2, nodeParams, journal_); - BEAST_EXPECT(db->earliestLedgerSeq() == kXrpLedgerEarliestSeq); - } - - // Set an invalid earliest ledger sequence - try - { - nodeParams.set(Keys::kEarliestSeq, "0"); - std::unique_ptr const db = Manager::instance().makeDatabase( - megabytes(4), scheduler, 2, nodeParams, journal_); - } - catch (std::runtime_error const& e) - { - BEAST_EXPECT(std::strcmp(e.what(), "Invalid earliest_seq") == 0); - } - - { - // Set a valid earliest ledger sequence - nodeParams.set(Keys::kEarliestSeq, "1"); - std::unique_ptr db = Manager::instance().makeDatabase( - megabytes(4), scheduler, 2, nodeParams, journal_); - - // Verify database uses the earliest ledger sequence setting - BEAST_EXPECT(db->earliestLedgerSeq() == 1); - } - - // Create another database that attempts to set the value again - try - { - // Set to default earliest ledger sequence - nodeParams.set(Keys::kEarliestSeq, std::to_string(kXrpLedgerEarliestSeq)); - std::unique_ptr const db2 = Manager::instance().makeDatabase( - megabytes(4), scheduler, 2, nodeParams, journal_); - } - catch (std::runtime_error const& e) - { - BEAST_EXPECT(std::strcmp(e.what(), "earliest_seq set more than once") == 0); - } - } - } - - //-------------------------------------------------------------------------- - void run() override { - std::int64_t const seedValue = 50; - testConfig(); - - testNodeStore("memory", false, seedValue); - - // Persistent backend tests - { - testNodeStore("nudb", true, seedValue); - -#if XRPL_ROCKSDB_AVAILABLE - testNodeStore("rocksdb", true, seedValue); -#endif - } - - // Import tests - { - testImport("nudb", "nudb", seedValue); - -#if XRPL_ROCKSDB_AVAILABLE - testImport("rocksdb", "rocksdb", seedValue); -#endif - -#if XRPL_ENABLE_SQLITE_BACKEND_TESTS - testImport("sqlite", "sqlite", seedValue); -#endif - } } }; -BEAST_DEFINE_TESTSUITE(Database, nodestore, xrpl); +BEAST_DEFINE_TESTSUITE(DatabaseConfig, nodestore, xrpl); -} // namespace xrpl::NodeStore +} // namespace xrpl::node_store diff --git a/src/test/nodestore/NuDBFactory_test.cpp b/src/test/nodestore/NuDBFactory_test.cpp deleted file mode 100644 index d0675b3893..0000000000 --- a/src/test/nodestore/NuDBFactory_test.cpp +++ /dev/null @@ -1,443 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -namespace xrpl::NodeStore { - -class NuDBFactory_test : public TestBase -{ -private: - // Helper function to create a Section with specified parameters - static Section - createSection(std::string const& path, std::string const& blockSize = "") - { - Section params; - params.set(Keys::kType, "nudb"); - params.set(Keys::kPath, path); - if (!blockSize.empty()) - params.set(Keys::kNudbBlockSize, blockSize); - return params; - } - - // Helper function to create a backend and test basic functionality - bool - testBackendFunctionality(Section const& params, std::size_t expectedBlocksize) - { - try - { - DummyScheduler scheduler; - test::SuiteJournal journal("NuDBFactory_test", *this); - - auto backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - - if (!BEAST_EXPECT(backend)) - return false; - - if (!BEAST_EXPECT(backend->getBlockSize() == expectedBlocksize)) - return false; - - backend->open(); - - if (!BEAST_EXPECT(backend->isOpen())) - return false; - - // Test basic store/fetch functionality - auto batch = createPredictableBatch(10, 12345); - storeBatch(*backend, batch); - - Batch copy; - fetchCopyOfBatch(*backend, ©, batch); - - backend->close(); - - return areBatchesEqual(batch, copy); - } - catch (...) - { - return false; - } - } - - // Helper function to test log messages - void - testLogMessage(Section const& params, beast::Severity level, std::string const& expectedMessage) - { - test::StreamSink sink(level); - beast::Journal const journal(sink); - - DummyScheduler scheduler; - auto backend = Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - - std::string const logOutput = sink.messages().str(); - BEAST_EXPECT(logOutput.contains(expectedMessage)); - } - - // Helper function to test power of two validation - void - testPowerOfTwoValidation(std::string const& size, bool shouldWork) - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), size); - - test::StreamSink sink(beast::Severity::Warning); - beast::Journal const journal(sink); - - DummyScheduler scheduler; - auto backend = Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - - std::string const logOutput = sink.messages().str(); - bool const hasWarning = logOutput.contains("Invalid nudb_block_size"); - - BEAST_EXPECT(hasWarning == !shouldWork); - } - -public: - void - testDefaultBlockSize() - { - testcase("Default block size (no nudb_block_size specified)"); - - beast::TempDir const tempDir; - auto params = createSection(tempDir.path()); - - // Should work with default 4096 block size - BEAST_EXPECT(testBackendFunctionality(params, 4096)); - } - - void - testValidBlockSizes() - { - testcase("Valid block sizes"); - - std::vector const validSizes = {4096, 8192, 16384, 32768}; - - for (auto const& size : validSizes) - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), to_string(size)); - - BEAST_EXPECT(testBackendFunctionality(params, size)); - } - // Empty value is ignored by the config parser, so uses the - // default - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), ""); - - BEAST_EXPECT(testBackendFunctionality(params, 4096)); - } - - void - testInvalidBlockSizes() - { - testcase("Invalid block sizes"); - - std::vector const invalidSizes = { - "2048", // Too small - "1024", // Too small - "65536", // Too large - "131072", // Too large - "5000", // Not power of 2 - "6000", // Not power of 2 - "10000", // Not power of 2 - "0", // Zero - "-1", // Negative - "abc", // Non-numeric - "4k", // Invalid format - "4096.5" // Decimal - }; - - for (auto const& size : invalidSizes) - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), size); - - // Fails - BEAST_EXPECT(!testBackendFunctionality(params, 4096)); - } - - // Test whitespace cases separately since lexical_cast may handle them - std::vector const whitespaceInvalidSizes = { - "4096 ", // Trailing space - might be handled by lexical_cast - " 4096" // Leading space - might be handled by lexical_cast - }; - - for (auto const& size : whitespaceInvalidSizes) - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), size); - - // Fails - BEAST_EXPECT(!testBackendFunctionality(params, 4096)); - } - } - - void - testLogMessages() - { - testcase("Log message verification"); - - // Test valid custom block size logging - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), "8192"); - - testLogMessage(params, beast::Severity::Info, "Using custom NuDB block size: 8192"); - } - - // Test invalid block size failure - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), "5000"); - - test::StreamSink sink(beast::Severity::Warning); - beast::Journal const journal(sink); - - DummyScheduler scheduler; - try - { - auto backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - fail(); - } - catch (std::exception const& e) - { - std::string const logOutput{e.what()}; - BEAST_EXPECT(logOutput.contains("Invalid nudb_block_size: 5000")); - BEAST_EXPECT(logOutput.contains("Must be power of 2 between 4096 and 32768")); - } - } - - // Test non-numeric value failure - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), "invalid"); - - test::StreamSink sink(beast::Severity::Warning); - beast::Journal const journal(sink); - - DummyScheduler scheduler; - try - { - auto backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - - fail(); - } - catch (std::exception const& e) - { - std::string const logOutput{e.what()}; - BEAST_EXPECT(logOutput.contains("Invalid nudb_block_size value: invalid")); - } - } - } - - void - testPowerOfTwoValidation() - { - testcase("Power of 2 validation logic"); - - // Test edge cases around valid range - std::vector> const testCases = { - {"4095", false}, // Just below minimum - {"4096", true}, // Minimum valid - {"4097", false}, // Just above minimum, not power of 2 - {"8192", true}, // Valid power of 2 - {"8193", false}, // Just above valid power of 2 - {"16384", true}, // Valid power of 2 - {"32768", true}, // Maximum valid - {"32769", false}, // Just above maximum - {"65536", false} // Power of 2 but too large - }; - - for (auto const& [size, shouldWork] : testCases) - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), size); - - // We test the validation logic by catching exceptions for invalid - // values - test::StreamSink sink(beast::Severity::Warning); - beast::Journal const journal(sink); - - DummyScheduler scheduler; - try - { - auto backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - BEAST_EXPECT(shouldWork); - } - catch (std::exception const& e) - { - std::string const logOutput{e.what()}; - BEAST_EXPECT(logOutput.contains("Invalid nudb_block_size")); - } - } - } - - void - testBothConstructorVariants() - { - testcase("Both constructor variants work with custom block size"); - - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), "16384"); - - DummyScheduler scheduler; - test::SuiteJournal journal("NuDBFactory_test", *this); - - // Test first constructor (without nudb::context) - { - auto backend1 = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - BEAST_EXPECT(backend1 != nullptr); - BEAST_EXPECT(testBackendFunctionality(params, 16384)); - } - - // Test second constructor (with nudb::context) - // Note: This would require access to nudb::context, which might not be - // easily testable without more complex setup. For now, we test that - // the factory can create backends with the first constructor. - } - - void - testConfigurationParsing() - { - testcase("Configuration parsing edge cases"); - - // Test that whitespace is handled correctly - std::vector const validFormats = { - "8192" // Basic valid format - }; - - // Test whitespace handling separately since lexical_cast behavior may - // vary - std::vector const whitespaceFormats = { - " 8192", // Leading space - may or may not be handled by - // lexical_cast - "8192 " // Trailing space - may or may not be handled by - // lexical_cast - }; - - // Test basic valid format - for (auto const& format : validFormats) - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), format); - - test::StreamSink sink(beast::Severity::Info); - beast::Journal const journal(sink); - - DummyScheduler scheduler; - auto backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - - // Should log success message for valid values - std::string const logOutput = sink.messages().str(); - bool const hasSuccessMessage = logOutput.contains("Using custom NuDB block size"); - BEAST_EXPECT(hasSuccessMessage); - } - - // Test whitespace formats - these should work if lexical_cast handles - // them - for (auto const& format : whitespaceFormats) - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), format); - - // Use a lower threshold to capture both info and warning messages - test::StreamSink sink(beast::Severity::Debug); - beast::Journal const journal(sink); - - DummyScheduler scheduler; - try - { - auto backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - fail(); - } - catch (...) - { - // Fails - BEAST_EXPECT(!testBackendFunctionality(params, 8192)); - } - } - } - - void - testDataPersistence() - { - testcase("Data persistence with different block sizes"); - - std::vector const blockSizes = {"4096", "8192", "16384", "32768"}; - - for (auto const& size : blockSizes) - { - beast::TempDir const tempDir; - auto params = createSection(tempDir.path(), size); - - DummyScheduler scheduler; - test::SuiteJournal journal("NuDBFactory_test", *this); - - // Create test data - auto batch = createPredictableBatch(50, 54321); - - // Store data - { - auto backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - backend->open(); - storeBatch(*backend, batch); - backend->close(); - } - - // Retrieve data in new backend instance - { - auto backend = - Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); - backend->open(); - - Batch copy; - fetchCopyOfBatch(*backend, ©, batch); - - BEAST_EXPECT(areBatchesEqual(batch, copy)); - backend->close(); - } - } - } - - void - run() override - { - testDefaultBlockSize(); - testValidBlockSizes(); - testInvalidBlockSizes(); - testLogMessages(); - testPowerOfTwoValidation(); - testBothConstructorVariants(); - testConfigurationParsing(); - testDataPersistence(); - } -}; - -BEAST_DEFINE_TESTSUITE(NuDBFactory, xrpl_core, xrpl); - -} // namespace xrpl::NodeStore diff --git a/src/test/nodestore/TestBase.h b/src/test/nodestore/TestBase.h deleted file mode 100644 index 235e76501f..0000000000 --- a/src/test/nodestore/TestBase.h +++ /dev/null @@ -1,202 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -namespace xrpl::NodeStore { - -/** - * Binary function that satisfies the strict-weak-ordering requirement. - * - * This compares the hashes of both objects and returns true if - * the first hash is considered to go before the second. - * - * @see std::sort - */ -struct LessThan -{ - bool - operator()(std::shared_ptr const& lhs, std::shared_ptr const& rhs) - const noexcept - { - return lhs->getHash() < rhs->getHash(); - } -}; - -/** - * Returns `true` if objects are identical. - */ -inline bool -isSame(std::shared_ptr const& lhs, std::shared_ptr const& rhs) -{ - return (lhs->getType() == rhs->getType()) && (lhs->getHash() == rhs->getHash()) && - (lhs->getData() == rhs->getData()); -} - -// Some common code for the unit tests -// -class TestBase : public beast::unit_test::Suite -{ -public: - // Tunable parameters - // - static std::size_t const kMinPayloadBytes = 1; - static std::size_t const kMaxPayloadBytes = 2000; - static int const kNumObjectsToTest = 2000; - -public: - // Create a predictable batch of objects - static Batch - createPredictableBatch(int numObjects, std::uint64_t seed) - { - Batch batch; - batch.reserve(numObjects); - - beast::xor_shift_engine rng(seed); - - for (int i = 0; i < numObjects; ++i) - { - NodeObjectType const type = [&] { - switch (randInt(rng, 3)) - { - case 0: - return NodeObjectType::Ledger; - case 1: - return NodeObjectType::AccountNode; - case 2: - return NodeObjectType::TransactionNode; - case 3: - default: - return NodeObjectType::Unknown; - } - }(); - - uint256 hash; - beast::rngfill(hash.begin(), hash.size(), rng); - - Blob blob(randInt(rng, kMinPayloadBytes, kMaxPayloadBytes)); - beast::rngfill(blob.data(), blob.size(), rng); - - batch.push_back(NodeObject::createObject(type, std::move(blob), hash)); - } - - return batch; - } - - // Compare two batches for equality - static bool - areBatchesEqual(Batch const& lhs, Batch const& rhs) - { - bool result = true; - - if (lhs.size() == rhs.size()) - { - for (int i = 0; i < lhs.size(); ++i) - { - if (!isSame(lhs[i], rhs[i])) - { - result = false; - break; - } - } - } - else - { - result = false; - } - - return result; - } - - // Store a batch in a backend - static void - storeBatch(Backend& backend, Batch const& batch) - { - for (auto const& object : batch) - { - backend.store(object); - } - } - - // Get a copy of a batch in a backend - void - fetchCopyOfBatch(Backend& backend, Batch* pCopy, Batch const& batch) - { - pCopy->clear(); - pCopy->reserve(batch.size()); - - for (auto const& expected : batch) - { - std::shared_ptr object; - - Status const status = backend.fetch(expected->getHash(), &object); - - BEAST_EXPECT(status == Status::Ok); - - if (status == Status::Ok) - { - BEAST_EXPECT(object != nullptr); - - pCopy->push_back(object); - } - } - } - - void - fetchMissing(Backend& backend, Batch const& batch) - { - for (auto const& expected : batch) - { - std::shared_ptr object; - - Status const status = backend.fetch(expected->getHash(), &object); - - BEAST_EXPECT(status == Status::NotFound); - } - } - - // Store all objects in a batch - static void - storeBatch(Database& db, Batch const& batch) - { - for (auto const& object : batch) - { - Blob data(object->getData()); - - db.store(object->getType(), std::move(data), object->getHash(), db.earliestLedgerSeq()); - } - } - - // Fetch all the hashes in one batch, into another batch. - static void - fetchCopyOfBatch(Database& db, Batch* pCopy, Batch const& batch) - { - pCopy->clear(); - pCopy->reserve(batch.size()); - - for (auto const& expected : batch) - { - std::shared_ptr const object = db.fetchNodeObject(expected->getHash(), 0); - - if (object != nullptr) - pCopy->push_back(object); - } - } -}; - -} // namespace xrpl::NodeStore diff --git a/src/test/nodestore/import_test.cpp b/src/test/nodestore/import_test.cpp index d8c4a96713..c30d77029d 100644 --- a/src/test/nodestore/import_test.cpp +++ b/src/test/nodestore/import_test.cpp @@ -195,7 +195,7 @@ fmtdur(std::chrono::duration const& d) } // namespace detail -namespace NodeStore { +namespace node_store { //------------------------------------------------------------------------------ @@ -552,5 +552,5 @@ BEAST_DEFINE_TESTSUITE_MANUAL(import, nodestore, xrpl); //------------------------------------------------------------------------------ -} // namespace NodeStore +} // namespace node_store } // namespace xrpl diff --git a/src/test/nodestore/varint_test.cpp b/src/test/nodestore/varint_test.cpp deleted file mode 100644 index 68e88d831a..0000000000 --- a/src/test/nodestore/varint_test.cpp +++ /dev/null @@ -1,57 +0,0 @@ -#include -#include - -#include -#include -#include -#include - -namespace xrpl::NodeStore::tests { - -class varint_test : public beast::unit_test::Suite -{ -public: - void - testVarints(std::vector vv) - { - testcase("encode, decode"); - for (auto const v : vv) - { - std::array::kMax> vi{}; - auto const n0 = writeVarint(vi.data(), v); - expect(n0 > 0, "write error"); - expect(n0 == sizeVarint(v), "size error"); - std::size_t v1 = 0; - auto const n1 = readVarint(vi.data(), n0, v1); - expect(n1 == n0, "read error"); - expect(v == v1, "wrong value"); - } - } - - void - run() override - { - testVarints( - {0, - 1, - 2, - 126, - 127, - 128, - 253, - 254, - 255, - 16127, - 16128, - 16129, - 0xff, - 0xffff, - 0xffffffff, - 0xffffffffffffUL, - 0xffffffffffffffffUL}); - } -}; - -BEAST_DEFINE_TESTSUITE(varint, nodestore, xrpl); - -} // namespace xrpl::NodeStore::tests diff --git a/src/test/overlay/TMGetObjectByHash_test.cpp b/src/test/overlay/TMGetObjectByHash_test.cpp index e579989181..f52220a90c 100644 --- a/src/test/overlay/TMGetObjectByHash_test.cpp +++ b/src/test/overlay/TMGetObjectByHash_test.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -16,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/src/test/overlay/tx_reduce_relay_test.cpp b/src/test/overlay/tx_reduce_relay_test.cpp index a0d91d3aed..43f6ef2506 100644 --- a/src/test/overlay/tx_reduce_relay_test.cpp +++ b/src/test/overlay/tx_reduce_relay_test.cpp @@ -9,12 +9,12 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include diff --git a/src/test/peerfinder/Livecache_test.cpp b/src/test/peerfinder/Livecache_test.cpp deleted file mode 100644 index 4f2d6e97e1..0000000000 --- a/src/test/peerfinder/Livecache_test.cpp +++ /dev/null @@ -1,212 +0,0 @@ -#include -#include - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace xrpl::PeerFinder { - -bool -operator==(Endpoint const& a, Endpoint const& b) -{ - return (a.hops == b.hops && a.address == b.address); -} - -class Livecache_test : public beast::unit_test::Suite -{ - TestStopwatch clock_; - test::SuiteJournal journal_; - -public: - Livecache_test() : journal_("Livecache_test", *this) - { - } - - // Add the address as an endpoint - template - void - add(beast::IP::Endpoint ep, C& c, std::uint32_t hops = 0) - { - Endpoint const cep{ep, hops}; - c.insert(cep); - } - - void - testBasicInsert() - { - testcase("Basic Insert"); - Livecache<> c(clock_, journal_); - BEAST_EXPECT(c.empty()); - - for (auto i = 0; i < 10; ++i) - add(beast::IP::randomEP(true), c); - - BEAST_EXPECT(!c.empty()); - BEAST_EXPECT(c.size() == 10); - - for (auto i = 0; i < 10; ++i) - add(beast::IP::randomEP(false), c); - - BEAST_EXPECT(!c.empty()); - BEAST_EXPECT(c.size() == 20); - } - - void - testInsertUpdate() - { - testcase("Insert/Update"); - Livecache<> c(clock_, journal_); - - auto ep1 = Endpoint{beast::IP::randomEP(), 2}; - c.insert(ep1); - BEAST_EXPECT(c.size() == 1); - // third position list will contain the entry - BEAST_EXPECT((c.hops.begin() + 2)->begin()->hops == 2); - - auto ep2 = Endpoint{ep1.address, 4}; - // this will not change the entry has higher hops - c.insert(ep2); - BEAST_EXPECT(c.size() == 1); - // still in third position list - BEAST_EXPECT((c.hops.begin() + 2)->begin()->hops == 2); - - auto ep3 = Endpoint{ep1.address, 2}; - // this will not change the entry has the same hops as existing - c.insert(ep3); - BEAST_EXPECT(c.size() == 1); - // still in third position list - BEAST_EXPECT((c.hops.begin() + 2)->begin()->hops == 2); - - auto ep4 = Endpoint{ep1.address, 1}; - c.insert(ep4); - BEAST_EXPECT(c.size() == 1); - // now at second position list - BEAST_EXPECT((c.hops.begin() + 1)->begin()->hops == 1); - } - - void - testExpire() - { - testcase("Expire"); - using namespace std::chrono_literals; - Livecache<> c(clock_, journal_); - - auto ep1 = Endpoint{beast::IP::randomEP(), 1}; - c.insert(ep1); - BEAST_EXPECT(c.size() == 1); - c.expire(); - BEAST_EXPECT(c.size() == 1); - // verify that advancing to 1 sec before expiration - // leaves our entry intact - clock_.advance(Tuning::kLiveCacheSecondsToLive - 1s); - c.expire(); - BEAST_EXPECT(c.size() == 1); - // now advance to the point of expiration - clock_.advance(1s); - c.expire(); - BEAST_EXPECT(c.empty()); - } - - void - testHistogram() - { - testcase("Histogram"); - static constexpr auto kNumEps = 40; - Livecache<> c(clock_, journal_); - for (auto i = 0; i < kNumEps; ++i) - add(beast::IP::randomEP(true), c, xrpl::randInt()); - auto h = c.hops.histogram(); - if (!BEAST_EXPECT(!h.empty())) - return; - std::vector v; - boost::split(v, h, boost::algorithm::is_any_of(",")); - auto sum = 0; - for (auto const& n : v) - { - auto val = boost::lexical_cast(boost::trim_copy(n)); - sum += val; - BEAST_EXPECT(val >= 0); - } - BEAST_EXPECT(sum == kNumEps); - } - - void - testShuffle() - { - testcase("Shuffle"); - Livecache<> c(clock_, journal_); - for (auto i = 0; i < 100; ++i) - add(beast::IP::randomEP(true), c, xrpl::randInt(Tuning::kMaxHops + 1)); - - using at_hop = std::vector; - using all_hops = std::array; - - auto cmpEp = [](Endpoint const& a, Endpoint const& b) { - return (b.hops < a.hops || (b.hops == a.hops && b.address < a.address)); - }; - all_hops before; - all_hops beforeSorted; - for (auto i = std::make_pair(0, c.hops.begin()); i.second != c.hops.end(); - ++i.first, ++i.second) - { - std::ranges::copy(*i.second, std::back_inserter(before[i.first])); - std::ranges::copy(*i.second, std::back_inserter(beforeSorted[i.first])); - std::ranges::sort(beforeSorted[i.first], cmpEp); - } - - c.hops.shuffle(); - - all_hops after; - all_hops afterSorted; - for (auto i = std::make_pair(0, c.hops.begin()); i.second != c.hops.end(); - ++i.first, ++i.second) - { - std::ranges::copy(*i.second, std::back_inserter(after[i.first])); - std::ranges::copy(*i.second, std::back_inserter(afterSorted[i.first])); - std::ranges::sort(afterSorted[i.first], cmpEp); - } - - // each hop bucket should contain the same items - // before and after sort, albeit in different order - bool allMatch = true; - for (auto i = 0; i < before.size(); ++i) - { - BEAST_EXPECT(before[i].size() == after[i].size()); - allMatch = allMatch && (before[i] == after[i]); - BEAST_EXPECT(beforeSorted[i] == afterSorted[i]); - } - BEAST_EXPECT(!allMatch); - } - - void - run() override - { - testBasicInsert(); - testInsertUpdate(); - testExpire(); - testHistogram(); - testShuffle(); - } -}; - -BEAST_DEFINE_TESTSUITE(Livecache, peerfinder, xrpl); - -} // namespace xrpl::PeerFinder diff --git a/src/test/peerfinder/PeerFinder_test.cpp b/src/test/peerfinder/PeerFinder_test.cpp deleted file mode 100644 index cf91800951..0000000000 --- a/src/test/peerfinder/PeerFinder_test.cpp +++ /dev/null @@ -1,789 +0,0 @@ -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace xrpl::PeerFinder { - -class PeerFinder_test : public beast::unit_test::Suite -{ - test::SuiteJournal journal_; - -public: - PeerFinder_test() : journal_("PeerFinder_test", *this) - { - } - - struct TestStore : Store - { - std::size_t - load(load_callback const& cb) override - { - return 0; - } - - void - save(std::vector const&) override - { - } - }; - - struct TestChecker - { - void - stop() - { - } - - void - wait() - { - } - - template - void - asyncConnect(beast::IP::Endpoint const& ep, Handler&& handler) - { - // NOLINTNEXTLINE(misc-const-correctness) - boost::system::error_code ec; - handler(ec); - } - }; - - void - testBackoff1() - { - auto const seconds = 10000; - testcase("backoff 1"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - logic.addFixedPeer("test", beast::IP::Endpoint::fromString("65.0.0.1:5")); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - logic.config(c); - } - std::size_t n = 0; - for (std::size_t i = 0; i < seconds; ++i) - { - auto const list = logic.autoconnect(); - if (!list.empty()) - { - BEAST_EXPECT(list.size() == 1); - auto const [slot, _] = logic.newOutboundSlot(list.front()); - BEAST_EXPECT( - logic.onConnected(slot, beast::IP::Endpoint::fromString("65.0.0.2:5"))); - logic.onClosed(slot); - ++n; - } - clock.advance(std::chrono::seconds(1)); - logic.oncePerSecond(); - } - // Less than 20 attempts - BEAST_EXPECT(n < 20); - } - - // with activate - void - testBackoff2() - { - auto const seconds = 10000; - testcase("backoff 2"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - logic.addFixedPeer("test", beast::IP::Endpoint::fromString("65.0.0.1:5")); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - logic.config(c); - } - - PublicKey const pk(randomKeyPair(KeyType::Secp256k1).first); - std::size_t n = 0; - - for (std::size_t i = 0; i < seconds; ++i) - { - auto const list = logic.autoconnect(); - if (!list.empty()) - { - BEAST_EXPECT(list.size() == 1); - auto const [slot, _] = logic.newOutboundSlot(list.front()); - if (!BEAST_EXPECT( - logic.onConnected(slot, beast::IP::Endpoint::fromString("65.0.0.2:5")))) - return; - if (!BEAST_EXPECT(logic.activate(slot, pk, false) == PeerFinder::Result::Success)) - return; - logic.onClosed(slot); - ++n; - } - clock.advance(std::chrono::seconds(1)); - logic.oncePerSecond(); - } - // No more often than once per minute - BEAST_EXPECT(n <= (seconds + 59) / 60); - } - - // test accepting an incoming slot for an already existing outgoing slot - void - testDuplicateOutIn() - { - testcase("duplicate out/in"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - auto const remote = beast::IP::Endpoint::fromString("65.0.0.1:5"); - auto const [slot1, r] = logic.newOutboundSlot(remote); - BEAST_EXPECT(slot1 != nullptr); - BEAST_EXPECT(r == Result::Success); - BEAST_EXPECT(logic.connectedAddresses.count(remote.address()) == 1); - - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - auto const [slot2, r2] = logic.newInboundSlot(local, remote); - BEAST_EXPECT(logic.connectedAddresses.count(remote.address()) == 1); - BEAST_EXPECT(r2 == Result::DuplicatePeer); - - if (!BEAST_EXPECT(slot2 == nullptr)) - logic.onClosed(slot2); - - logic.onClosed(slot1); - } - - // test establishing outgoing slot for an already existing incoming slot - void - testDuplicateInOut() - { - testcase("duplicate in/out"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - auto const remote = beast::IP::Endpoint::fromString("65.0.0.1:5"); - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - - auto const [slot1, r] = logic.newInboundSlot(local, remote); - BEAST_EXPECT(slot1 != nullptr); - BEAST_EXPECT(r == Result::Success); - BEAST_EXPECT(logic.connectedAddresses.count(remote.address()) == 1); - - auto const [slot2, r2] = logic.newOutboundSlot(remote); - BEAST_EXPECT(r2 == Result::DuplicatePeer); - BEAST_EXPECT(logic.connectedAddresses.count(remote.address()) == 1); - if (!BEAST_EXPECT(slot2 == nullptr)) - logic.onClosed(slot2); - logic.onClosed(slot1); - } - - void - testPeerLimitExceeded() - { - testcase("peer limit exceeded"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - auto const [slot, r] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1025")); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(r == Result::Success); - - auto const [slot1, r1] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1026")); - BEAST_EXPECT(slot1 != nullptr); - BEAST_EXPECT(r1 == Result::Success); - - auto const [slot2, r2] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1027")); - BEAST_EXPECT(r2 == Result::IpLimitExceeded); - - if (!BEAST_EXPECT(slot2 == nullptr)) - logic.onClosed(slot2); - logic.onClosed(slot1); - logic.onClosed(slot); - } - - void - testActivateDuplicatePeer() - { - testcase("test activate duplicate peer"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - - PublicKey const pk1(randomKeyPair(KeyType::Secp256k1).first); - - auto const [slot, rSlot] = - logic.newOutboundSlot(beast::IP::Endpoint::fromString("55.104.0.2:1025")); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(rSlot == Result::Success); - - auto const [slot2, r2Slot] = - logic.newOutboundSlot(beast::IP::Endpoint::fromString("55.104.0.2:1026")); - BEAST_EXPECT(slot2 != nullptr); - BEAST_EXPECT(r2Slot == Result::Success); - - BEAST_EXPECT(logic.onConnected(slot, local)); - BEAST_EXPECT(logic.onConnected(slot2, local)); - - BEAST_EXPECT(logic.activate(slot, pk1, false) == Result::Success); - - // activating a different slot with the same node ID (pk) must fail - BEAST_EXPECT(logic.activate(slot2, pk1, false) == Result::DuplicatePeer); - - logic.onClosed(slot); - - // accept the same key for a new slot after removing the old slot - BEAST_EXPECT(logic.activate(slot2, pk1, false) == Result::Success); - logic.onClosed(slot2); - } - - void - testActivateInboundDisabled() - { - testcase("test activate inbound disabled"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - logic.config(c); - } - - PublicKey const pk1(randomKeyPair(KeyType::Secp256k1).first); - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - - auto const [slot, rSlot] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1025")); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(rSlot == Result::Success); - - BEAST_EXPECT(logic.activate(slot, pk1, false) == Result::InboundDisabled); - - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - c.inPeers = 1; - logic.config(c); - } - // new inbound slot must succeed when inbound connections are enabled - BEAST_EXPECT(logic.activate(slot, pk1, false) == Result::Success); - - // creating a new inbound slot must succeed as IP Limit is not exceeded - auto const [slot2, r2Slot] = - logic.newInboundSlot(local, beast::IP::Endpoint::fromString("55.104.0.2:1026")); - BEAST_EXPECT(slot2 != nullptr); - BEAST_EXPECT(r2Slot == Result::Success); - - PublicKey const pk2(randomKeyPair(KeyType::Secp256k1).first); - - // an inbound slot exceeding inPeers limit must fail - BEAST_EXPECT(logic.activate(slot2, pk2, false) == Result::Full); - - logic.onClosed(slot2); - logic.onClosed(slot); - } - - void - testAddFixedPeerNoPort() - { - testcase("test addFixedPeer no port"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - try - { - logic.addFixedPeer("test", beast::IP::Endpoint::fromString("65.0.0.2")); - fail("invalid endpoint successfully added"); - } - catch (std::runtime_error const& e) - { - pass(); - } - } - - void - testIsValidAddress() - { - testcase("is_valid_address"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - - auto const pass = [&](std::string const& s) { - BEAST_EXPECT(logic.isValidAddress(beast::IP::Endpoint::fromString(s))); - }; - auto const fail = [&](std::string const& s) { - BEAST_EXPECT(!logic.isValidAddress(beast::IP::Endpoint::fromString(s))); - }; - - // Invalid: port 0 - fail("65.0.0.1:0"); - - // --- IPv4 ranges --- - // For each range: 1 before (pass), first (fail), last (fail), - // 1 after (pass) - - // 0.0.0.0/8 - "This network" - // No "before" - nothing before 0.0.0.0 - fail("0.0.0.0:8080"); - fail("0.255.255.255:8080"); - pass("1.0.0.0:8080"); - - // 10.0.0.0/8 - Private (RFC 1918) - pass("9.255.255.255:8080"); - fail("10.0.0.0:8080"); - fail("10.255.255.255:8080"); - pass("11.0.0.0:8080"); - - // 100.64.0.0/10 - Shared Address Space / CGNAT (RFC 6598) - pass("100.63.255.255:8080"); - fail("100.64.0.0:8080"); - fail("100.127.255.255:8080"); - pass("100.128.0.0:8080"); - - // 127.0.0.0/8 - Loopback - pass("126.255.255.255:8080"); - fail("127.0.0.0:8080"); - fail("127.255.255.255:8080"); - pass("128.0.0.0:8080"); - - // 169.254.0.0/16 - Link-local - pass("169.253.255.255:8080"); - fail("169.254.0.0:8080"); - fail("169.254.255.255:8080"); - pass("169.255.0.0:8080"); - - // 172.16.0.0/12 - Private (RFC 1918) - pass("172.15.255.255:8080"); - fail("172.16.0.0:8080"); - fail("172.31.255.255:8080"); - pass("172.32.0.0:8080"); - - // 192.0.0.0/24 - IETF Protocol Assignments (RFC 6890) - pass("191.255.255.255:8080"); - fail("192.0.0.0:8080"); - fail("192.0.0.255:8080"); - pass("192.0.1.0:8080"); - - // 192.0.2.0/24 - TEST-NET-1 (RFC 5737) - pass("192.0.1.255:8080"); - fail("192.0.2.0:8080"); - fail("192.0.2.255:8080"); - pass("192.0.3.0:8080"); - - // 192.88.99.0/24 - 6to4 Relay Anycast (RFC 7526) - pass("192.88.98.255:8080"); - fail("192.88.99.0:8080"); - fail("192.88.99.255:8080"); - pass("192.88.100.0:8080"); - - // 192.168.0.0/16 - Private (RFC 1918) - pass("192.167.255.255:8080"); - fail("192.168.0.0:8080"); - fail("192.168.255.255:8080"); - pass("192.169.0.0:8080"); - - // 198.18.0.0/15 - Benchmarking (RFC 2544) - pass("198.17.255.255:8080"); - fail("198.18.0.0:8080"); - fail("198.19.255.255:8080"); - pass("198.20.0.0:8080"); - - // 198.51.100.0/24 - TEST-NET-2 (RFC 5737) - pass("198.51.99.255:8080"); - fail("198.51.100.0:8080"); - fail("198.51.100.255:8080"); - pass("198.51.101.0:8080"); - - // 203.0.113.0/24 - TEST-NET-3 (RFC 5737) - pass("203.0.112.255:8080"); - fail("203.0.113.0:8080"); - fail("203.0.113.255:8080"); - pass("203.0.114.0:8080"); - - // 224.0.0.0/4 - Multicast - pass("223.255.255.255:8080"); - fail("224.0.0.0:8080"); - fail("239.255.255.255:8080"); - // 240.0.0.0 (after multicast) is also blocked (reserved) - - // 240.0.0.0/4 - Reserved (RFC 1112) - // 239.255.255.255 (before reserved) is also blocked (multicast) - fail("240.0.0.0:8080"); - fail("255.255.255.255:8080"); - - // --- IPv6 ranges --- - - // ::1 - Loopback (single address) - fail("[::1]:8080"); - - // :: - Unspecified (single address) - fail("[::]:8080"); - - // fc00::/7 - Unique Local Address (ULA) - pass("[fb00::1]:8080"); - fail("[fc00::1]:8080"); - fail("[fdff::1]:8080"); - pass("[fe00::1]:8080"); - - // fe80::/10 - Link-local - pass("[fe7f::1]:8080"); - fail("[fe80::1]:8080"); - fail("[febf::1]:8080"); - pass("[fec0::1]:8080"); - - // ff00::/8 - Multicast - pass("[feff::1]:8080"); - fail("[ff00::1]:8080"); - fail("[ffff::1]:8080"); - // No "after" - ffff:... is the highest IPv6 range - - // 100::/64 - Discard prefix (RFC 6666) - pass("[ff::1]:8080"); - fail("[100::]:8080"); - fail("[100::ffff:ffff:ffff:ffff]:8080"); - pass("[100:0:0:1::1]:8080"); - - // 2001::/32 - IETF Protocol Assignments / Teredo (RFC 4380) - pass("[2000:ffff::1]:8080"); - fail("[2001::]:8080"); - fail("[2001:0:ffff::1]:8080"); - pass("[2001:1::1]:8080"); - - // 2001:20::/28 - ORCHIDv2 (RFC 7343) - pass("[2001:1f::1]:8080"); - fail("[2001:20::1]:8080"); - fail("[2001:2f::1]:8080"); - pass("[2001:30::1]:8080"); - - // 2001:db8::/32 - Documentation (RFC 3849) - pass("[2001:db7::1]:8080"); - fail("[2001:db8::1]:8080"); - fail("[2001:db8:ffff::1]:8080"); - pass("[2001:db9::1]:8080"); - - // 2002::/16 - 6to4 (RFC 3056, deprecated) - pass("[2001:ffff::1]:8080"); - fail("[2002::1]:8080"); - fail("[2002:ffff::1]:8080"); - pass("[2003::1]:8080"); - - // --- IPv6 v4-mapped (delegates to IPv4 checks) --- - fail("[::ffff:10.0.0.1]:8080"); - fail("[::ffff:100.64.0.1]:8080"); - fail("[::ffff:169.254.1.1]:8080"); - fail("[::ffff:192.0.2.1]:8080"); - fail("[::ffff:198.18.0.1]:8080"); - fail("[::ffff:224.0.0.1]:8080"); - fail("[::ffff:240.0.0.1]:8080"); - - // --- Valid public addresses --- - pass("8.8.8.8:443"); - pass("65.0.0.1:8080"); - pass("[2001:4860:4860::8888]:8080"); - pass("[2606:4700:4700::1111]:8080"); - } - - void - testVerifyEndpoints() - { - // Helper that sets up a Logic instance, creates and activates a slot, - // then calls on_endpoints with the given list and returns the - // livecache size afterwards. - auto run = [&](bool verifyEndpoints, Endpoints eps) -> std::size_t { - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - { - Config c; - c.autoConnect = false; - c.listeningPort = 1024; - c.ipLimit = 2; - c.verifyEndpoints = verifyEndpoints; - logic.config(c); - } - - auto const remote = beast::IP::Endpoint::fromString("65.0.0.1:5"); - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1024"); - - auto const [slot, r] = logic.newOutboundSlot(remote); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(r == Result::Success); - BEAST_EXPECT(logic.onConnected(slot, local)); - - PublicKey const pk(randomKeyPair(KeyType::Secp256k1).first); - BEAST_EXPECT(logic.activate(slot, pk, false) == Result::Success); - - logic.onEndpoints(slot, std::move(eps)); - - auto const size = logic.livecache.size(); - logic.onClosed(slot); - return size; - }; - - { - testcase("verify_endpoints enabled"); - - // Valid public addresses - Endpoints eps; - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.1:5"), 1); - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.2:6"), 1); - // Invalid: private address - eps.emplace_back(beast::IP::Endpoint::fromString("10.0.0.1:5"), 1); - // Invalid: port 0 - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.3:0"), 1); - - // With verification enabled, only the 2 valid endpoints survive - BEAST_EXPECT(run(true, eps) == 2); - } - { - testcase("verify_endpoints disabled"); - - Endpoints eps; - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.1:5"), 1); - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.2:6"), 1); - // Private address — kept when verification is off - eps.emplace_back(beast::IP::Endpoint::fromString("10.0.0.1:5"), 1); - // Port 0 — kept when verification is off - eps.emplace_back(beast::IP::Endpoint::fromString("44.0.0.3:0"), 1); - - // Without verification, all 4 endpoints survive - BEAST_EXPECT(run(false, eps) == 4); - } - } - - void - testOnConnectedSelfConnection() - { - testcase("test onConnected self connection"); - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - - auto const local = beast::IP::Endpoint::fromString("65.0.0.2:1234"); - auto const [slot, r] = logic.newOutboundSlot(local); - BEAST_EXPECT(slot != nullptr); - BEAST_EXPECT(r == Result::Success); - - // Must fail when a slot is to our own IP address - BEAST_EXPECT(!logic.onConnected(slot, local)); - logic.onClosed(slot); - } - - void - testConfig() - { - // if peers_max is configured then peers_in_max and peers_out_max - // are ignored - auto run = [&](std::string const& test, - std::optional maxPeers, - std::optional maxIn, - std::optional maxOut, - std::uint16_t port, - std::uint16_t expectOut, - std::uint16_t expectIn, - std::uint16_t expectIpLimit) { - xrpl::Config c; - - testcase(test); - - std::string toLoad; - int max = 0; - if (maxPeers) - { - max = maxPeers.value(); - toLoad += "[peers_max]\n" + std::to_string(max) + "\n" + "[peers_in_max]\n" + - std::to_string(maxIn.value_or(0)) + "\n" + "[peers_out_max]\n" + - std::to_string(maxOut.value_or(0)) + "\n"; - } - else if (maxIn && maxOut) - { - toLoad += "[peers_in_max]\n" + std::to_string(*maxIn) + "\n" + "[peers_out_max]\n" + - std::to_string(*maxOut) + "\n"; - } - - c.loadFromString(toLoad); - BEAST_EXPECT( - (c.peersMax == max && c.peersInMax == 0 && c.peersOutMax == 0) || - (c.peersInMax == *maxIn && c.peersOutMax == *maxOut)); - - Config const config = Config::makeConfig(c, port, false, 0, true); - - Counts counts; - counts.onConfig(config); - BEAST_EXPECT( - counts.outMax() == expectOut && counts.inMax() == expectIn && - config.ipLimit == expectIpLimit); - - TestStore store; - TestChecker checker; - TestStopwatch clock; - Logic logic(clock, store, checker, journal_); - logic.config(config); - - BEAST_EXPECT(logic.config() == config); - }; - - // if max_peers == 0 => maxPeers = 21, - // else if max_peers < 10 => maxPeers = 10 else maxPeers = - // max_peers - // expectOut => if legacy => max(0.15 * maxPeers, 10), - // if legacy && !wantIncoming => maxPeers else max_out_peers - // expectIn => if legacy && wantIncoming => maxPeers - outPeers - // else if !wantIncoming => 0 else max_in_peers - // ipLimit => if expectIn <= 21 => 2 else 2 + min(5, expectIn/21) - // ipLimit = max(1, min(ipLimit, expectIn/2)) - - // legacy test with max_peers - run("legacy no config", {}, {}, {}, 4000, 10, 11, 2); - run("legacy max_peers 0", 0, 100, 10, 4000, 10, 11, 2); - run("legacy max_peers 5", 5, 100, 10, 4000, 10, 0, 1); - run("legacy max_peers 20", 20, 100, 10, 4000, 10, 10, 2); - run("legacy max_peers 100", 100, 100, 10, 4000, 15, 85, 6); - run("legacy max_peers 20, private", 20, 100, 10, 0, 20, 0, 1); - - // test with max_in_peers and max_out_peers - run("new in 100/out 10", {}, 100, 10, 4000, 10, 100, 6); - run("new in 0/out 10", {}, 0, 10, 4000, 10, 0, 1); - run("new in 100/out 10, private", {}, 100, 10, 0, 10, 0, 6); - } - - void - testInvalidConfig() - { - testcase("invalid config"); - - auto run = [&](std::string const& toLoad) { - xrpl::Config c; - try - { - c.loadFromString(toLoad); - fail(); - } - catch (...) - { - pass(); - } - }; - run(R"xrpldConfig( -[peers_in_max] -100 -)xrpldConfig"); - run(R"xrpldConfig( -[peers_out_max] -100 -)xrpldConfig"); - run(R"xrpldConfig( -[peers_in_max] -100 -[peers_out_max] -5 -)xrpldConfig"); - run(R"xrpldConfig( -[peers_in_max] -1001 -[peers_out_max] -10 -)xrpldConfig"); - run(R"xrpldConfig( -[peers_in_max] -10 -[peers_out_max] -1001 -)xrpldConfig"); - } - - void - run() override - { - testBackoff1(); - testBackoff2(); - testDuplicateOutIn(); - testDuplicateInOut(); - testConfig(); - testInvalidConfig(); - testPeerLimitExceeded(); - testActivateDuplicatePeer(); - testActivateInboundDisabled(); - testAddFixedPeerNoPort(); - testOnConnectedSelfConnection(); - testIsValidAddress(); - testVerifyEndpoints(); - } -}; - -BEAST_DEFINE_TESTSUITE(PeerFinder, peerfinder, xrpl); - -} // namespace xrpl::PeerFinder diff --git a/src/test/server/ServerStatus_test.cpp b/src/test/server/ServerStatus_test.cpp index 60ea622616..5adf6a08f5 100644 --- a/src/test/server/ServerStatus_test.cpp +++ b/src/test/server/ServerStatus_test.cpp @@ -558,10 +558,12 @@ class ServerStatus_test : public beast::unit_test::Suite, public beast::test::En using namespace test::jtx; using namespace boost::asio; using namespace boost::beast::http; - Env env{*this, envconfig([&](std::unique_ptr cfg) { + // Run the server with a single io thread so disconnectClient() below + // can deterministically drain the server's io_context (see its docs). + Env env{*this, singleThreadIo(envconfig([&](std::unique_ptr cfg) { (*cfg)[Sections::kPortRpc].set(Keys::kLimit, std::to_string(limit)); return cfg; - })}; + }))}; auto const section = env.app().config().section(Sections::kPortRpc); // NOLINTBEGIN(bugprone-unchecked-optional-access) @@ -580,16 +582,27 @@ class ServerStatus_test : public beast::unit_test::Suite, public beast::test::En BEAST_EXPECT(!ec); std::vector> clients; - int connectionCount{1}; // starts at 1 because the Env already has one - // for JSONRPCCLient - // for nonzero limits, go one past the limit, although failures happen - // at the limit, so this really leads to the last two clients failing. - // for zero limit, pick an arbitrary nonzero number of clients - all - // should connect fine. + // Env owns a persistent JSON-RPC HTTP client connection to port_rpc as + // part of startup, which counts against this port's connection limit. + // This test wants a known starting occupancy of zero, so for nonzero + // limits it deterministically drops that hidden client and waits for + // the server to register the disconnect before opening its own clients. + // + // Starting from zero is important because the port limit rejects once + // the incremented connection count reaches the configured limit. With a + // zero baseline and N = limit + 1 test-owned clients, exactly the last + // two requests should be rejected. + if (limit != 0) + BEAST_EXPECT(env.disconnectClient()); + + // For nonzero limits, go one past the limit. The port rejects at the + // limit, not only above it, so this yields the last two clients + // failing. For zero limit, pick an arbitrary nonzero number of clients + // and expect them all to succeed. int const testTo = (limit == 0) ? 50 : limit + 1; - while (connectionCount < testTo) + while (static_cast(clients.size()) < testTo) { clients.emplace_back(ip::tcp::socket{ios}, boost::beast::multi_buffer{}); async_connect(clients.back().first, it, yield[ec]); @@ -597,19 +610,24 @@ class ServerStatus_test : public beast::unit_test::Suite, public beast::test::En auto req = makeHTTPRequest(ip, port, to_string(jr), {}); async_write(clients.back().first, req, yield[ec]); BEAST_EXPECT(!ec); - ++connectionCount; } - int readCount = 0; + int successfulReads = 0; for (auto& [soc, buf] : clients) { boost::beast::http::response resp; async_read(soc, buf, resp, yield[ec]); - ++readCount; - // expect the reads to fail for the clients that connected at or - // above the limit. If limit is 0, all reads should succeed - BEAST_EXPECT((limit == 0 || readCount < limit - 1) ? (!ec) : bool(ec)); + if (!ec) + ++successfulReads; } + + // This test cares about the exact number of accepted requests, not which + // specific client observed the rejection. With a zero baseline (the + // hidden Env client dropped above), the server accepts until the + // connection count reaches the limit: all clients for limit 0, else + // limit - 1 of the limit + 1 clients (the last two are rejected). + int const expectedReads = (limit == 0) ? static_cast(clients.size()) : limit - 1; + BEAST_EXPECT(successfulReads == expectedReads); } void diff --git a/src/tests/libxrpl/CMakeLists.txt b/src/tests/libxrpl/CMakeLists.txt index cafe72eff9..8e4ece1234 100644 --- a/src/tests/libxrpl/CMakeLists.txt +++ b/src/tests/libxrpl/CMakeLists.txt @@ -21,18 +21,21 @@ set_target_properties( ) # Lets test sources include the shared helpers as . target_include_directories(xrpl_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) -target_link_libraries(xrpl_tests PRIVATE GTest::gtest xrpl.libxrpl) +target_link_libraries(xrpl_tests PRIVATE GTest::gtest GTest::gmock xrpl.libxrpl) # One source subdirectory per module. Network unit tests are currently not # supported on Windows. set(test_modules basics + consensus crypto json + peerfinder resource shamap tx protocol_autogen + nodestore ) if(NOT WIN32) list(APPEND test_modules net) @@ -57,6 +60,16 @@ foreach(module IN LISTS test_modules) ) endforeach() +# The consensus tests use the CSF (Consensus Simulation Framework) helpers, so +# compile the CSF sources into the test binary. The consensus engine itself now +# lives in libxrpl, so no xrpld sources or include paths are needed here. +file( + GLOB_RECURSE csf_sources + CONFIGURE_DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/csf/*.cpp" +) +target_sources(xrpl_tests PRIVATE ${csf_sources}) + # The test helpers and per-module test headers are not built with add_module, # so verify them against the test binary's own compile environment. if(verify_headers) diff --git a/src/tests/libxrpl/basics/base_uint_test.cpp b/src/tests/libxrpl/basics/base_uint.cpp similarity index 99% rename from src/tests/libxrpl/basics/base_uint_test.cpp rename to src/tests/libxrpl/basics/base_uint.cpp index c9bfc35c94..174cc33aa0 100644 --- a/src/tests/libxrpl/basics/base_uint_test.cpp +++ b/src/tests/libxrpl/basics/base_uint.cpp @@ -1,5 +1,6 @@ -#include #include + +#include #include #include diff --git a/src/tests/libxrpl/consensus/ByzantineFailureSim.cpp b/src/tests/libxrpl/consensus/ByzantineFailureSim.cpp new file mode 100644 index 0000000000..e712817121 --- /dev/null +++ b/src/tests/libxrpl/consensus/ByzantineFailureSim.cpp @@ -0,0 +1,81 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace xrpl::test { + +TEST(ByzantineFailureSimTest, DISABLED_byzantine_failure_sim) +{ + using namespace csf; + using namespace std::chrono; + + // This test simulates a specific topology with nodes generating + // different ledgers due to a simulated byzantine failure (injecting + // an extra non-consensus transaction). + + Sim sim; + ConsensusParms const parms{}; + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + PeerGroup a = sim.createGroup(1); + PeerGroup b = sim.createGroup(1); + PeerGroup c = sim.createGroup(1); + PeerGroup d = sim.createGroup(1); + PeerGroup e = sim.createGroup(1); + PeerGroup f = sim.createGroup(1); + PeerGroup g = sim.createGroup(1); + + a.trustAndConnect(a + b + c + g, delay); + b.trustAndConnect(b + a + c + d + e, delay); + c.trustAndConnect(c + a + b + d + e, delay); + d.trustAndConnect(d + b + c + e + f, delay); + e.trustAndConnect(e + b + c + d + f, delay); + f.trustAndConnect(f + d + e + g, delay); + g.trustAndConnect(g + a + f, delay); + + PeerGroup const network = a + b + c + d + e + f + g; + + StreamCollector sc{std::cout}; + + sim.collectors.add(sc); + + for (TrustGraph::ForkInfo const& fi : sim.trustGraph.forkablePairs(0.8)) + { + std::cout << "Can fork " << PeerGroup{fi.unlA} << " " + << " " << PeerGroup{fi.unlB} << " overlap " << fi.overlap << " required " + << fi.required << "\n"; + }; + + // set prior state + sim.run(1); + + PeerGroup byzantineNodes = a + b + c + g; + // All peers see some TX 0 + for (Peer* peer : network) + { + peer->submit(Tx{0}); + // Peers 0,1,2,6 will close the next ledger differently by injecting + // a non-consensus approved transaction + if (byzantineNodes.contains(peer)) + { + peer->txInjections.emplace(peer->lastClosedLedger.seq(), Tx{42}); + } + } + sim.run(4); + std::cout << "Branches: " << sim.branches() << "\n"; + std::cout << "Fully synchronized: " << std::boolalpha << sim.synchronized() << "\n"; + // Not tessting anything currently. + SUCCEED(); +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/CensorshipDetector.cpp b/src/tests/libxrpl/consensus/CensorshipDetector.cpp new file mode 100644 index 0000000000..aa6b2d086b --- /dev/null +++ b/src/tests/libxrpl/consensus/CensorshipDetector.cpp @@ -0,0 +1,81 @@ +#include + +#include + +#include +#include +#include + +namespace xrpl::test { + +namespace { + +void +runRound( + CensorshipDetector& cdet, + int round, + std::vector proposed, + std::vector accepted, + std::vector remain, + std::vector remove) +{ + // Begin tracking what we're proposing this round + CensorshipDetector::TxIDSeqVec proposal; + for (auto const& i : proposed) + proposal.emplace_back(i, round); + cdet.propose(std::move(proposal)); + + // Finalize the round, by processing what we accepted; then + // remove anything that needs to be removed and ensure that + // what remains is correct. + cdet.check(std::move(accepted), [&remove, &remain](auto id, auto seq) { + // If the item is supposed to be removed from the censorship + // detector internal tracker manually, do it now: + if (std::ranges::find(remove, id) != remove.end()) + return true; + + // If the item is supposed to still remain in the censorship + // detector internal tracker; remove it from the vector. + auto it = std::ranges::find(remain, id); + if (it != remain.end()) + remain.erase(it); + return false; + }); + + // On entry, this set contained all the elements that should be tracked + // by the detector after we process this round. We removed all the items + // that actually were in the tracker, so this should now be empty: + EXPECT_TRUE(remain.empty()); +} + +} // namespace + +TEST(CensorshipDetectorTest, censorship_detector) +{ + SCOPED_TRACE("Censorship Detector"); + + CensorshipDetector cdet; + int round = 0; + // proposed accepted remain remove + runRound(cdet, ++round, {}, {}, {}, {}); + runRound(cdet, ++round, {10, 11, 12, 13}, {11, 2}, {10, 13}, {}); + runRound(cdet, ++round, {10, 13, 14, 15}, {14}, {10, 13, 15}, {}); + runRound(cdet, ++round, {10, 13, 15, 16}, {15, 16}, {10, 13}, {}); + runRound(cdet, ++round, {10, 13}, {17, 18}, {10, 13}, {}); + runRound(cdet, ++round, {10, 19}, {}, {10, 19}, {}); + runRound(cdet, ++round, {10, 19, 20}, {20}, {10}, {19}); + runRound(cdet, ++round, {21}, {21}, {}, {}); + runRound(cdet, ++round, {}, {22}, {}, {}); + runRound(cdet, ++round, {23, 24, 25, 26}, {25, 27}, {23, 26}, {24}); + runRound(cdet, ++round, {23, 26, 28}, {26, 28}, {23}, {}); + + for (int i = 0; i != 10; ++i) + runRound(cdet, ++round, {23}, {}, {23}, {}); + + runRound(cdet, ++round, {23, 29}, {29}, {23}, {}); + runRound(cdet, ++round, {30, 31}, {31}, {30}, {}); + runRound(cdet, ++round, {30}, {30}, {}, {}); + runRound(cdet, ++round, {}, {}, {}, {}); +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/Consensus.cpp b/src/tests/libxrpl/consensus/Consensus.cpp new file mode 100644 index 0000000000..d303d28e89 --- /dev/null +++ b/src/tests/libxrpl/consensus/Consensus.cpp @@ -0,0 +1,1455 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::test { + +namespace { + +beast::Journal +journal() +{ + return beast::Journal{TestSink::instance()}; +} + +bool +shouldCloseLedger( + bool anyTransactions, + std::size_t prevProposers, + std::size_t proposersClosed, + std::size_t proposersValidated, + std::chrono::milliseconds prevRoundTime, + std::chrono::milliseconds timeSincePrevClose, + std::chrono::milliseconds openTime, + std::chrono::milliseconds idleInterval, + ConsensusParms const& parms, + std::unique_ptr const& clog = {}) +{ + return xrpl::shouldCloseLedger( + anyTransactions, + prevProposers, + proposersClosed, + proposersValidated, + prevRoundTime, + timeSincePrevClose, + openTime, + idleInterval, + parms, + journal(), + clog); +} + +ConsensusState +checkConsensus( + std::size_t prevProposers, + std::size_t currentProposers, + std::size_t currentAgree, + std::size_t currentFinished, + std::chrono::milliseconds previousAgreeTime, + std::chrono::milliseconds currentAgreeTime, + bool stalled, + ConsensusParms const& parms, + bool proposing, + std::unique_ptr const& clog = {}) +{ + return xrpl::checkConsensus( + prevProposers, + currentProposers, + currentAgree, + currentFinished, + previousAgreeTime, + currentAgreeTime, + stalled, + parms, + proposing, + journal(), + clog); +} + +using CsfDisputedTx = DisputedTx; + +CsfDisputedTx +makeDisputedTx(csf::Tx tx, bool ourVote, std::size_t numPeers) +{ + return CsfDisputedTx{tx, ourVote, numPeers, journal()}; +} + +bool +isStalled( + CsfDisputedTx const& dispute, + ConsensusParms const& parms, + bool proposing, + int peersUnchanged, + std::unique_ptr const& clog) +{ + return dispute.stalled(parms, proposing, peersUnchanged, journal(), clog); +} + +// Helper collector for testPreferredByBranch +// Invasively disconnects network at bad times to cause splits +struct Disruptor +{ + csf::PeerGroup& network; + csf::PeerGroup& groupCfast; + csf::PeerGroup& groupCsplit; + csf::SimDuration delay; + bool reconnected = false; + + Disruptor(csf::PeerGroup& net, csf::PeerGroup& c, csf::PeerGroup& split, csf::SimDuration d) + : network(net), groupCfast(c), groupCsplit(split), delay(d) + { + } + + template + void + on(csf::PeerID, csf::SimTime, E const&) + { + } + + void + on(csf::PeerID who, csf::SimTime, csf::FullyValidateLedger const& e) + { + using namespace std::chrono; + // As soon as the fastC node fully validates C, disconnect + // ALL c nodes from the network. The fast C node needs to disconnect + // as well to prevent it from relaying the validations it did see + if (who == groupCfast[0]->id && e.ledger.seq() == csf::Ledger::Seq{2}) + { + network.disconnect(groupCsplit); + network.disconnect(groupCfast); + } + } + + void + on(csf::PeerID who, csf::SimTime, csf::AcceptLedger const& e) + { + // As soon as anyone generates a child of B or C, reconnect the + // network so those validations make it through + if (!reconnected && e.ledger.seq() == csf::Ledger::Seq{3}) + { + reconnected = true; + network.connect(groupCsplit, delay); + } + } +}; + +// Helper collector for testPauseForLaggards +// This will remove the ledgerAccept delay used to +// initially create the slow vs. fast validator groups. +struct UndoDelay +{ + csf::PeerGroup& g; + + UndoDelay(csf::PeerGroup& a) : g(a) + { + } + + template + void + on(csf::PeerID, csf::SimTime, E const&) + { + } + + void + on(csf::PeerID who, csf::SimTime, csf::AcceptLedger const& e) + { + for (csf::Peer* p : g) + { + if (p->id == who) + p->delays.ledgerAccept = std::chrono::seconds{0}; + } + } +}; + +} // namespace + +TEST(ConsensusTest, should_close_ledger) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("should close ledger"); + + // Use default parameters + ConsensusParms const p{}; + + // Bizarre times forcibly close + EXPECT_TRUE(shouldCloseLedger(true, 10, 10, 10, -10s, 10s, 1s, 1s, p)); + EXPECT_TRUE(shouldCloseLedger(true, 10, 10, 10, 100h, 10s, 1s, 1s, p)); + EXPECT_TRUE(shouldCloseLedger(true, 10, 10, 10, 10s, 100h, 1s, 1s, p)); + + // Rest of network has closed + EXPECT_TRUE(shouldCloseLedger(true, 10, 3, 5, 10s, 10s, 10s, 10s, p)); + + // No transactions means wait until end of internval + EXPECT_TRUE(!shouldCloseLedger(false, 10, 0, 0, 1s, 1s, 1s, 10s, p)); + EXPECT_TRUE(shouldCloseLedger(false, 10, 0, 0, 1s, 10s, 1s, 10s, p)); + + // Enforce minimum ledger open time + EXPECT_TRUE(!shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 1s, 10s, p)); + + // Don't go too much faster than last time + EXPECT_TRUE(!shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 3s, 10s, p)); + + EXPECT_TRUE(shouldCloseLedger(true, 10, 0, 0, 10s, 10s, 10s, 10s, p)); +} + +TEST(ConsensusTest, check_consensus) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("check consensus"); + + // Use default parameters + ConsensusParms const p{}; + + /////////////// + // Disputes still in doubt + // + // Not enough time has elapsed + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 2s, false, p, true)); + + // If not enough peers have proposed, ensure + // more time for proposals + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 4s, false, p, true)); + + // Enough time has elapsed and we all agree + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 2, 2, 0, 3s, 10s, false, p, true)); + + // Enough time has elapsed and we don't yet agree + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 1, 0, 3s, 10s, false, p, true)); + + // Our peers have moved on + // Enough time has elapsed and we all agree + EXPECT_TRUE(ConsensusState::MovedOn == checkConsensus(10, 2, 1, 8, 3s, 10s, false, p, true)); + + // If no peers, don't agree until time has passed. + EXPECT_TRUE(ConsensusState::No == checkConsensus(0, 0, 0, 0, 3s, 10s, false, p, true)); + + // Agree if no peers and enough time has passed. + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(0, 0, 0, 0, 3s, 16s, false, p, true)); + + // Expire if too much time has passed without agreement + EXPECT_TRUE(ConsensusState::Expired == checkConsensus(10, 8, 1, 0, 1s, 19s, false, p, true)); + + /////////////// + // Stalled + // + // Not enough time has elapsed + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 2s, true, p, true)); + + // If not enough peers have proposed, ensure + // more time for proposals + EXPECT_TRUE(ConsensusState::No == checkConsensus(10, 2, 2, 0, 3s, 4s, true, p, true)); + + // Enough time has elapsed and we all agree + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 2, 2, 0, 3s, 10s, true, p, true)); + + // Enough time has elapsed and we don't yet agree, but there's nothing + // left to dispute + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 2, 1, 0, 3s, 10s, true, p, true)); + + // Our peers have moved on + // Enough time has elapsed and we all agree, nothing left to dispute + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 2, 1, 8, 3s, 10s, true, p, true)); + + // If no peers, don't agree until time has passed. + EXPECT_TRUE(ConsensusState::No == checkConsensus(0, 0, 0, 0, 3s, 10s, true, p, true)); + + // Agree if no peers and enough time has passed. + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(0, 0, 0, 0, 3s, 16s, true, p, true)); + + // We are done if there's nothing left to dispute, no matter how much + // time has passed + EXPECT_TRUE(ConsensusState::Yes == checkConsensus(10, 8, 1, 0, 1s, 19s, true, p, true)); +} + +TEST(ConsensusTest, standalone) +{ + using namespace std::chrono_literals; + using namespace csf; + SCOPED_TRACE("standalone"); + + Sim s; + PeerGroup const peers = s.createGroup(1); + Peer* peer = peers[0]; + peer->targetLedgers = 1; + peer->start(); + peer->submit(Tx{1}); + + s.scheduler.step(); + + // Inspect that the proper ledger was created + auto const& lcl = peer->lastClosedLedger; + EXPECT_TRUE(peer->prevLedgerID() == lcl.id()); + EXPECT_TRUE(lcl.seq() == Ledger::Seq{1}); + EXPECT_TRUE(lcl.txs().size() == 1); + EXPECT_TRUE(lcl.txs().contains(Tx{1})); + EXPECT_TRUE(peer->prevProposers == 0); +} + +TEST(ConsensusTest, peers_agree) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("peers agree"); + + ConsensusParms const parms{}; + Sim sim; + PeerGroup peers = sim.createGroup(5); + + // Connected trust and network graphs with single fixed delay + peers.trustAndConnect(peers, round(0.2 * parms.ledgerGRANULARITY)); + + // everyone submits their own ID as a TX + for (Peer* p : peers) + p->submit(Tx(static_cast(p->id))); + + sim.run(1); + + // All peers are in sync + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + for (Peer const* peer : peers) + { + auto const& lcl = peer->lastClosedLedger; + EXPECT_TRUE(lcl.id() == peer->prevLedgerID()); + EXPECT_TRUE(lcl.seq() == Ledger::Seq{1}); + // All peers proposed + EXPECT_TRUE(peer->prevProposers == peers.size() - 1); + // All transactions were accepted + for (std::uint32_t i = 0; i < peers.size(); ++i) + EXPECT_TRUE(lcl.txs().contains(Tx{i})); + } + } +} + +TEST(ConsensusTest, slow_peers) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("slow peers"); + + // Several tests of a complete trust graph with a subset of peers + // that have significantly longer network delays to the rest of the + // network + + // Test when a slow peer doesn't delay a consensus quorum (4/5 agree) + { + ConsensusParms const parms{}; + Sim sim; + PeerGroup slow = sim.createGroup(1); + PeerGroup fast = sim.createGroup(4); + PeerGroup network = fast + slow; + + // Fully connected trust graph + network.trust(network); + + // Fast and slow network connections + fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); + + slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); + + // All peers submit their own ID as a transaction + for (Peer* peer : network) + peer->submit(Tx{static_cast(peer->id)}); + + sim.run(1); + + // Verify all peers have same LCL but are missing transaction 0 + // All peers are in sync even with a slower peer 0 + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + for (Peer const* peer : network) + { + auto const& lcl = peer->lastClosedLedger; + EXPECT_TRUE(lcl.id() == peer->prevLedgerID()); + EXPECT_TRUE(lcl.seq() == Ledger::Seq{1}); + + EXPECT_TRUE(peer->prevProposers == network.size() - 1); + EXPECT_TRUE(peer->prevRoundTime == network[0]->prevRoundTime); + + EXPECT_TRUE(not lcl.txs().contains(Tx{0})); + for (std::uint32_t i = 2; i < network.size(); ++i) + EXPECT_TRUE(lcl.txs().contains(Tx{i})); + + // Tx 0 didn't make it + EXPECT_TRUE(peer->openTxs.contains(Tx{0})); + } + } + } + + // Test when the slow peers delay a consensus quorum (4/6 agree) + { + // Run two tests + // 1. The slow peers are participating in consensus + // 2. The slow peers are just observing + + for (auto isParticipant : {true, false}) + { + ConsensusParms const parms{}; + + Sim sim; + PeerGroup slow = sim.createGroup(2); + PeerGroup fast = sim.createGroup(4); + PeerGroup network = fast + slow; + + // Connected trust graph + network.trust(network); + + // Fast and slow network connections + fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); + + slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); + + for (Peer* peer : slow) + peer->runAsValidator = isParticipant; + + // All peers submit their own ID as a transaction and relay it + // to peers + for (Peer* peer : network) + peer->submit(Tx{static_cast(peer->id)}); + + sim.run(1); + + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + // Verify all peers have same LCL but are missing + // transaction 0,1 which was not received by all peers + // before the ledger closed + for (Peer const* peer : network) + { + // Closed ledger has all but transaction 0,1 + auto const& lcl = peer->lastClosedLedger; + EXPECT_TRUE(lcl.seq() == Ledger::Seq{1}); + EXPECT_TRUE(not lcl.txs().contains(Tx{0})); + EXPECT_TRUE(not lcl.txs().contains(Tx{1})); + for (std::uint32_t i = slow.size(); i < network.size(); ++i) + EXPECT_TRUE(lcl.txs().contains(Tx{i})); + + // Tx 0-1 didn't make it + EXPECT_TRUE(peer->openTxs.contains(Tx{0})); + EXPECT_TRUE(peer->openTxs.contains(Tx{1})); + } + + Peer const* slowPeer = slow[0]; + if (isParticipant) + { + EXPECT_TRUE(slowPeer->prevProposers == network.size() - 1); + } + else + { + EXPECT_TRUE(slowPeer->prevProposers == fast.size()); + } + + for (Peer const* peer : fast) + { + // Due to the network link delay settings + // Peer 0 initially proposes {0} + // Peer 1 initially proposes {1} + // Peers 2-5 initially propose {2,3,4,5} + // Since peers 2-5 agree, 4/6 > the initial 50% needed + // to include a disputed transaction, so Peer 0/1 switch + // to agree with those peers. Peer 0/1 then closes with + // an 80% quorum of agreeing positions (5/6) match. + // + // Peers 2-5 do not change position, since tx 0 or tx 1 + // have less than the 50% initial threshold. They also + // cannot declare consensus, since 4/6 agreeing + // positions are < 80% threshold. They therefore need an + // additional timerEntry call to see the updated + // positions from Peer 0 & 1. + + if (isParticipant) + { + EXPECT_TRUE(peer->prevProposers == network.size() - 1); + EXPECT_TRUE(peer->prevRoundTime > slowPeer->prevRoundTime); + } + else + { + EXPECT_TRUE(peer->prevProposers == fast.size() - 1); + // so all peers should have closed together + EXPECT_TRUE(peer->prevRoundTime == slowPeer->prevRoundTime); + } + } + } + } + } +} + +TEST(ConsensusTest, close_time_disagree) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("close time disagree"); + + // This is a very specialized test to get ledgers to disagree on + // the close time. It unfortunately assumes knowledge about current + // timing constants. This is a necessary evil to get coverage up + // pending more extensive refactorings of timing constants. + + // In order to agree-to-disagree on the close time, there must be no + // clear majority of nodes agreeing on a close time. This test + // sets a relative offset to the peers internal clocks so that they + // send proposals with differing times. + + // However, agreement is on the effective close time, not the + // exact close time. The minimum closeTimeResolution is given by + // ledgerPossibleTimeResolutions[0], which is currently 10s. This means + // the skews need to be at least 10 seconds to have different effective + // close times. + + // Complicating this matter is that nodes will ignore proposals + // with times more than proposeFRESHNESS =20s in the past. So at + // the minimum granularity, we have at most 3 types of skews + // (0s,10s,20s). + + // This test therefore has 6 nodes, with 2 nodes having each type of + // skew. Then no majority (1/3 < 1/2) of nodes will agree on an + // actual close time. + + ConsensusParms const parms{}; + Sim sim; + + PeerGroup groupA = sim.createGroup(2); + PeerGroup const groupB = sim.createGroup(2); + PeerGroup const groupC = sim.createGroup(2); + PeerGroup network = groupA + groupB + groupC; + + network.trust(network); + network.connect(network, round(0.2 * parms.ledgerGRANULARITY)); + + // Run consensus without skew until we have a short close time + // resolution + Peer const* firstPeer = *groupA.begin(); + while (firstPeer->lastClosedLedger.closeTimeResolution() >= parms.proposeFRESHNESS) + sim.run(1); + + // Introduce a shift on the time of 2/3 of peers + for (Peer* peer : groupA) + peer->clockSkew = parms.proposeFRESHNESS / 2; + for (Peer* peer : groupB) + peer->clockSkew = parms.proposeFRESHNESS; + + sim.run(1); + + // All nodes agreed to disagree on the close time + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + for (Peer const* peer : network) + EXPECT_TRUE(!peer->lastClosedLedger.closeAgree()); + } +} + +TEST(ConsensusTest, wrong_lcl) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("wrong LCL"); + + // Specialized test to exercise a temporary fork in which some peers + // are working on an incorrect prior ledger. + + ConsensusParms const parms{}; + + // Vary the time it takes to process validations to exercise detecting + // the wrong LCL at different phases of consensus + for (auto validationDelay : {0ms, parms.ledgerMinClose}) + { + // Consider 10 peers: + // 0 1 2 3 4 5 6 7 8 9 + // minority majorityA majorityB + // + // Nodes 0-1 trust nodes 0-4 + // Nodes 2-9 trust nodes 2-9 + // + // By submitting tx 0 to nodes 0-4 and tx 1 to nodes 5-9, + // nodes 0-1 will generate the wrong LCL (with tx 0). The remaining + // nodes will instead accept the ledger with tx 1. + + // Nodes 0-1 will detect this mismatch during a subsequent round + // since nodes 2-4 will validate a different ledger. + + // Nodes 0-1 will acquire the proper ledger from the network and + // resume consensus and eventually generate the dominant network + // ledger. + + // This topology can potentially fork with the above trust relations + // but that is intended for this test. + + Sim sim; + + PeerGroup minority = sim.createGroup(2); + PeerGroup const majorityA = sim.createGroup(3); + PeerGroup const majorityB = sim.createGroup(5); + + PeerGroup majority = majorityA + majorityB; + PeerGroup const network = minority + majority; + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + minority.trustAndConnect(minority + majorityA, delay); + majority.trustAndConnect(majority, delay); + + CollectByNode jumps; + sim.collectors.add(jumps); + + EXPECT_TRUE(sim.trustGraph.canFork(parms.minConsensusPct / 100.)); + + // initial round to set prior state + sim.run(1); + + // Nodes in smaller UNL have seen tx 0, nodes in other unl have seen + // tx 1 + for (Peer* peer : network) + peer->delays.recvValidation = validationDelay; + for (Peer* peer : (minority + majorityA)) + peer->openTxs.insert(Tx{0}); + for (Peer* peer : majorityB) + peer->openTxs.insert(Tx{1}); + + // Run for additional rounds + // With no validation delay, only 2 more rounds are needed. + // 1. Round to generate different ledgers + // 2. Round to detect different prior ledgers (but still generate + // wrong ones) and recover within that round since wrong LCL + // is detected before we close + // + // With a validation delay of ledgerMinClose, we need 3 more + // rounds. + // 1. Round to generate different ledgers + // 2. Round to detect different prior ledgers (but still generate + // wrong ones) but end up declaring consensus on wrong LCL (but + // with the right transaction set!). This is because we detect + // the wrong LCL after we have closed the ledger, so we declare + // consensus based solely on our peer proposals. But we haven't + // had time to acquire the right ledger. + // 3. Round to correct + sim.run(3); + + // The network never actually forks, since node 0-1 never see a + // quorum of validations to fully validate the incorrect chain. + + // However, for a non zero-validation delay, the network is not + // synchronized because nodes 0 and 1 are running one ledger behind + EXPECT_TRUE(sim.branches() == 1); + if (sim.branches() == 1) + { + for (Peer const* peer : majority) + { + // No jumps for majority nodes + EXPECT_TRUE(jumps[peer->id].closeJumps.empty()); + EXPECT_TRUE(jumps[peer->id].fullyValidatedJumps.empty()); + } + for (Peer const* peer : minority) + { + auto& peerJumps = jumps[peer->id]; + // last closed ledger jump between chains + { + EXPECT_TRUE(peerJumps.closeJumps.size() == 1); + if (peerJumps.closeJumps.size() == 1) + { + JumpCollector::Jump const& jump = peerJumps.closeJumps.front(); + // Jump is to a different chain + EXPECT_TRUE(jump.from.seq() <= jump.to.seq()); + EXPECT_TRUE(!jump.to.isAncestor(jump.from)); + } + } + // fully validated jump forward in same chain + { + EXPECT_TRUE(peerJumps.fullyValidatedJumps.size() == 1); + if (peerJumps.fullyValidatedJumps.size() == 1) + { + JumpCollector::Jump const& jump = peerJumps.fullyValidatedJumps.front(); + // Jump is to a different chain with same seq + EXPECT_TRUE(jump.from.seq() < jump.to.seq()); + EXPECT_TRUE(jump.to.isAncestor(jump.from)); + } + } + } + } + } + + { + // Additional test engineered to switch LCL during the establish + // phase. This was added to trigger a scenario that previously + // crashed, in which switchLCL switched from establish to open + // phase, but still processed the establish phase logic. + + // Loner node will accept an initial ledger A, but all other nodes + // accept ledger B a bit later. By delaying the time it takes + // to process a validation, loner node will detect the wrongLCL + // after it is already in the establish phase of the next round. + + Sim sim; + PeerGroup loner = sim.createGroup(1); + PeerGroup const friends = sim.createGroup(3); + loner.trust(loner + friends); + + PeerGroup const others = sim.createGroup(6); + PeerGroup clique = friends + others; + clique.trust(clique); + + PeerGroup network = loner + clique; + network.connect(network, round(0.2 * parms.ledgerGRANULARITY)); + + // initial round to set prior state + sim.run(1); + for (Peer* peer : (loner + friends)) + peer->openTxs.insert(Tx(0)); + for (Peer* peer : others) + peer->openTxs.insert(Tx(1)); + + // Delay validation processing + for (Peer* peer : network) + peer->delays.recvValidation = parms.ledgerGRANULARITY; + + // additional rounds to generate wrongLCL and recover + sim.run(2); + + // Check all peers recovered + for (Peer const* p : network) + EXPECT_TRUE(p->prevLedgerID() == network[0]->prevLedgerID()); + } +} + +TEST(ConsensusTest, consensus_close_time_rounding) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("consensus close time rounding"); + + // This is a specialized test engineered to yield ledgers with different + // close times even though the peers believe they had close time + // consensus on the ledger. + ConsensusParms const parms; + + Sim sim; + + // This requires a group of 4 fast and 2 slow peers to create a + // situation in which a subset of peers requires seeing additional + // proposals to declare consensus. + PeerGroup slow = sim.createGroup(2); + PeerGroup fast = sim.createGroup(4); + PeerGroup network = fast + slow; + + // Connected trust graph + network.trust(network); + + // Fast and slow network connections + fast.connect(fast, round(0.2 * parms.ledgerGRANULARITY)); + slow.connect(network, round(1.1 * parms.ledgerGRANULARITY)); + + // Run to the ledger *prior* to decreasing the resolution + sim.run(kIncreaseLedgerTimeResolutionEvery - 2); + + // In order to create the discrepancy, we want a case where if + // X = effCloseTime(closeTime, resolution, parentCloseTime) + // X != effCloseTime(X, resolution, parentCloseTime) + // + // That is, the effective close time is not a fixed point. This can + // happen if X = parentCloseTime + 1, but a subsequent rounding goes + // to the next highest multiple of resolution. + + // So we want to find an offset (now + offset) % 30s = 15 + // (now + offset) % 20s = 15 + // This way, the next ledger will close and round up Due to the + // network delay settings, the round of consensus will take 5s, so + // the next ledger's close time will + + NetClock::duration when = network[0]->now().time_since_epoch(); + + // Check we are before the 30s to 20s transition + NetClock::duration const resolution = network[0]->lastClosedLedger.closeTimeResolution(); + EXPECT_TRUE(resolution == NetClock::duration{30s}); + + while (((when % NetClock::duration{30s}) != NetClock::duration{15s}) || + ((when % NetClock::duration{20s}) != NetClock::duration{15s})) + when += 1s; + // Advance the clock without consensus running (IS THIS WHAT + // PREVENTS IT IN PRACTICE?) + sim.scheduler.stepFor(NetClock::time_point{when} - network[0]->now()); + + // Run one more ledger with 30s resolution + sim.run(1); + EXPECT_TRUE(sim.synchronized()); + if (sim.synchronized()) + { + // close time should be ahead of clock time since we engineered + // the close time to round up + for (Peer const* peer : network) + { + EXPECT_TRUE(peer->lastClosedLedger.closeTime() > peer->now()); + EXPECT_TRUE(peer->lastClosedLedger.closeAgree()); + } + } + + // All peers submit their own ID as a transaction + for (Peer* peer : network) + peer->submit(Tx{static_cast(peer->id)}); + + // Run 1 more round, this time it will have a decreased + // resolution of 20 seconds. + + // The network delays are engineered so that the slow peers + // initially have the wrong tx hash, but they see a majority + // of agreement from their peers and declare consensus + // + // The trick is that everyone starts with a raw close time of + // 84681s + // Which has + // effCloseTime(86481s, 20s, 86490s) = 86491s + // However, when the slow peers update their position, they change + // the close time to 86451s. The fast peers declare consensus with + // the 86481s as their position still. + // + // When accepted the ledger + // - fast peers use eff(86481s) -> 86491s as the close time + // - slow peers use eff(eff(86481s)) -> eff(86491s) -> 86500s! + + sim.run(1); + + EXPECT_TRUE(sim.synchronized()); +} + +TEST(ConsensusTest, fork) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("fork"); + + std::uint32_t const numPeers = 10; + // Vary overlap between two UNLs + for (std::uint32_t overlap = 0; overlap <= numPeers; ++overlap) + { + ConsensusParms const parms{}; + Sim sim; + + std::uint32_t const numA = (numPeers - overlap) / 2; + std::uint32_t const numB = numPeers - numA - overlap; + + PeerGroup const aOnly = sim.createGroup(numA); + PeerGroup const bOnly = sim.createGroup(numB); + PeerGroup const commonOnly = sim.createGroup(overlap); + + PeerGroup a = aOnly + commonOnly; + PeerGroup b = bOnly + commonOnly; + + PeerGroup const network = a + b; + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + a.trustAndConnect(a, delay); + b.trustAndConnect(b, delay); + + // Initial round to set prior state + sim.run(1); + for (Peer* peer : network) + { + // Nodes have only seen transactions from their neighbors + peer->openTxs.insert(Tx{static_cast(peer->id)}); + for (Peer const* to : sim.trustGraph.trustedPeers(peer)) + peer->openTxs.insert(Tx{static_cast(to->id)}); + } + sim.run(1); + + // Fork should not happen for 40% or greater overlap + // Since the overlapped nodes have a UNL that is the union of the + // two cliques, the maximum sized UNL list is the number of peers + if (overlap > 0.4 * numPeers) + { + EXPECT_TRUE(sim.synchronized()); + } + else + { + // Even if we do fork, there shouldn't be more than 3 ledgers + // One for cliqueA, one for cliqueB and one for nodes in both + EXPECT_TRUE(sim.branches() <= 3); + } + } +} + +TEST(ConsensusTest, hub_network) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("hub network"); + + // Simulate a set of 5 validators that aren't directly connected but + // rely on a single hub node for communication + + ConsensusParms const parms{}; + Sim sim; + PeerGroup validators = sim.createGroup(5); + PeerGroup center = sim.createGroup(1); + validators.trust(validators); + center.trust(validators); + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + validators.connect(center, delay); + + center[0]->runAsValidator = false; + + // prep round to set initial state. + sim.run(1); + + // everyone submits their own ID as a TX and relay it to peers + for (Peer* p : validators) + p->submit(Tx(static_cast(p->id))); + + sim.run(1); + + // All peers are in sync + EXPECT_TRUE(sim.synchronized()); +} + +TEST(ConsensusTest, preferred_by_branch) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("preferred by branch"); + + // Simulate network splits that are prevented from forking when using + // preferred ledger by trie. This is a contrived example that involves + // excessive network splits, but demonstrates the safety improvement + // from the preferred ledger by trie approach. + + // Consider 10 validating nodes that comprise a single common UNL + // Ledger history: + // 1: A + // _/ \_ + // 2: B C + // _/ _/ \_ + // 3: D C' |||||||| (8 different ledgers) + + // - All nodes generate the common ledger A + // - 2 nodes generate B and 8 nodes generate C + // - Only 1 of the C nodes sees all the C validations and fully + // validates C. The rest of the C nodes split at just the right time + // such that they never see any C validations but their own. + // - The C nodes continue and generate 8 different child ledgers. + // - Meanwhile, the D nodes only saw 1 validation for C and 2 + // validations + // for B. + // - The network reconnects and the validations for generation 3 ledgers + // are observed (D and the 8 C's) + // - In the old approach, 2 votes for D outweighs 1 vote for each C' + // so the network would avalanche towards D and fully validate it + // EVEN though C was fully validated by one node + // - In the new approach, 2 votes for D are not enough to outweight the + // 8 implicit votes for C, so nodes will avalanche to C instead + + ConsensusParms const parms{}; + Sim sim; + + // Goes A->B->D + PeerGroup const groupABD = sim.createGroup(2); + // Single node that initially fully validates C before the split + PeerGroup groupCfast = sim.createGroup(1); + // Generates C, but fails to fully validate before the split + PeerGroup groupCsplit = sim.createGroup(7); + + PeerGroup groupNotFastC = groupABD + groupCsplit; + PeerGroup network = groupABD + groupCsplit + groupCfast; + + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + SimDuration const fDelay = round(0.1 * parms.ledgerGRANULARITY); + + network.trust(network); + // C must have a shorter delay to see all the validations before the + // other nodes + network.connect(groupCfast, fDelay); + // The rest of the network is connected at the same speed + groupNotFastC.connect(groupNotFastC, delay); + + Disruptor dc(network, groupCfast, groupCsplit, delay); + sim.collectors.add(dc); + + // Consensus round to generate ledger A + sim.run(1); + EXPECT_TRUE(sim.synchronized()); + + // Next round generates B and C + // To force B, we inject an extra transaction in to those nodes + for (Peer* peer : groupABD) + { + peer->txInjections.emplace(peer->lastClosedLedger.seq(), Tx{42}); + } + // The Disruptor will ensure that nodes disconnect before the C + // validations make it to all but the fastC node + sim.run(1); + + // We are no longer in sync, but have not yet forked: + // 9 nodes consider A the last fully validated ledger and fastC sees C + EXPECT_TRUE(!sim.synchronized()); + EXPECT_TRUE(sim.branches() == 1); + + // Run another round to generate the 8 different C' ledgers + for (Peer* p : network) + p->submit(Tx(static_cast(p->id))); + sim.run(1); + + // Still not forked + EXPECT_TRUE(!sim.synchronized()); + EXPECT_TRUE(sim.branches() == 1); + + // Disruptor will reconnect all but the fastC node + sim.run(1); + + EXPECT_TRUE(sim.branches() == 1); + if (sim.branches() == 1) + { + EXPECT_TRUE(sim.synchronized()); + } + else // old approach caused a fork + { + EXPECT_TRUE(sim.branches(groupNotFastC) == 1); + EXPECT_TRUE(sim.synchronized(groupNotFastC) == 1); + } +} + +TEST(ConsensusTest, pause_for_laggards) +{ + using namespace csf; + using namespace std::chrono; + SCOPED_TRACE("pause for laggards"); + + // Test that validators that jump ahead of the network slow + // down. + + // We engineer the following validated ledger history scenario: + // + // / --> B1 --> C1 --> ... -> G1 "ahead" + // A + // \ --> B2 --> C2 "behind" + // + // After validating a common ledger A, a set of "behind" validators + // briefly run slower and validate the lower chain of ledgers. + // The "ahead" validators run normal speed and run ahead validating the + // upper chain of ledgers. + // + // Due to the uncommitted support definition of the preferred branch + // protocol, even if the "behind" validators are a majority, the "ahead" + // validators cannot jump to the proper branch until the "behind" + // validators catch up to the same sequence number. For this test to + // succeed, the ahead validators need to briefly slow down consensus. + + ConsensusParms const parms{}; + Sim sim; + SimDuration const delay = round(0.2 * parms.ledgerGRANULARITY); + + PeerGroup behind = sim.createGroup(3); + PeerGroup const ahead = sim.createGroup(2); + PeerGroup network = ahead + behind; + + hash_set trustedKeys; + for (Peer const* p : network) + trustedKeys.insert(p->key); + for (Peer* p : network) + p->trustedKeys = trustedKeys; + + network.trustAndConnect(network, delay); + + // Initial seed round to set prior state + sim.run(1); + + // Have the "behind" group initially take a really long time to + // accept a ledger after ending deliberation + for (Peer* p : behind) + p->delays.ledgerAccept = 20s; + + // Use the collector to revert the delay after the single + // slow ledger is generated + UndoDelay undoDelay{behind}; + sim.collectors.add(undoDelay); + + // Run the simulation for 100 seconds of simulation time with + std::chrono::nanoseconds const simDuration = 100s; + + // Simulate clients submitting 1 tx every 5 seconds to a random + // validator + Rate const rate{.count = 1, .duration = 5s}; + auto peerSelector = makeSelector( + network.begin(), network.end(), std::vector(network.size(), 1.), sim.rng); + auto txSubmitter = makeSubmitter( + ConstantDistribution{rate.inv()}, + sim.scheduler.now(), + sim.scheduler.now() + simDuration, + peerSelector, + sim.scheduler, + sim.rng); + + // Run simulation + sim.run(simDuration); + + // Verify that the network recovered + EXPECT_TRUE(sim.synchronized()); +} + +TEST(ConsensusTest, disputes) +{ + SCOPED_TRACE("disputes"); + + using namespace csf; + + // Test dispute objects directly + using Dispute = CsfDisputedTx; + + Tx const txTrue{99}; + Tx const txFalse{98}; + Tx const txFollowingTrue{97}; + Tx const txFollowingFalse{96}; + int const numPeers = 100; + ConsensusParms const p; + std::size_t peersUnchanged = 0; + + auto clog = std::make_unique(); + + // Three cases: + // 1 proposing, initial vote yes + // 2 proposing, initial vote no + // 3 not proposing, initial vote doesn't matter after the first update, + // use yes + { + Dispute proposingTrue = makeDisputedTx(txTrue, true, numPeers); + Dispute proposingFalse = makeDisputedTx(txFalse, false, numPeers); + Dispute followingTrue = makeDisputedTx(txFollowingTrue, true, numPeers); + Dispute followingFalse = makeDisputedTx(txFollowingFalse, false, numPeers); + EXPECT_TRUE(proposingTrue.id() == 99); + EXPECT_TRUE(proposingFalse.id() == 98); + EXPECT_TRUE(followingTrue.id() == 97); + EXPECT_TRUE(followingFalse.id() == 96); + + // Create an even split in the peer votes + for (int i = 0; i < numPeers; ++i) + { + EXPECT_TRUE(proposingTrue.setVote(PeerID(i), i < 50)); + EXPECT_TRUE(proposingFalse.setVote(PeerID(i), i < 50)); + EXPECT_TRUE(followingTrue.setVote(PeerID(i), i < 50)); + EXPECT_TRUE(followingFalse.setVote(PeerID(i), i < 50)); + } + // Switch the middle vote to match mine + EXPECT_TRUE(proposingTrue.setVote(PeerID(50), true)); + EXPECT_TRUE(proposingFalse.setVote(PeerID(49), false)); + EXPECT_TRUE(followingTrue.setVote(PeerID(50), true)); + EXPECT_TRUE(followingFalse.setVote(PeerID(49), false)); + + // no changes yet + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(proposingFalse, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingTrue, p, false, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingFalse, p, false, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()); + + // I'm in the majority, my vote should not change + EXPECT_TRUE(!proposingTrue.updateVote(5, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(5, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(5, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(5, false, p)); + + EXPECT_TRUE(!proposingTrue.updateVote(10, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(10, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(10, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(10, false, p)); + + peersUnchanged = 2; + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(proposingFalse, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingTrue, p, false, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingFalse, p, false, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()); + + // Right now, the vote is 51%. The requirement is about to jump to + // 65% + EXPECT_TRUE(proposingTrue.updateVote(55, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(55, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(55, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(55, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == false); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + // 16 validators change their vote to match my original vote + for (int i = 0; i < 16; ++i) + { + auto pTrue = PeerID(numPeers - i - 1); + auto pFalse = PeerID(i); + EXPECT_TRUE(proposingTrue.setVote(pTrue, true)); + EXPECT_TRUE(proposingFalse.setVote(pFalse, false)); + EXPECT_TRUE(followingTrue.setVote(pTrue, true)); + EXPECT_TRUE(followingFalse.setVote(pFalse, false)); + } + // The vote should now be 66%, threshold is 65% + EXPECT_TRUE(proposingTrue.updateVote(60, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(60, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(60, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(60, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // Threshold jumps to 70% + EXPECT_TRUE(proposingTrue.updateVote(86, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(86, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(86, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(86, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == false); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // 5 more validators change their vote to match my original vote + for (int i = 16; i < 21; ++i) + { + auto pTrue = PeerID(numPeers - i - 1); + auto pFalse = PeerID(i); + EXPECT_TRUE(proposingTrue.setVote(pTrue, true)); + EXPECT_TRUE(proposingFalse.setVote(pFalse, false)); + EXPECT_TRUE(followingTrue.setVote(pTrue, true)); + EXPECT_TRUE(followingFalse.setVote(pFalse, false)); + } + + // The vote should now be 71%, threshold is 70% + EXPECT_TRUE(proposingTrue.updateVote(90, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(90, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(90, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(90, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // The vote should now be 71%, threshold is 70% + EXPECT_TRUE(!proposingTrue.updateVote(150, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(150, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(150, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(150, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // The vote should now be 71%, threshold is 70% + EXPECT_TRUE(!proposingTrue.updateVote(190, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(190, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(190, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(190, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + peersUnchanged = 3; + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(proposingFalse, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingTrue, p, false, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingFalse, p, false, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()); + + // Threshold jumps to 95% + EXPECT_TRUE(proposingTrue.updateVote(220, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(220, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(220, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(220, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == false); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // 25 more validators change their vote to match my original vote + for (int i = 21; i < 46; ++i) + { + auto pTrue = PeerID(numPeers - i - 1); + auto pFalse = PeerID(i); + EXPECT_TRUE(proposingTrue.setVote(pTrue, true)); + EXPECT_TRUE(proposingFalse.setVote(pFalse, false)); + EXPECT_TRUE(followingTrue.setVote(pTrue, true)); + EXPECT_TRUE(followingFalse.setVote(pFalse, false)); + } + + // The vote should now be 96%, threshold is 95% + EXPECT_TRUE(proposingTrue.updateVote(250, true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(250, true, p)); + EXPECT_TRUE(!followingTrue.updateVote(250, false, p)); + EXPECT_TRUE(!followingFalse.updateVote(250, false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + for (peersUnchanged = 0; peersUnchanged < 6; ++peersUnchanged) + { + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(proposingFalse, p, true, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingTrue, p, false, peersUnchanged, clog)); + EXPECT_TRUE(!isStalled(followingFalse, p, false, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()); + } + + auto expectStalled = [&clog]( + int txid, + bool ourVote, + int ourTime, + int peerTime, + int support, + std::uint32_t line) { + using namespace std::string_literals; + + auto const s = clog->str(); + SCOPED_TRACE(::testing::Message() << __FILE__ << ":" << line); + EXPECT_NE(s.find("stalled"), s.npos) << s; + EXPECT_TRUE(s.starts_with("Transaction "s + std::to_string(txid))) << s; + EXPECT_NE(s.find("voting "s + (ourVote ? "YES" : "NO")), s.npos) << s; + EXPECT_NE(s.find("for "s + std::to_string(ourTime) + " rounds."s), s.npos) << s; + EXPECT_NE(s.find("votes in "s + std::to_string(peerTime) + " rounds."), s.npos) << s; + EXPECT_TRUE(s.ends_with("has "s + std::to_string(support) + "% support. "s)) << s; + clog = std::make_unique(); + }; + + for (int i = 0; i < 1; ++i) + { + EXPECT_TRUE(!proposingTrue.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!followingTrue.updateVote(250 + (10 * i), false, p)); + EXPECT_TRUE(!followingFalse.updateVote(250 + (10 * i), false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // true vote has changed recently, so not stalled + EXPECT_TRUE(!isStalled(proposingTrue, p, true, 0, clog)); + EXPECT_TRUE(clog->str().empty()); + // remaining votes have been unchanged in so long that we only + // need to hit the second round at 95% to be stalled, regardless + // of peers + EXPECT_TRUE(isStalled(proposingFalse, p, true, 0, clog)); + expectStalled(98, false, 11, 0, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, 0, clog)); + expectStalled(97, true, 11, 0, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, 0, clog)); + expectStalled(96, false, 11, 0, 3, __LINE__); + + // true vote has changed recently, so not stalled + EXPECT_TRUE(!isStalled(proposingTrue, p, true, peersUnchanged, clog)); + EXPECT_TRUE(clog->str().empty()) << clog->str(); + // remaining votes have been unchanged in so long that we only + // need to hit the second round at 95% to be stalled, regardless + // of peers + EXPECT_TRUE(isStalled(proposingFalse, p, true, peersUnchanged, clog)); + expectStalled(98, false, 11, 6, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, peersUnchanged, clog)); + expectStalled(97, true, 11, 6, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, peersUnchanged, clog)); + expectStalled(96, false, 11, 6, 3, __LINE__); + } + for (int i = 1; i < 3; ++i) + { + EXPECT_TRUE(!proposingTrue.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!followingTrue.updateVote(250 + (10 * i), false, p)); + EXPECT_TRUE(!followingFalse.updateVote(250 + (10 * i), false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + // true vote changed 2 rounds ago, and peers are changing, so + // not stalled + EXPECT_TRUE(!isStalled(proposingTrue, p, true, 0, clog)); + EXPECT_TRUE(clog->str().empty()) << clog->str(); + // still stalled + EXPECT_TRUE(isStalled(proposingFalse, p, true, 0, clog)); + expectStalled(98, false, 11 + i, 0, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, 0, clog)); + expectStalled(97, true, 11 + i, 0, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, 0, clog)); + expectStalled(96, false, 11 + i, 0, 3, __LINE__); + + // true vote changed 2 rounds ago, and peers are NOT changing, + // so stalled + EXPECT_TRUE(isStalled(proposingTrue, p, true, peersUnchanged, clog)); + expectStalled(99, true, 1 + i, 6, 97, __LINE__); + // still stalled + EXPECT_TRUE(isStalled(proposingFalse, p, true, peersUnchanged, clog)); + expectStalled(98, false, 11 + i, 6, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, peersUnchanged, clog)); + expectStalled(97, true, 11 + i, 6, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, peersUnchanged, clog)); + expectStalled(96, false, 11 + i, 6, 3, __LINE__); + } + for (int i = 3; i < 5; ++i) + { + EXPECT_TRUE(!proposingTrue.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!proposingFalse.updateVote(250 + (10 * i), true, p)); + EXPECT_TRUE(!followingTrue.updateVote(250 + (10 * i), false, p)); + EXPECT_TRUE(!followingFalse.updateVote(250 + (10 * i), false, p)); + + EXPECT_TRUE(proposingTrue.getOurVote() == true); + EXPECT_TRUE(proposingFalse.getOurVote() == false); + EXPECT_TRUE(followingTrue.getOurVote() == true); + EXPECT_TRUE(followingFalse.getOurVote() == false); + + EXPECT_TRUE(isStalled(proposingTrue, p, true, 0, clog)); + expectStalled(99, true, 1 + i, 0, 97, __LINE__); + EXPECT_TRUE(isStalled(proposingFalse, p, true, 0, clog)); + expectStalled(98, false, 11 + i, 0, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, 0, clog)); + expectStalled(97, true, 11 + i, 0, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, 0, clog)); + expectStalled(96, false, 11 + i, 0, 3, __LINE__); + + EXPECT_TRUE(isStalled(proposingTrue, p, true, peersUnchanged, clog)); + expectStalled(99, true, 1 + i, 6, 97, __LINE__); + EXPECT_TRUE(isStalled(proposingFalse, p, true, peersUnchanged, clog)); + expectStalled(98, false, 11 + i, 6, 2, __LINE__); + EXPECT_TRUE(isStalled(followingTrue, p, false, peersUnchanged, clog)); + expectStalled(97, true, 11 + i, 6, 97, __LINE__); + EXPECT_TRUE(isStalled(followingFalse, p, false, peersUnchanged, clog)); + expectStalled(96, false, 11 + i, 6, 3, __LINE__); + } + } +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/DistributedValidatorsSim.cpp b/src/tests/libxrpl/consensus/DistributedValidatorsSim.cpp new file mode 100644 index 0000000000..f5ba508cbc --- /dev/null +++ b/src/tests/libxrpl/consensus/DistributedValidatorsSim.cpp @@ -0,0 +1,252 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::test { + +namespace { + +[[nodiscard]] std::string const& +arg() +{ + static std::string const kEMPTY; + return kEMPTY; +} + +void +completeTrustCompleteConnectFixedDelay( + std::size_t numPeers, + std::chrono::milliseconds delay = std::chrono::milliseconds(200), + bool printHeaders = false) +{ + using namespace csf; + using namespace std::chrono; + + // Initialize persistent collector logs specific to this method + std::string const prefix = + "DistributedValidators_" + "completeTrustCompleteConnectFixedDelay"; + std::fstream txLog(prefix + "_tx.csv", std::ofstream::app), + ledgerLog(prefix + "_ledger.csv", std::ofstream::app); + + // title + std::cout << prefix << "(" << numPeers << "," << delay.count() << ")" << std::endl; + + // number of peers, UNLs, connections + EXPECT_TRUE(numPeers >= 1); + + Sim sim; + PeerGroup peers = sim.createGroup(numPeers); + + // complete trust graph + peers.trust(peers); + + // complete connect graph with fixed delay + peers.connect(peers, delay); + + // Initialize collectors to track statistics to report + TxCollector txCollector; + LedgerCollector ledgerCollector; + auto colls = makeCollectors(txCollector, ledgerCollector); + sim.collectors.add(colls); + + // Initial round to set prior state + sim.run(1); + + // Run for 10 minutes, submitting 100 tx/second + std::chrono::nanoseconds const simDuration = 10min; + std::chrono::nanoseconds const quiet = 10s; + Rate const rate{.count = 100, .duration = 1000ms}; + + // Initialize timers + HeartbeatTimer heart(sim.scheduler); + + // txs, start/stop/step, target + auto peerSelector = + makeSelector(peers.begin(), peers.end(), std::vector(numPeers, 1.), sim.rng); + auto txSubmitter = makeSubmitter( + ConstantDistribution{rate.inv()}, + sim.scheduler.now() + quiet, + sim.scheduler.now() + simDuration - quiet, + peerSelector, + sim.scheduler, + sim.rng); + + // run simulation for given duration + heart.start(); + sim.run(simDuration); + + // EXPECT_TRUE(sim.branches() == 1); + // EXPECT_TRUE(sim.synchronized()); + + std::cout << std::right; + std::cout << "| Peers: " << std::setw(2) << peers.size(); + std::cout << " | Duration: " << std::setw(6) << duration_cast(simDuration).count() + << " ms"; + std::cout << " | Branches: " << std::setw(1) << sim.branches(); + std::cout << " | Synchronized: " << std::setw(1) << (sim.synchronized() ? "Y" : "N"); + std::cout << " |" << std::endl; + + txCollector.report(simDuration, std::cout, true); + ledgerCollector.report(simDuration, std::cout, false); + + std::string const tag = std::to_string(numPeers); + txCollector.csv(simDuration, txLog, tag, printHeaders); + ledgerCollector.csv(simDuration, ledgerLog, tag, printHeaders); + + std::cout << std::endl; +} + +void +completeTrustScaleFreeConnectFixedDelay( + std::size_t numPeers, + std::chrono::milliseconds delay = std::chrono::milliseconds(200), + bool printHeaders = false) +{ + using namespace csf; + using namespace std::chrono; + + // Initialize persistent collector logs specific to this method + std::string const prefix = + "DistributedValidators__" + "completeTrustScaleFreeConnectFixedDelay"; + std::fstream txLog(prefix + "_tx.csv", std::ofstream::app), + ledgerLog(prefix + "_ledger.csv", std::ofstream::app); + + // title + std::cout << prefix << "(" << numPeers << "," << delay.count() << ")" << std::endl; + + // number of peers, UNLs, connections + int const numCNLs = std::max(int(1.00 * numPeers), 1); + int const minCNLSize = std::max(int(0.25 * numCNLs), 1); + int const maxCNLSize = std::max(int(0.50 * numCNLs), 1); + EXPECT_TRUE(numPeers >= 1); + EXPECT_TRUE(numCNLs >= 1); + EXPECT_TRUE(1 <= minCNLSize && minCNLSize <= maxCNLSize && maxCNLSize <= numPeers); + + Sim sim; + PeerGroup peers = sim.createGroup(numPeers); + + // complete trust graph + peers.trust(peers); + + // scale-free connect graph with fixed delay + std::vector const ranks = sample(peers.size(), PowerLawDistribution{1, 3}, sim.rng); + randomRankedConnect( + peers, + ranks, + numCNLs, + std::uniform_int_distribution<>{minCNLSize, maxCNLSize}, + sim.rng, + delay); + + // Initialize collectors to track statistics to report + TxCollector txCollector; + LedgerCollector ledgerCollector; + auto colls = makeCollectors(txCollector, ledgerCollector); + sim.collectors.add(colls); + + // Initial round to set prior state + sim.run(1); + + // Run for 10 minutes, submitting 100 tx/second + std::chrono::nanoseconds const simDuration = 10min; + std::chrono::nanoseconds const quiet = 10s; + Rate const rate{.count = 100, .duration = 1000ms}; + + // Initialize timers + HeartbeatTimer heart(sim.scheduler); + + // txs, start/stop/step, target + auto peerSelector = + makeSelector(peers.begin(), peers.end(), std::vector(numPeers, 1.), sim.rng); + auto txSubmitter = makeSubmitter( + ConstantDistribution{rate.inv()}, + sim.scheduler.now() + quiet, + sim.scheduler.now() + simDuration - quiet, + peerSelector, + sim.scheduler, + sim.rng); + + // run simulation for given duration + heart.start(); + sim.run(simDuration); + + // EXPECT_TRUE(sim.branches() == 1); + // EXPECT_TRUE(sim.synchronized()); + + std::cout << std::right; + std::cout << "| Peers: " << std::setw(2) << peers.size(); + std::cout << " | Duration: " << std::setw(6) << duration_cast(simDuration).count() + << " ms"; + std::cout << " | Branches: " << std::setw(1) << sim.branches(); + std::cout << " | Synchronized: " << std::setw(1) << (sim.synchronized() ? "Y" : "N"); + std::cout << " |" << std::endl; + + txCollector.report(simDuration, std::cout, true); + ledgerCollector.report(simDuration, std::cout, false); + + std::string const tag = std::to_string(numPeers); + txCollector.csv(simDuration, txLog, tag, printHeaders); + ledgerCollector.csv(simDuration, ledgerLog, tag, printHeaders); + + std::cout << std::endl; +} + +} // namespace + +// In progress simulations for diversifying and distributing validators +TEST(DistributedValidatorsTest, DISABLED_distributed_validators) +{ + std::string const defaultArgs = "5 200"; + std::string const args = arg().empty() ? defaultArgs : arg(); + std::stringstream argStream(args); + + int maxNumValidators = 0; + int delayCount(200); + argStream >> maxNumValidators; + argStream >> delayCount; + + std::chrono::milliseconds const delay(delayCount); + + std::cout << "DistributedValidators: 1 to " << maxNumValidators << " Peers" << std::endl; + + // Simulate with N = 1 to N + // - complete trust graph is complete + // - complete network connectivity + // - fixed delay for network links + completeTrustCompleteConnectFixedDelay(1, delay, true); + for (int i = 2; i <= maxNumValidators; i++) + { + completeTrustCompleteConnectFixedDelay(i, delay); + } + + // Simulate with N = 1 to N + // - complete trust graph is complete + // - scale-free network connectivity + // - fixed delay for network links + completeTrustScaleFreeConnectFixedDelay(1, delay, true); + for (int i = 2; i <= maxNumValidators; i++) + { + completeTrustScaleFreeConnectFixedDelay(i, delay); + } +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/LedgerTiming.cpp b/src/tests/libxrpl/consensus/LedgerTiming.cpp new file mode 100644 index 0000000000..0cab895fda --- /dev/null +++ b/src/tests/libxrpl/consensus/LedgerTiming.cpp @@ -0,0 +1,105 @@ +#include + +#include + +#include + +#include +#include +#include + +namespace xrpl::test { + +TEST(LedgerTimingTest, get_next_ledger_time_resolution) +{ + // helper to iteratively call into getNextLedgerTimeResolution + struct TestRes + { + std::uint32_t decrease = 0; + std::uint32_t equal = 0; + std::uint32_t increase = 0; + + static TestRes + run(bool previousAgree, std::uint32_t rounds) + { + TestRes res; + auto closeResolution = kLedgerDefaultTimeResolution; + auto nextCloseResolution = closeResolution; + std::uint32_t round = 0; + do + { + nextCloseResolution = + getNextLedgerTimeResolution(closeResolution, previousAgree, ++round); + if (nextCloseResolution < closeResolution) + { + ++res.decrease; + } + else if (nextCloseResolution > closeResolution) + { + ++res.increase; + } + else + { + ++res.equal; + } + std::swap(nextCloseResolution, closeResolution); + } while (round < rounds); + return res; + } + }; + + // If we never agree on close time, only can increase resolution + // until hit the max + auto decreases = TestRes::run(false, 10); + EXPECT_TRUE(decreases.increase == 3); + EXPECT_TRUE(decreases.decrease == 0); + EXPECT_TRUE(decreases.equal == 7); + + // If we always agree on close time, only can decrease resolution + // until hit the min + auto increases = TestRes::run(false, 100); + EXPECT_TRUE(increases.increase == 3); + EXPECT_TRUE(increases.decrease == 0); + EXPECT_TRUE(increases.equal == 97); +} + +TEST(LedgerTimingTest, round_close_time) +{ + using namespace std::chrono_literals; + // A closeTime equal to the epoch is not modified + using tp = NetClock::time_point; + tp const def; + EXPECT_TRUE(def == roundCloseTime(def, 30s)); + + // Otherwise, the closeTime is rounded to the nearest + // rounding up on ties + EXPECT_TRUE(tp{0s} == roundCloseTime(tp{29s}, 60s)); + EXPECT_TRUE(tp{30s} == roundCloseTime(tp{30s}, 1s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{31s}, 60s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{30s}, 60s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{59s}, 60s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{60s}, 60s)); + EXPECT_TRUE(tp{60s} == roundCloseTime(tp{61s}, 60s)); +} + +TEST(LedgerTimingTest, eff_close_time) +{ + using namespace std::chrono_literals; + using tp = NetClock::time_point; + tp close = effCloseTime(tp{10s}, 30s, tp{0s}); + EXPECT_TRUE(close == tp{1s}); + + close = effCloseTime(tp{16s}, 30s, tp{0s}); + EXPECT_TRUE(close == tp{30s}); + + close = effCloseTime(tp{16s}, 30s, tp{30s}); + EXPECT_TRUE(close == tp{31s}); + + close = effCloseTime(tp{16s}, 30s, tp{60s}); + EXPECT_TRUE(close == tp{61s}); + + close = effCloseTime(tp{31s}, 30s, tp{0s}); + EXPECT_TRUE(close == tp{30s}); +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/LedgerTrie.cpp b/src/tests/libxrpl/consensus/LedgerTrie.cpp new file mode 100644 index 0000000000..1259a5049a --- /dev/null +++ b/src/tests/libxrpl/consensus/LedgerTrie.cpp @@ -0,0 +1,693 @@ +#include + +#include +#include + +#include +#include +#include + +namespace xrpl::test { + +TEST(LedgerTrieTest, insert) +{ + using namespace csf; + // Single entry by itself + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + + t.insert(h["abc"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 2); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + } + // Suffix of existing (extending tree) + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + EXPECT_TRUE(t.checkInvariants()); + // extend with no siblings + t.insert(h["abcd"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + + // extend with existing sibling + t.insert(h["abce"]); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 3); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abce"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abce"]) == 1); + } + // uncommitted of existing node + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + EXPECT_TRUE(t.checkInvariants()); + // uncommitted with no siblings + t.insert(h["abcdf"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcdf"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcdf"]) == 1); + + // uncommitted with existing child + t.insert(h["abc"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 3); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcdf"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcdf"]) == 1); + } + // Suffix + uncommitted of existing node + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + EXPECT_TRUE(t.checkInvariants()); + t.insert(h["abce"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abce"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abce"]) == 1); + } + // Suffix + uncommitted with existing child + { + // abcd : abcde, abcf + + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + EXPECT_TRUE(t.checkInvariants()); + t.insert(h["abcde"]); + EXPECT_TRUE(t.checkInvariants()); + t.insert(h["abcf"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 3); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcf"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcf"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abcde"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcde"]) == 1); + } + + // Multiple counts + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"], 4); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 4); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 4); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.branchSupport(h["a"]) == 4); + + t.insert(h["abc"], 2); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 2); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 4); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 6); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.branchSupport(h["a"]) == 6); + } +} + +TEST(LedgerTrieTest, remove) +{ + using namespace csf; + // Not in trie + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + + EXPECT_TRUE(!t.remove(h["ab"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(!t.remove(h["a"])); + EXPECT_TRUE(t.checkInvariants()); + } + // In trie but with 0 tip support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"]); + t.insert(h["abce"]); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(!t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + } + // In trie with > 1 tip support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"], 2); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 2); + EXPECT_TRUE(t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + + t.insert(h["abc"], 1); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 2); + EXPECT_TRUE(t.remove(h["abc"], 2)); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + + t.insert(h["abc"], 3); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 3); + EXPECT_TRUE(t.remove(h["abc"], 300)); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + } + // In trie with = 1 tip support, no children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + + EXPECT_TRUE(t.tipSupport(h["ab"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + + EXPECT_TRUE(t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 0); + } + // In trie with = 1 tip support, 1 child + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + t.insert(h["abcd"]); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + + EXPECT_TRUE(t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 1); + } + // In trie with = 1 tip support, > 1 children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + t.insert(h["abcd"]); + t.insert(h["abce"]); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 3); + + EXPECT_TRUE(t.remove(h["abc"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 2); + } + + // In trie with = 1 tip support, parent compaction + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["abc"]); + t.insert(h["abd"]); + EXPECT_TRUE(t.checkInvariants()); + t.remove(h["ab"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abd"]) == 1); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 0); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 2); + + t.remove(h["abd"]); + EXPECT_TRUE(t.checkInvariants()); + + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 1); + } +} + +TEST(LedgerTrieTest, empty) +{ + using namespace csf; + LedgerTrie t; + LedgerHistoryHelper h; + EXPECT_TRUE(t.empty()); + + Ledger const genesis = h[""]; + t.insert(genesis); + EXPECT_TRUE(!t.empty()); + t.remove(genesis); + EXPECT_TRUE(t.empty()); + + t.insert(h["abc"]); + EXPECT_TRUE(!t.empty()); + t.remove(h["abc"]); + EXPECT_TRUE(t.empty()); +} + +TEST(LedgerTrieTest, support) +{ + using namespace csf; + + LedgerTrie t; + LedgerHistoryHelper h; + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.tipSupport(h["axy"]) == 0); + + EXPECT_TRUE(t.branchSupport(h["a"]) == 0); + EXPECT_TRUE(t.branchSupport(h["axy"]) == 0); + + t.insert(h["abc"]); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 0); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abcd"]) == 0); + + EXPECT_TRUE(t.branchSupport(h["a"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abcd"]) == 0); + + t.insert(h["abe"]); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 0); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 1); + EXPECT_TRUE(t.tipSupport(h["abe"]) == 1); + + EXPECT_TRUE(t.branchSupport(h["a"]) == 2); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 2); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abe"]) == 1); + + t.remove(h["abc"]); + EXPECT_TRUE(t.tipSupport(h["a"]) == 0); + EXPECT_TRUE(t.tipSupport(h["ab"]) == 0); + EXPECT_TRUE(t.tipSupport(h["abc"]) == 0); + EXPECT_TRUE(t.tipSupport(h["abe"]) == 1); + + EXPECT_TRUE(t.branchSupport(h["a"]) == 1); + EXPECT_TRUE(t.branchSupport(h["ab"]) == 1); + EXPECT_TRUE(t.branchSupport(h["abc"]) == 0); + EXPECT_TRUE(t.branchSupport(h["abe"]) == 1); +} + +TEST(LedgerTrieTest, get_preferred) +{ + using namespace csf; + using Seq = Ledger::Seq; + // Empty + { + LedgerTrie const t; + EXPECT_TRUE(t.getPreferred(Seq{0}) == std::nullopt); + EXPECT_TRUE(t.getPreferred(Seq{2}) == std::nullopt); + } + // Genesis support is NOT empty + { + LedgerTrie t; + LedgerHistoryHelper h; + Ledger const genesis = h[""]; + t.insert(genesis); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{0})->id == genesis.id()); + EXPECT_TRUE(t.remove(genesis)); + EXPECT_TRUE(t.getPreferred(Seq{0}) == std::nullopt); + EXPECT_TRUE(!t.remove(genesis)); + } + // Single node no children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + } + // Single node smaller child support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"]); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + } + // Single node larger child + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"], 2); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abcd"].id()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcd"].id()); + } + // Single node smaller children support + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"]); + t.insert(h["abce"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + + t.insert(h["abc"]); + + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + // Single node larger children + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"], 2); + t.insert(h["abce"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + + t.insert(h["abcd"]); + + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abcd"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcd"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + // Tie-breaker by id + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abcd"], 2); + t.insert(h["abce"], 2); + + EXPECT_TRUE(h["abce"].id() > h["abcd"].id()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abce"].id()); + + t.insert(h["abcd"]); + EXPECT_TRUE(h["abce"].id() > h["abcd"].id()); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcd"].id()); + } + + // Tie-breaker not needed + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"]); + t.insert(h["abce"], 2); + // abce only has a margin of 1, but it owns the tie-breaker + EXPECT_TRUE(h["abce"].id() > h["abcd"].id()); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abce"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abce"].id()); + + // Switch support from abce to abcd, tie-breaker now needed + t.remove(h["abce"]); + t.insert(h["abcd"]); + + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + + // Single node larger grand child + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcd"], 2); + t.insert(h["abcde"], 4); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abcde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["abcde"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + + // Too much uncommitted support from competing branches + { + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["abc"]); + t.insert(h["abcde"], 2); + t.insert(h["abcfg"], 2); + // 'de' and 'fg' are tied without 'abc' vote + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abc"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["abc"].id()); + + t.remove(h["abc"]); + t.insert(h["abcd"]); + + // 'de' branch has 3 votes to 2, so earlier sequences see it as preferred + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abcde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["abcde"].id()); + + // However, if you validated a ledger with Seq 5, potentially on + // a different branch, you do not yet know if they chose abcd + // or abcf because of you, so abc remains preferred + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["abc"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } + + // Changing largestSeq perspective changes preferred branch + { + /** + * Build the tree below with initial tip support annotated + * A + * / \ + * B(1) C(1) + * / | | + * H D F(1) + * | + * E(2) + * | + * G + */ + LedgerTrie t; + LedgerHistoryHelper h; + t.insert(h["ab"]); + t.insert(h["ac"]); + t.insert(h["acf"]); + t.insert(h["abde"], 2); + + // B has more branch support + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{1})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{2})->id == h["ab"].id()); + + // But if you last validated D,F or E, you do not yet know + // if someone used that validation to commit to B or C + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["a"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["a"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + + /** + * One of E advancing to G doesn't change anything + * A + * / \ + * B(1) C(1) + * / | | + * H D F(1) + * | + * E(1) + * | + * G(1) + */ + t.remove(h["abde"]); + t.insert(h["abdeg"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{1})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{2})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["a"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["a"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["a"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + + /** + * C advancing to H does advance the seq 3 preferred ledger + * A + * / \ + * B(1) C + * / | | + * H(1)D F(1) + * | + * E(1) + * | + * G(1) + */ + t.remove(h["ac"]); + t.insert(h["abh"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{1})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{2})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["a"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["a"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + + /** + * F advancing to E also moves the preferred ledger forward + * A + * / \ + * B(1) C + * / | | + * H(1)D F + * | + * E(2) + * | + * G(1) + */ + t.remove(h["acf"]); + t.insert(h["abde"]); + + // NOLINTBEGIN(bugprone-unchecked-optional-access) + EXPECT_TRUE(t.getPreferred(Seq{1})->id == h["abde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{2})->id == h["abde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{3})->id == h["abde"].id()); + EXPECT_TRUE(t.getPreferred(Seq{4})->id == h["ab"].id()); + EXPECT_TRUE(t.getPreferred(Seq{5})->id == h["ab"].id()); + // NOLINTEND(bugprone-unchecked-optional-access) + } +} + +TEST(LedgerTrieTest, root_related) +{ + using namespace csf; + // Since the root is a special node that breaks the no-single child + // invariant, do some tests that exercise it. + + LedgerTrie t; + LedgerHistoryHelper h; + EXPECT_TRUE(!t.remove(h[""])); + EXPECT_TRUE(t.branchSupport(h[""]) == 0); + EXPECT_TRUE(t.tipSupport(h[""]) == 0); + + t.insert(h["a"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.branchSupport(h[""]) == 1); + EXPECT_TRUE(t.tipSupport(h[""]) == 0); + + t.insert(h["e"]); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.branchSupport(h[""]) == 2); + EXPECT_TRUE(t.tipSupport(h[""]) == 0); + + EXPECT_TRUE(t.remove(h["e"])); + EXPECT_TRUE(t.checkInvariants()); + EXPECT_TRUE(t.branchSupport(h[""]) == 1); + EXPECT_TRUE(t.tipSupport(h[""]) == 0); +} + +TEST(LedgerTrieTest, stress) +{ + using namespace csf; + LedgerTrie t; + LedgerHistoryHelper h; + + // Test quasi-randomly add/remove supporting for different ledgers + // from a branching history. + + // Ledgers have sequence 1,2,3,4 + std::uint32_t const depthConst = 4; + // Each ledger has 4 possible children + std::uint32_t const width = 4; + + std::uint32_t const iterations = 10000; + + // Use explicit seed to have same results for CI + // NOLINTNEXTLINE(bugprone-random-generator-seed): fixed seed for reproducible test + std::mt19937 gen{42}; + std::uniform_int_distribution<> depthDist(0, depthConst - 1); + std::uniform_int_distribution<> widthDist(0, width - 1); + std::uniform_int_distribution<> flip(0, 1); + for (std::uint32_t i = 0; i < iterations; ++i) + { + // pick a random ledger history + std::string curr; + char const depth = depthDist(gen); + char offset = 0; + for (char d = 0; d < depth; ++d) + { + char const a = offset + widthDist(gen); + curr += a; + offset = (a + 1) * width; + } + + // 50-50 to add remove + if (flip(gen) == 0) + { + t.insert(h[curr]); + } + else + { + t.remove(h[curr]); + } + EXPECT_TRUE(t.checkInvariants()); + if (!(t.checkInvariants())) + return; + } +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/ScaleFreeSim.cpp b/src/tests/libxrpl/consensus/ScaleFreeSim.cpp new file mode 100644 index 0000000000..2b07b29900 --- /dev/null +++ b/src/tests/libxrpl/consensus/ScaleFreeSim.cpp @@ -0,0 +1,100 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace xrpl::test { + +TEST(ScaleFreeSimTest, DISABLED_scale_free_sim) +{ + using namespace std::chrono; + using namespace csf; + + std::ostream& log = std::cout; + + // Generate a quasi-random scale free network and simulate consensus + // as we vary transaction submission rates + + int const n = 100; // Peers + + int const numUNLs = 15; // UNL lists + int const minUNLSize = n / 4, maxUNLSize = n / 2; + + ConsensusParms const parms{}; + Sim sim; + PeerGroup network = sim.createGroup(n); + + // generate trust ranks + std::vector const ranks = sample(network.size(), PowerLawDistribution{1, 3}, sim.rng); + + // generate scale-free trust graph + randomRankedTrust( + network, ranks, numUNLs, std::uniform_int_distribution<>{minUNLSize, maxUNLSize}, sim.rng); + + // nodes with a trust line in either direction are network-connected + network.connectFromTrust(round(0.2 * parms.ledgerGRANULARITY)); + + // Initialize collectors to track statistics to report + TxCollector txCollector; + LedgerCollector ledgerCollector; + auto colls = makeCollectors(txCollector, ledgerCollector); + sim.collectors.add(colls); + + // Initial round to set prior state + sim.run(1); + + // Initialize timers + HeartbeatTimer heart(sim.scheduler, seconds(10s)); + + // Run for 10 minutes, submitting 100 tx/second + std::chrono::nanoseconds const simDuration = 10min; + std::chrono::nanoseconds const quiet = 10s; + Rate const rate{.count = 100, .duration = 1000ms}; + + // txs, start/stop/step, target + auto peerSelector = makeSelector(network.begin(), network.end(), ranks, sim.rng); + auto txSubmitter = makeSubmitter( + ConstantDistribution{rate.inv()}, + sim.scheduler.now() + quiet, + sim.scheduler.now() + (simDuration - quiet), + peerSelector, + sim.scheduler, + sim.rng); + + // run simulation for given duration + heart.start(); + sim.run(simDuration); + + EXPECT_TRUE(sim.branches() == 1); + EXPECT_TRUE(sim.synchronized()); + + // TODO: Clean up this formatting mess!! + + log << "Peers: " << network.size() << std::endl; + log << "Simulated Duration: " << duration_cast(simDuration).count() << " ms" + << std::endl; + log << "Branches: " << sim.branches() << std::endl; + log << "Synchronized: " << (sim.synchronized() ? "Y" : "N") << std::endl; + log << std::endl; + + txCollector.report(simDuration, log); + ledgerCollector.report(simDuration, log); + // Print summary? + // # forks? # of LCLs? + // # peers + // # tx submitted + // # ledgers/sec etc.? +} + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/consensus/Validations.cpp b/src/tests/libxrpl/consensus/Validations.cpp new file mode 100644 index 0000000000..56964f56a6 --- /dev/null +++ b/src/tests/libxrpl/consensus/Validations.cpp @@ -0,0 +1,1030 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::test::csf { + +namespace { + +beast::Journal +journal() +{ + return beast::Journal{TestSink::instance()}; +} + +template +void +expireValidations(ValidationStore& validations) +{ + auto j = journal(); + validations.expire(j); +} + +using clock_type = beast::AbstractClock const; + +// Helper to convert steady_clock to a reasonable NetClock +// This allows a single manual clock in the unit tests +NetClock::time_point +toNetClock(clock_type const& c) +{ + // We don't care about the actual epochs, but do want the + // generated NetClock time to be well past its epoch to ensure + // any subtractions are positive + using namespace std::chrono; + return NetClock::time_point( + duration_cast(c.now().time_since_epoch() + 86400s)); +} + +// Represents a node that can issue validations +class Node +{ + clock_type const& c_; + PeerID nodeID_; + bool trusted_ = true; + std::size_t signIdx_{1}; + std::optional loadFee_; + +public: + Node(PeerID nodeID, clock_type const& c) : c_(c), nodeID_(nodeID) + { + } + + void + untrust() + { + trusted_ = false; + } + + void + trust() + { + trusted_ = true; + } + + void + setLoadFee(std::uint32_t fee) + { + loadFee_ = fee; + } + + [[nodiscard]] PeerID + nodeID() const + { + return nodeID_; + } + + void + advanceKey() + { + signIdx_++; + } + + [[nodiscard]] PeerKey + currKey() const + { + return std::make_pair(nodeID_, signIdx_); + } + + [[nodiscard]] PeerKey + masterKey() const + { + return std::make_pair(nodeID_, 0); + } + [[nodiscard]] NetClock::time_point + now() const + { + return toNetClock(c_); + } + + // Issue a new validation with given sequence number and id and + // with signing and seen times offset from the common clock + [[nodiscard]] Validation + validate( + Ledger::ID id, + Ledger::Seq seq, + NetClock::duration signOffset, + NetClock::duration seenOffset, + bool full) const + { + Validation v{ + id, seq, now() + signOffset, now() + seenOffset, currKey(), nodeID_, full, loadFee_}; + if (trusted_) + v.setTrusted(); + return v; + } + + [[nodiscard]] Validation + validate(Ledger ledger, NetClock::duration signOffset, NetClock::duration seenOffset) const + { + return validate(ledger.id(), ledger.seq(), signOffset, seenOffset, true); + } + + [[nodiscard]] Validation + validate(Ledger ledger) const + { + return validate( + ledger.id(), ledger.seq(), NetClock::duration{0}, NetClock::duration{0}, true); + } + + [[nodiscard]] Validation + partial(Ledger ledger) const + { + return validate( + ledger.id(), ledger.seq(), NetClock::duration{0}, NetClock::duration{0}, false); + } +}; + +// Generic Validations adaptor +class Adaptor +{ + clock_type& c_; + LedgerOracle& oracle_; + +public: + // Non-locking mutex to avoid locks in generic Validations + struct Mutex + { + void + lock() + { + } + + void + unlock() + { + } + }; + + using Validation = csf::Validation; + using Ledger = csf::Ledger; + + Adaptor(clock_type& c, LedgerOracle& o) : c_{c}, oracle_{o} + { + } + + [[nodiscard]] NetClock::time_point + now() const + { + return toNetClock(c_); + } + + std::optional + acquire(Ledger::ID const& id) + { + return oracle_.lookup(id); + } +}; + +// Specialize generic Validations using the above types +using TestValidations = Validations; + +// Gather the dependencies of TestValidations in a single class and provide +// accessors for simplifying test logic +class TestHarness +{ + ValidationParms p_; + beast::ManualClock clock_; + TestValidations tv_; + PeerID nextNodeId_{0}; + +public: + explicit TestHarness(LedgerOracle& o) : tv_(p_, clock_, clock_, o) + { + } + + ValStatus + add(Validation const& v) + { + return tv_.add(v.nodeID(), v); + } + + TestValidations& + vals() + { + return tv_; + } + + Node + makeNode() + { + return Node(nextNodeId_++, clock_); + } + + ValidationParms + parms() const + { + return p_; + } + + auto& + clock() + { + return clock_; + } +}; + +Ledger const kGenesisLedger{Ledger::MakeGenesis{}}; + +} // namespace + +TEST(ValidationsTest, add_validation) +{ + using namespace std::chrono_literals; + + SCOPED_TRACE("Add validation"); + LedgerHistoryHelper h; + Ledger const ledgerA = h["a"]; + Ledger ledgerAB = h["ab"]; + Ledger ledgerAZ = h["az"]; + Ledger ledgerABC = h["abc"]; + Ledger const ledgerABCD = h["abcd"]; + Ledger const ledgerABCDE = h["abcde"]; + + { + TestHarness harness(h.oracle); + Node n = harness.makeNode(); + + auto const v = n.validate(ledgerA); + + // Add a current validation + EXPECT_TRUE(ValStatus::Current == harness.add(v)); + + // Re-adding violates the increasing seq requirement for full + // validations + EXPECT_TRUE(ValStatus::BadSeq == harness.add(v)); + + harness.clock().advance(1s); + + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerAB))); + + // Test the node changing signing key + + // Confirm old ledger on hand, but not new ledger + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerAB.id()) == 1); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerABC.id()) == 0); + + // Rotate signing keys + n.advanceKey(); + + harness.clock().advance(1s); + + // Cannot re-do the same full validation sequence + EXPECT_TRUE(ValStatus::Conflicting == harness.add(n.validate(ledgerAB))); + // Cannot send the same partial validation sequence + EXPECT_TRUE(ValStatus::Conflicting == harness.add(n.partial(ledgerAB))); + + // Now trusts the newest ledger too + harness.clock().advance(1s); + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerABC))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerAB.id()) == 1); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerABC.id()) == 1); + + // Processing validations out of order should ignore the older + // validation + harness.clock().advance(2s); + auto const valABCDE = n.validate(ledgerABCDE); + + harness.clock().advance(4s); + auto const valABCD = n.validate(ledgerABCD); + + EXPECT_TRUE(ValStatus::Current == harness.add(valABCD)); + + EXPECT_TRUE(ValStatus::Stale == harness.add(valABCDE)); + } + + { + // Process validations out of order with shifted times + + TestHarness harness(h.oracle); + Node const n = harness.makeNode(); + + // Establish a new current validation + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerA))); + + // Process a validation that has "later" seq but early sign time + EXPECT_TRUE(ValStatus::Stale == harness.add(n.validate(ledgerAB, -1s, -1s))); + + // Process a validation that has a later seq and later sign + // time + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerABC, 1s, 1s))); + } + + { + // Test stale on arrival validations + TestHarness harness(h.oracle); + Node const n = harness.makeNode(); + + EXPECT_TRUE( + ValStatus::Stale == + harness.add(n.validate(ledgerA, -harness.parms().validationCurrentEarly, 0s))); + + EXPECT_TRUE( + ValStatus::Stale == + harness.add(n.validate(ledgerA, harness.parms().validationCurrentWall, 0s))); + + EXPECT_TRUE( + ValStatus::Stale == + harness.add(n.validate(ledgerA, 0s, harness.parms().validationCurrentLocal))); + } + + { + // Test that full or partials cannot be sent for older sequence + // numbers, unless time-out has happened + for (bool doFull : {true, false}) + { + TestHarness harness(h.oracle); + Node n = harness.makeNode(); + + auto process = [&](Ledger& lgr) { + if (doFull) + return harness.add(n.validate(lgr)); + return harness.add(n.partial(lgr)); + }; + + EXPECT_TRUE(ValStatus::Current == process(ledgerABC)); + harness.clock().advance(1s); + EXPECT_TRUE(ledgerAB.seq() < ledgerABC.seq()); + EXPECT_TRUE(ValStatus::BadSeq == process(ledgerAB)); + + // If we advance far enough for AB to expire, we can fully + // validate or partially validate that sequence number again + EXPECT_TRUE(ValStatus::Conflicting == process(ledgerAZ)); + harness.clock().advance(harness.parms().validationSetExpires + 1ms); + EXPECT_TRUE(ValStatus::Current == process(ledgerAZ)); + } + } +} + +TEST(ValidationsTest, on_stale) +{ + SCOPED_TRACE("Stale validation"); + // Verify validation becomes stale based solely on time passing, but + // use different functions to trigger the check for staleness + + LedgerHistoryHelper h; + Ledger ledgerA = h["a"]; + Ledger const ledgerAB = h["ab"]; + + using Trigger = std::function; + + std::vector const triggers = { + [&](TestValidations& vals) { vals.currentTrusted(); }, + [&](TestValidations& vals) { vals.getCurrentNodeIDs(); }, + [&](TestValidations& vals) { vals.getPreferred(kGenesisLedger); }, + [&](TestValidations& vals) { vals.getNodesAfter(ledgerA, ledgerA.id()); }}; + for (Trigger const& trigger : triggers) + { + TestHarness harness(h.oracle); + Node const n = harness.makeNode(); + + EXPECT_TRUE(ValStatus::Current == harness.add(n.validate(ledgerAB))); + trigger(harness.vals()); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 1); + EXPECT_TRUE( + harness.vals().getPreferred(kGenesisLedger) == + std::make_pair(ledgerAB.seq(), ledgerAB.id())); + harness.clock().advance(harness.parms().validationCurrentLocal); + + // trigger check for stale + trigger(harness.vals()); + + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 0); + EXPECT_TRUE(harness.vals().getPreferred(kGenesisLedger) == std::nullopt); + } +} + +TEST(ValidationsTest, get_nodes_after) +{ + // Test getting number of nodes working on a validation descending + // a prescribed one. This count should only be for trusted nodes, but + // includes partial and full validations + + using namespace std::chrono_literals; + SCOPED_TRACE("Get nodes after"); + + LedgerHistoryHelper h; + Ledger const ledgerA = h["a"]; + Ledger const ledgerAB = h["ab"]; + Ledger const ledgerABC = h["abc"]; + Ledger const ledgerAD = h["ad"]; + + TestHarness harness(h.oracle); + Node const trustedNode1 = harness.makeNode(); + Node const trustedNode2 = harness.makeNode(); + Node const trustedNode3 = harness.makeNode(); + + Node notTrustedNode = harness.makeNode(); + notTrustedNode.untrust(); + + // first round a,b,c agree, d has is partial + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode1.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode2.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode3.partial(ledgerA))); + + for (Ledger const& ledger : {ledgerA, ledgerAB, ledgerABC, ledgerAD}) + EXPECT_TRUE(harness.vals().getNodesAfter(ledger, ledger.id()) == 0); + + harness.clock().advance(5s); + + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode1.validate(ledgerAB))); + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode2.validate(ledgerABC))); + EXPECT_TRUE(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerAB))); + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode3.partial(ledgerABC))); + + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerA, ledgerA.id()) == 3); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerAB, ledgerAB.id()) == 2); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerABC, ledgerABC.id()) == 0); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerAD, ledgerAD.id()) == 0); + + // If given a ledger inconsistent with the id, is still able to check using slower method + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerAD, ledgerA.id()) == 1); + EXPECT_TRUE(harness.vals().getNodesAfter(ledgerAD, ledgerAB.id()) == 2); +} + +TEST(ValidationsTest, current_trusted) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Current trusted validations"); + + LedgerHistoryHelper h; + Ledger const ledgerA = h["a"]; + Ledger const ledgerB = h["b"]; + Ledger const ledgerAC = h["ac"]; + + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Node b = harness.makeNode(); + b.untrust(); + + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(b.validate(ledgerB))); + + // Only a is trusted + EXPECT_TRUE(harness.vals().currentTrusted().size() == 1); + EXPECT_TRUE(harness.vals().currentTrusted()[0].ledgerID() == ledgerA.id()); + EXPECT_TRUE(harness.vals().currentTrusted()[0].seq() == ledgerA.seq()); + + harness.clock().advance(3s); + + for (auto const& node : {a, b}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.validate(ledgerAC))); + + // New validation for a + EXPECT_TRUE(harness.vals().currentTrusted().size() == 1); + EXPECT_TRUE(harness.vals().currentTrusted()[0].ledgerID() == ledgerAC.id()); + EXPECT_TRUE(harness.vals().currentTrusted()[0].seq() == ledgerAC.seq()); + + // Pass enough time for it to go stale + harness.clock().advance(harness.parms().validationCurrentLocal); + EXPECT_TRUE(harness.vals().currentTrusted().empty()); +} + +TEST(ValidationsTest, get_current_public_keys) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Current public keys"); + + LedgerHistoryHelper h; + Ledger const ledgerA = h["a"]; + Ledger const ledgerAC = h["ac"]; + + TestHarness harness(h.oracle); + Node a = harness.makeNode(), b = harness.makeNode(); + b.untrust(); + + for (auto const& node : {a, b}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.validate(ledgerA))); + + { + hash_set const expectedKeys = {a.nodeID(), b.nodeID()}; + EXPECT_TRUE(harness.vals().getCurrentNodeIDs() == expectedKeys); + } + + harness.clock().advance(3s); + + // Change keys and issue partials + a.advanceKey(); + b.advanceKey(); + + for (auto const& node : {a, b}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.partial(ledgerAC))); + + { + hash_set const expectedKeys = {a.nodeID(), b.nodeID()}; + EXPECT_TRUE(harness.vals().getCurrentNodeIDs() == expectedKeys); + } + + // Pass enough time for them to go stale + harness.clock().advance(harness.parms().validationCurrentLocal); + EXPECT_TRUE(harness.vals().getCurrentNodeIDs().empty()); +} + +TEST(ValidationsTest, trusted_by_ledger_functions) +{ + // Test the Validations functions that calculate a value by ledger ID + using namespace std::chrono_literals; + SCOPED_TRACE("By ledger functions"); + + // Several Validations functions return a set of values associated + // with trusted ledgers sharing the same ledger ID. The tests below + // exercise this logic by saving the set of trusted Validations, and + // verifying that the Validations member functions all calculate the + // proper transformation of the available ledgers. + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + + Node a = harness.makeNode(), b = harness.makeNode(), c = harness.makeNode(), + d = harness.makeNode(), e = harness.makeNode(); + + c.untrust(); + // Mix of load fees + a.setLoadFee(12); + b.setLoadFee(1); + c.setLoadFee(12); + e.setLoadFee(12); + + hash_map, std::vector> trustedValidations; + + //---------------------------------------------------------------------- + // checkers + auto sorted = [](auto vec) { + std::sort(vec.begin(), vec.end()); + return vec; + }; + auto compare = [&]() { + for (auto& it : trustedValidations) + { + auto const& id = it.first.first; + auto const& seq = it.first.second; + auto const& expectedValidations = it.second; + + EXPECT_TRUE(harness.vals().numTrustedForLedger(id) == expectedValidations.size()); + EXPECT_TRUE( + sorted(harness.vals().getTrustedForLedger(id, seq)) == sorted(expectedValidations)); + + std::uint32_t const baseFee = 0; + std::vector expectedFees; + expectedFees.reserve(expectedValidations.size()); + for (auto const& val : expectedValidations) + { + expectedFees.push_back(val.loadFee().value_or(baseFee)); + } + + EXPECT_TRUE(sorted(harness.vals().fees(id, baseFee)) == sorted(expectedFees)); + } + }; + + //---------------------------------------------------------------------- + Ledger const ledgerA = h["a"]; + Ledger const ledgerB = h["b"]; + Ledger const ledgerAC = h["ac"]; + + // Add a dummy ID to cover unknown ledger identifiers + trustedValidations[{Ledger::ID{100}, Ledger::Seq{100}}] = {}; + + // first round a,b,c agree + for (auto const& node : {a, b, c}) + { + auto const val = node.validate(ledgerA); + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + if (val.trusted()) + trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); + } + // d disagrees + { + auto const val = d.validate(ledgerB); + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); + } + // e only issues partials + { + EXPECT_TRUE(ValStatus::Current == harness.add(e.partial(ledgerA))); + } + + harness.clock().advance(5s); + // second round, a,b,c move to ledger 2 + for (auto const& node : {a, b, c}) + { + auto const val = node.validate(ledgerAC); + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + if (val.trusted()) + trustedValidations[{val.ledgerID(), val.seq()}].emplace_back(val); + } + // d now thinks ledger 1, but cannot re-issue a previously used seq + // and attempting it should generate a conflict. + { + EXPECT_TRUE(ValStatus::Conflicting == harness.add(d.partial(ledgerA))); + } + // e only issues partials + { + EXPECT_TRUE(ValStatus::Current == harness.add(e.partial(ledgerAC))); + } + + compare(); +} + +TEST(ValidationsTest, expire) +{ + // Verify expiring clears out validations stored by ledger + SCOPED_TRACE("Expire validations"); + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + constexpr Ledger::Seq kOne(1); + constexpr Ledger::Seq kTwo(2); + + // simple cases + Ledger const ledgerA = h["a"]; + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerA))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); + expireValidations(harness.vals()); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); + harness.clock().advance(harness.parms().validationSetExpires); + expireValidations(harness.vals()); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 0); + + // use setSeqToKeep to keep the validation from expire + Ledger const ledgerB = h["ab"]; + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerB))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); + harness.vals().setSeqToKeep(ledgerB.seq(), ledgerB.seq() + kOne); + harness.clock().advance(harness.parms().validationSetExpires); + expireValidations(harness.vals()); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); + // change toKeep + harness.vals().setSeqToKeep(ledgerB.seq() + kOne, ledgerB.seq() + kTwo); + // advance clock slowly + int const loops = + harness.parms().validationSetExpires / harness.parms().validationFRESHNESS + 1; + for (int i = 0; i < loops; ++i) + { + harness.clock().advance(harness.parms().validationFRESHNESS); + expireValidations(harness.vals()); + } + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerB.id()) == 0); + + // Allow the validation with high seq to expire + Ledger const ledgerC = h["abc"]; + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerC))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerC.id()) == 1); + harness.vals().setSeqToKeep(ledgerC.seq() - kOne, ledgerC.seq()); + harness.clock().advance(harness.parms().validationSetExpires); + expireValidations(harness.vals()); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerC.id()) == 0); +} + +TEST(ValidationsTest, flush) +{ + // Test final flush of validations + using namespace std::chrono_literals; + SCOPED_TRACE("Flush validations"); + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const trustedNode1 = harness.makeNode(); + Node const trustedNode2 = harness.makeNode(); + Node notTrustedNode = harness.makeNode(); + notTrustedNode.untrust(); + + Ledger const ledgerA = h["a"]; + Ledger const ledgerAB = h["ab"]; + + hash_map expected; + for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode}) + { + auto const val = node.validate(ledgerA); + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + expected.emplace(node.nodeID(), val); + } + + // Send in a new validation for a, saving the new one into the expected + // map after setting the proper prior ledger ID it replaced + harness.clock().advance(1s); + auto newVal = trustedNode1.validate(ledgerAB); + EXPECT_TRUE(ValStatus::Current == harness.add(newVal)); + expected.find(trustedNode1.nodeID())->second = newVal; +} + +TEST(ValidationsTest, get_preferred_ledger) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Preferred Ledger"); + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const trustedNode1 = harness.makeNode(); + Node const trustedNode2 = harness.makeNode(); + Node const trustedNode3 = harness.makeNode(); + + Node notTrustedNode = harness.makeNode(); + notTrustedNode.untrust(); + + Ledger const ledgerA = h["a"]; + Ledger const ledgerB = h["b"]; + Ledger const ledgerAC = h["ac"]; + Ledger const ledgerACD = h["acd"]; + + using Seq = Ledger::Seq; + + auto pref = [](Ledger ledger) { return std::make_pair(ledger.seq(), ledger.id()); }; + + // Empty (no ledgers) + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == std::nullopt); + + // Single ledger + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode1.validate(ledgerB))); + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == pref(ledgerB)); + EXPECT_TRUE(harness.vals().getPreferred(ledgerB) == pref(ledgerB)); + + // Minimum valid sequence + EXPECT_TRUE(harness.vals().getPreferred(ledgerA, Seq{10}) == ledgerA.id()); + + // Untrusted doesn't impact preferred ledger + // (ledgerB has tie-break over ledgerA) + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode2.validate(ledgerA))); + EXPECT_TRUE(ValStatus::Current == harness.add(notTrustedNode.validate(ledgerA))); + EXPECT_TRUE(ledgerB.id() > ledgerA.id()); + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == pref(ledgerB)); + EXPECT_TRUE(harness.vals().getPreferred(ledgerB) == pref(ledgerB)); + + // Partial does break ties + EXPECT_TRUE(ValStatus::Current == harness.add(trustedNode3.partial(ledgerA))); + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == pref(ledgerA)); + EXPECT_TRUE(harness.vals().getPreferred(ledgerB) == pref(ledgerA)); + + harness.clock().advance(5s); + + // Parent of preferred-> stick with ledger + for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode, trustedNode3}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.validate(ledgerAC))); + // Parent of preferred stays put + EXPECT_TRUE(harness.vals().getPreferred(ledgerA) == pref(ledgerA)); + // Earlier different chain, switch + EXPECT_TRUE(harness.vals().getPreferred(ledgerB) == pref(ledgerAC)); + // Later on chain, stays where it is + EXPECT_TRUE(harness.vals().getPreferred(ledgerACD) == pref(ledgerACD)); + + // Any later grandchild or different chain is preferred + harness.clock().advance(5s); + for (auto const& node : {trustedNode1, trustedNode2, notTrustedNode, trustedNode3}) + EXPECT_TRUE(ValStatus::Current == harness.add(node.validate(ledgerACD))); + for (auto const& ledger : {ledgerA, ledgerB, ledgerACD}) + EXPECT_TRUE(harness.vals().getPreferred(ledger) == pref(ledgerACD)); +} + +TEST(ValidationsTest, get_preferred_lcl) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Get preferred LCL"); + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + + Ledger const ledgerA = h["a"]; + Ledger const ledgerB = h["b"]; + Ledger const ledgerC = h["c"]; + + using ID = Ledger::ID; + using Seq = Ledger::Seq; + + hash_map peerCounts; + + // No trusted validations or counts sticks with current ledger + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerA.id()); + + ++peerCounts[ledgerB.id()]; + + // No trusted validations, rely on peer counts + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerB.id()); + + ++peerCounts[ledgerC.id()]; + // No trusted validations, tied peers goes with larger ID + EXPECT_TRUE(ledgerC.id() > ledgerB.id()); + + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerC.id()); + + peerCounts[ledgerC.id()] += 1000; + + // Single trusted always wins over peer counts + EXPECT_TRUE(ValStatus::Current == harness.add(a.validate(ledgerA))); + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerA, Seq{0}, peerCounts) == ledgerA.id()); + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerB, Seq{0}, peerCounts) == ledgerA.id()); + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerC, Seq{0}, peerCounts) == ledgerA.id()); + + // Stick with current ledger if trusted validation ledger has too old + // of a sequence + EXPECT_TRUE(harness.vals().getPreferredLCL(ledgerB, Seq{2}, peerCounts) == ledgerB.id()); +} + +TEST(ValidationsTest, acquire_validated_ledger) +{ + using namespace std::chrono_literals; + SCOPED_TRACE("Acquire validated ledger"); + + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Node const b = harness.makeNode(); + + using ID = Ledger::ID; + using Seq = Ledger::Seq; + + // Validate the ledger before it is actually available + Validation const val = a.validate(ID{2}, Seq{2}, 0s, 0s, true); + + EXPECT_TRUE(ValStatus::Current == harness.add(val)); + // Validation is available + EXPECT_TRUE(harness.vals().numTrustedForLedger(ID{2}) == 1); + // but ledger based data is not + EXPECT_TRUE(harness.vals().getNodesAfter(kGenesisLedger, ID{0}) == 0); + // Initial preferred branch falls back to the ledger we are trying to + // acquire + EXPECT_TRUE(harness.vals().getPreferred(kGenesisLedger) == std::make_pair(Seq{2}, ID{2})); + + // After adding another unavailable validation, the preferred ledger + // breaks ties via higher ID + EXPECT_TRUE(ValStatus::Current == harness.add(b.validate(ID{3}, Seq{2}, 0s, 0s, true))); + EXPECT_TRUE(harness.vals().getPreferred(kGenesisLedger) == std::make_pair(Seq{2}, ID{3})); + + // Create the ledger + Ledger const ledgerAB = h["ab"]; + // Now it should be available + EXPECT_TRUE(harness.vals().getNodesAfter(kGenesisLedger, ID{0}) == 1); + + // Create a validation that is not available + harness.clock().advance(5s); + Validation const val2 = a.validate(ID{4}, Seq{4}, 0s, 0s, true); + EXPECT_TRUE(ValStatus::Current == harness.add(val2)); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ID{4}) == 1); + EXPECT_TRUE( + harness.vals().getPreferred(kGenesisLedger) == + std::make_pair(ledgerAB.seq(), ledgerAB.id())); + + // Another node requesting that ledger still doesn't change things + Validation const val3 = b.validate(ID{4}, Seq{4}, 0s, 0s, true); + EXPECT_TRUE(ValStatus::Current == harness.add(val3)); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ID{4}) == 2); + EXPECT_TRUE( + harness.vals().getPreferred(kGenesisLedger) == + std::make_pair(ledgerAB.seq(), ledgerAB.id())); + + // Switch to validation that is available + harness.clock().advance(5s); + Ledger const ledgerABCDE = h["abcde"]; + EXPECT_TRUE(ValStatus::Current == harness.add(a.partial(ledgerABCDE))); + EXPECT_TRUE(ValStatus::Current == harness.add(b.partial(ledgerABCDE))); + EXPECT_TRUE( + harness.vals().getPreferred(kGenesisLedger) == + std::make_pair(ledgerABCDE.seq(), ledgerABCDE.id())); +} + +TEST(ValidationsTest, num_trusted_for_ledger) +{ + SCOPED_TRACE("NumTrustedForLedger"); + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Node const b = harness.makeNode(); + Ledger const ledgerA = h["a"]; + + EXPECT_TRUE(ValStatus::Current == harness.add(a.partial(ledgerA))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 0); + + EXPECT_TRUE(ValStatus::Current == harness.add(b.validate(ledgerA))); + EXPECT_TRUE(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); +} + +TEST(ValidationsTest, seq_enforcer) +{ + SCOPED_TRACE("SeqEnforcer"); + using Seq = Ledger::Seq; + using namespace std::chrono; + + beast::ManualClock clock; + SeqEnforcer enforcer; + + ValidationParms const p; + + EXPECT_TRUE(enforcer(clock.now(), Seq{1}, p)); + EXPECT_TRUE(enforcer(clock.now(), Seq{10}, p)); + EXPECT_TRUE(!enforcer(clock.now(), Seq{5}, p)); + EXPECT_TRUE(!enforcer(clock.now(), Seq{9}, p)); + clock.advance(p.validationSetExpires - 1ms); + EXPECT_TRUE(!enforcer(clock.now(), Seq{1}, p)); + clock.advance(2ms); + EXPECT_TRUE(enforcer(clock.now(), Seq{1}, p)); +} + +TEST(ValidationsTest, trust_changed) +{ + SCOPED_TRACE("TrustChanged"); + using namespace std::chrono; + + auto checker = [&](TestValidations& vals, + hash_set const& listed, + std::vector const& trustedVals) { + Ledger::ID const testID = + trustedVals.empty() ? kGenesisLedger.id() : trustedVals[0].ledgerID(); + Ledger::Seq const testSeq = + trustedVals.empty() ? kGenesisLedger.seq() : trustedVals[0].seq(); + EXPECT_TRUE(vals.currentTrusted() == trustedVals); + EXPECT_TRUE(vals.getCurrentNodeIDs() == listed); + EXPECT_TRUE(vals.getNodesAfter(kGenesisLedger, kGenesisLedger.id()) == trustedVals.size()); + if (trustedVals.empty()) + { + EXPECT_TRUE(vals.getPreferred(kGenesisLedger) == std::nullopt); + } + else + { + EXPECT_TRUE(vals.getPreferred(kGenesisLedger)->second == testID); + } + EXPECT_TRUE(vals.getTrustedForLedger(testID, testSeq) == trustedVals); + EXPECT_TRUE(vals.numTrustedForLedger(testID) == trustedVals.size()); + }; + + { + // Trusted to untrusted + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Ledger const ledgerAB = h["ab"]; + Validation const v = a.validate(ledgerAB); + EXPECT_TRUE(ValStatus::Current == harness.add(v)); + + hash_set const listed({a.nodeID()}); + std::vector trustedVals({v}); + checker(harness.vals(), listed, trustedVals); + + trustedVals.clear(); + harness.vals().trustChanged({}, {a.nodeID()}); + checker(harness.vals(), listed, trustedVals); + } + + { + // Untrusted to trusted + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node a = harness.makeNode(); + a.untrust(); + Ledger const ledgerAB = h["ab"]; + Validation const v = a.validate(ledgerAB); + EXPECT_TRUE(ValStatus::Current == harness.add(v)); + + hash_set const listed({a.nodeID()}); + std::vector trustedVals; + checker(harness.vals(), listed, trustedVals); + + trustedVals.push_back(v); + harness.vals().trustChanged({a.nodeID()}, {}); + checker(harness.vals(), listed, trustedVals); + } + + { + // Trusted but not acquired -> untrusted + LedgerHistoryHelper h; + TestHarness harness(h.oracle); + Node const a = harness.makeNode(); + Validation const v = a.validate(Ledger::ID{2}, Ledger::Seq{2}, 0s, 0s, true); + EXPECT_TRUE(ValStatus::Current == harness.add(v)); + + hash_set const listed({a.nodeID()}); + std::vector trustedVals({v}); + auto& vals = harness.vals(); + EXPECT_TRUE(vals.currentTrusted() == trustedVals); + + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) + EXPECT_TRUE(vals.getPreferred(kGenesisLedger)->second == v.ledgerID()); + EXPECT_TRUE(vals.getNodesAfter(kGenesisLedger, kGenesisLedger.id()) == 0); + + trustedVals.clear(); + harness.vals().trustChanged({}, {a.nodeID()}); + // make acquiring ledger available + h["ab"]; + EXPECT_TRUE(vals.currentTrusted() == trustedVals); + EXPECT_TRUE(vals.getPreferred(kGenesisLedger) == std::nullopt); + EXPECT_TRUE(vals.getNodesAfter(kGenesisLedger, kGenesisLedger.id()) == 0); + } +} + +} // namespace xrpl::test::csf diff --git a/src/tests/libxrpl/csf/BasicNetwork.cpp b/src/tests/libxrpl/csf/BasicNetwork.cpp new file mode 100644 index 0000000000..a6fc2d90e6 --- /dev/null +++ b/src/tests/libxrpl/csf/BasicNetwork.cpp @@ -0,0 +1,122 @@ +#include + +#include +#include + +#include +#include + +namespace xrpl::test { + +namespace { + +struct Peer +{ + int id; + std::set set; + + Peer(Peer const&) = default; + Peer(Peer&&) = default; + + explicit Peer(int id) : id(id) + { + } + + template + void + start(csf::Scheduler& scheduler, Net& net) + { + using namespace std::chrono_literals; + auto t = scheduler.in(1s, [&] { set.insert(0); }); + if (id == 0) + { + for (auto const link : net.links(this)) + { + net.send(this, link.target, [&, to = link.target] { to->receive(net, this, 1); }); + } + } + else + { + scheduler.cancel(t); + } + } + + template + void + receive(Net& net, Peer* from, int m) + { + set.insert(m); + ++m; + if (m < 5) + { + for (auto const link : net.links(this)) + { + net.send(this, link.target, [&, mm = m, to = link.target] { + to->receive(net, this, mm); + }); + } + } + } +}; + +} // namespace + +TEST(BasicNetworkTest, network) +{ + using namespace std::chrono_literals; + std::vector pv; + pv.emplace_back(0); + pv.emplace_back(1); + pv.emplace_back(2); + csf::Scheduler scheduler; + csf::BasicNetwork net(scheduler); + EXPECT_TRUE(!net.connect(&pv[0], &pv[0])); + EXPECT_TRUE(net.connect(&pv[0], &pv[1], 1s)); + EXPECT_TRUE(net.connect(&pv[1], &pv[2], 1s)); + EXPECT_TRUE(!net.connect(&pv[0], &pv[1])); + for (auto& peer : pv) + peer.start(scheduler, net); + EXPECT_TRUE(scheduler.stepFor(0s)); + EXPECT_TRUE(scheduler.stepFor(1s)); + EXPECT_TRUE(scheduler.step()); + EXPECT_TRUE(!scheduler.step()); + EXPECT_TRUE(!scheduler.stepFor(1s)); + net.send(&pv[0], &pv[1], [] {}); + net.send(&pv[1], &pv[0], [] {}); + EXPECT_TRUE(net.disconnect(&pv[0], &pv[1])); + EXPECT_TRUE(!net.disconnect(&pv[0], &pv[1])); + for (;;) + { + auto const links = net.links(&pv[1]); + if (links.empty()) + break; + EXPECT_TRUE(net.disconnect(&pv[1], links[0].target)); + } + EXPECT_TRUE(pv[0].set == std::set({0, 2, 4})); + EXPECT_TRUE(pv[1].set == std::set({1, 3})); + EXPECT_TRUE(pv[2].set == std::set({2, 4})); +} + +TEST(BasicNetworkTest, disconnect) +{ + using namespace std::chrono_literals; + csf::Scheduler scheduler; + csf::BasicNetwork net(scheduler); + EXPECT_TRUE(net.connect(0, 1, 1s)); + EXPECT_TRUE(net.connect(0, 2, 2s)); + + std::set delivered; + net.send(0, 1, [&]() { delivered.insert(1); }); + net.send(0, 2, [&]() { delivered.insert(2); }); + + scheduler.in(1000ms, [&]() { EXPECT_TRUE(net.disconnect(0, 2)); }); + scheduler.in(1100ms, [&]() { EXPECT_TRUE(net.connect(0, 2)); }); + + scheduler.step(); + + // only the first message is delivered because the disconnect at 1 s + // purges all pending messages from 0 to 2 + EXPECT_TRUE(delivered == std::set({1})); +} + +} // namespace xrpl::test diff --git a/src/test/csf/BasicNetwork.h b/src/tests/libxrpl/csf/BasicNetwork.h similarity index 99% rename from src/test/csf/BasicNetwork.h rename to src/tests/libxrpl/csf/BasicNetwork.h index 0428475504..2c68bd282d 100644 --- a/src/test/csf/BasicNetwork.h +++ b/src/tests/libxrpl/csf/BasicNetwork.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include diff --git a/src/test/csf/CollectorRef.h b/src/tests/libxrpl/csf/CollectorRef.h similarity index 97% rename from src/test/csf/CollectorRef.h rename to src/tests/libxrpl/csf/CollectorRef.h index 3aef4d617f..b1da962f3d 100644 --- a/src/test/csf/CollectorRef.h +++ b/src/tests/libxrpl/csf/CollectorRef.h @@ -1,11 +1,11 @@ #pragma once -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/tests/libxrpl/csf/Digraph.cpp b/src/tests/libxrpl/csf/Digraph.cpp new file mode 100644 index 0000000000..83c15ec06a --- /dev/null +++ b/src/tests/libxrpl/csf/Digraph.cpp @@ -0,0 +1,72 @@ +#include + +#include + +#include +#include +#include +#include + +namespace xrpl::test { + +TEST(DigraphTest, digraph) +{ + using namespace csf; + using Graph = Digraph; + Graph graph; + + EXPECT_TRUE(!graph.connected('a', 'b')); + EXPECT_TRUE(!graph.edge('a', 'b')); + EXPECT_TRUE(!graph.disconnect('a', 'b')); + + EXPECT_TRUE(graph.connect('a', 'b', "foobar")); + EXPECT_TRUE(graph.connected('a', 'b')); + EXPECT_TRUE(*graph.edge('a', 'b') == "foobar"); // NOLINT(bugprone-unchecked-optional-access) + + EXPECT_TRUE(!graph.connect('a', 'b', "repeat")); + EXPECT_TRUE(graph.disconnect('a', 'b')); + EXPECT_TRUE(graph.connect('a', 'b', "repeat")); + EXPECT_TRUE(graph.connected('a', 'b')); + EXPECT_TRUE(*graph.edge('a', 'b') == "repeat"); // NOLINT(bugprone-unchecked-optional-access) + + EXPECT_TRUE(graph.connect('a', 'c', "tree")); + + { + std::vector> edges; + + for (auto const& edge : graph.outEdges('a')) + { + edges.emplace_back(edge.source, edge.target, edge.data); + } + + std::vector> expected; + expected.emplace_back('a', 'b', "repeat"); + expected.emplace_back('a', 'c', "tree"); + EXPECT_TRUE(edges == expected); + EXPECT_TRUE(graph.outDegree('a') == expected.size()); + } + + EXPECT_TRUE(graph.outEdges('r').size() == 0); + EXPECT_TRUE(graph.outDegree('r') == 0); + EXPECT_TRUE(graph.outDegree('c') == 0); + + // only 'a' has out edges + EXPECT_TRUE(graph.outVertices().size() == 1); + std::vector const expected = {'b', 'c'}; + + EXPECT_TRUE((graph.outVertices('a') == expected)); + EXPECT_TRUE(graph.outVertices('b').size() == 0); + EXPECT_TRUE(graph.outVertices('c').size() == 0); + EXPECT_TRUE(graph.outVertices('r').size() == 0); + + std::stringstream ss; + graph.saveDot(ss, [](char v) { return v; }); + std::string const expectedDot = + "digraph {\n" + "a -> b;\n" + "a -> c;\n" + "}\n"; + EXPECT_TRUE(ss.str() == expectedDot); +} + +} // namespace xrpl::test diff --git a/src/test/csf/Digraph.h b/src/tests/libxrpl/csf/Digraph.h similarity index 100% rename from src/test/csf/Digraph.h rename to src/tests/libxrpl/csf/Digraph.h diff --git a/src/tests/libxrpl/csf/Histogram.cpp b/src/tests/libxrpl/csf/Histogram.cpp new file mode 100644 index 0000000000..6de1cde593 --- /dev/null +++ b/src/tests/libxrpl/csf/Histogram.cpp @@ -0,0 +1,59 @@ +#include + +#include + +namespace xrpl::test { + +TEST(HistogramTest, histogram) +{ + using namespace csf; + Histogram hist; + + EXPECT_TRUE(hist.size() == 0); + EXPECT_TRUE(hist.numBins() == 0); + EXPECT_TRUE(hist.minValue() == 0); + EXPECT_TRUE(hist.maxValue() == 0); + EXPECT_TRUE(hist.avg() == 0); + EXPECT_TRUE(hist.percentile(0.0f) == hist.minValue()); + EXPECT_TRUE(hist.percentile(0.5f) == 0); + EXPECT_TRUE(hist.percentile(0.9f) == 0); + EXPECT_TRUE(hist.percentile(1.0f) == hist.maxValue()); + + hist.insert(1); + + EXPECT_TRUE(hist.size() == 1); + EXPECT_TRUE(hist.numBins() == 1); + EXPECT_TRUE(hist.minValue() == 1); + EXPECT_TRUE(hist.maxValue() == 1); + EXPECT_TRUE(hist.avg() == 1); + EXPECT_TRUE(hist.percentile(0.0f) == hist.minValue()); + EXPECT_TRUE(hist.percentile(0.5f) == 1); + EXPECT_TRUE(hist.percentile(0.9f) == 1); + EXPECT_TRUE(hist.percentile(1.0f) == hist.maxValue()); + + hist.insert(9); + + EXPECT_TRUE(hist.size() == 2); + EXPECT_TRUE(hist.numBins() == 2); + EXPECT_TRUE(hist.minValue() == 1); + EXPECT_TRUE(hist.maxValue() == 9); + EXPECT_TRUE(hist.avg() == 5); + EXPECT_TRUE(hist.percentile(0.0f) == hist.minValue()); + EXPECT_TRUE(hist.percentile(0.5f) == 1); + EXPECT_TRUE(hist.percentile(0.9f) == 9); + EXPECT_TRUE(hist.percentile(1.0f) == hist.maxValue()); + + hist.insert(1); + + EXPECT_TRUE(hist.size() == 3); + EXPECT_TRUE(hist.numBins() == 2); + EXPECT_TRUE(hist.minValue() == 1); + EXPECT_TRUE(hist.maxValue() == 9); + EXPECT_TRUE(hist.avg() == 11 / 3); + EXPECT_TRUE(hist.percentile(0.0f) == hist.minValue()); + EXPECT_TRUE(hist.percentile(0.5f) == 1); + EXPECT_TRUE(hist.percentile(0.9f) == 9); + EXPECT_TRUE(hist.percentile(1.0f) == hist.maxValue()); +} + +} // namespace xrpl::test diff --git a/src/test/csf/Histogram.h b/src/tests/libxrpl/csf/Histogram.h similarity index 100% rename from src/test/csf/Histogram.h rename to src/tests/libxrpl/csf/Histogram.h diff --git a/src/test/csf/Peer.h b/src/tests/libxrpl/csf/Peer.h similarity index 98% rename from src/test/csf/Peer.h rename to src/tests/libxrpl/csf/Peer.h index 79bffec9cb..d4b6f42bdb 100644 --- a/src/test/csf/Peer.h +++ b/src/tests/libxrpl/csf/Peer.h @@ -1,33 +1,32 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - #include #include #include #include #include #include +#include +#include +#include +#include #include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include #include diff --git a/src/test/csf/PeerGroup.h b/src/tests/libxrpl/csf/PeerGroup.h similarity index 98% rename from src/test/csf/PeerGroup.h rename to src/tests/libxrpl/csf/PeerGroup.h index 1c31209ef3..ff99b779a3 100644 --- a/src/test/csf/PeerGroup.h +++ b/src/tests/libxrpl/csf/PeerGroup.h @@ -1,9 +1,9 @@ #pragma once -#include -#include -#include -#include +#include +#include +#include +#include #include #include diff --git a/src/test/csf/Proposal.h b/src/tests/libxrpl/csf/Proposal.h similarity index 66% rename from src/test/csf/Proposal.h rename to src/tests/libxrpl/csf/Proposal.h index ecf430ae8d..b2a97f9731 100644 --- a/src/test/csf/Proposal.h +++ b/src/tests/libxrpl/csf/Proposal.h @@ -1,10 +1,10 @@ #pragma once -#include -#include -#include +#include -#include +#include +#include +#include namespace xrpl::test::csf { /** diff --git a/src/test/csf/README.md b/src/tests/libxrpl/csf/README.md similarity index 100% rename from src/test/csf/README.md rename to src/tests/libxrpl/csf/README.md diff --git a/src/tests/libxrpl/csf/Scheduler.cpp b/src/tests/libxrpl/csf/Scheduler.cpp new file mode 100644 index 0000000000..63871e0623 --- /dev/null +++ b/src/tests/libxrpl/csf/Scheduler.cpp @@ -0,0 +1,61 @@ +#include + +#include + +#include + +namespace xrpl::test { + +TEST(SchedulerTest, scheduler) +{ + using namespace std::chrono_literals; + csf::Scheduler scheduler; + std::set seen; + + scheduler.in(1s, [&] { seen.insert(1); }); + scheduler.in(2s, [&] { seen.insert(2); }); + auto token = scheduler.in(3s, [&] { seen.insert(3); }); + scheduler.at(scheduler.now() + 4s, [&] { seen.insert(4); }); + scheduler.at(scheduler.now() + 8s, [&] { seen.insert(8); }); + + auto start = scheduler.now(); + + // Process first event + EXPECT_TRUE(seen.empty()); + EXPECT_TRUE(scheduler.stepOne()); + EXPECT_TRUE(seen == std::set({1})); + EXPECT_TRUE(scheduler.now() == (start + 1s)); + + // No processing if stepping until current time + EXPECT_TRUE(scheduler.stepUntil(scheduler.now())); + EXPECT_TRUE(seen == std::set({1})); + EXPECT_TRUE(scheduler.now() == (start + 1s)); + + // Process next event + EXPECT_TRUE(scheduler.stepFor(1s)); + EXPECT_TRUE(seen == std::set({1, 2})); + EXPECT_TRUE(scheduler.now() == (start + 2s)); + + // Don't process cancelled event, but advance clock + scheduler.cancel(token); + EXPECT_TRUE(scheduler.stepFor(1s)); + EXPECT_TRUE(seen == std::set({1, 2})); + EXPECT_TRUE(scheduler.now() == (start + 3s)); + + // Process until 3 seen ints + EXPECT_TRUE(scheduler.stepWhile([&]() { return seen.size() < 3; })); + EXPECT_TRUE(seen == std::set({1, 2, 4})); + EXPECT_TRUE(scheduler.now() == (start + 4s)); + + // Process the rest + EXPECT_TRUE(scheduler.step()); + EXPECT_TRUE(seen == std::set({1, 2, 4, 8})); + EXPECT_TRUE(scheduler.now() == (start + 8s)); + + // Process the rest again doesn't advance + EXPECT_TRUE(!scheduler.step()); + EXPECT_TRUE(seen == std::set({1, 2, 4, 8})); + EXPECT_TRUE(scheduler.now() == (start + 8s)); +} + +} // namespace xrpl::test diff --git a/src/test/csf/Scheduler.h b/src/tests/libxrpl/csf/Scheduler.h similarity index 100% rename from src/test/csf/Scheduler.h rename to src/tests/libxrpl/csf/Scheduler.h diff --git a/src/test/csf/Sim.h b/src/tests/libxrpl/csf/Sim.h similarity index 94% rename from src/test/csf/Sim.h rename to src/tests/libxrpl/csf/Sim.h index 94d26d5e06..774537138a 100644 --- a/src/test/csf/Sim.h +++ b/src/tests/libxrpl/csf/Sim.h @@ -1,16 +1,16 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include - #include +#include +#include +#include +#include +#include +#include +#include +#include + #include #include #include diff --git a/src/test/csf/SimTime.h b/src/tests/libxrpl/csf/SimTime.h similarity index 100% rename from src/test/csf/SimTime.h rename to src/tests/libxrpl/csf/SimTime.h diff --git a/src/test/csf/TrustGraph.h b/src/tests/libxrpl/csf/TrustGraph.h similarity index 99% rename from src/test/csf/TrustGraph.h rename to src/tests/libxrpl/csf/TrustGraph.h index d46a887364..d010b954e0 100644 --- a/src/test/csf/TrustGraph.h +++ b/src/tests/libxrpl/csf/TrustGraph.h @@ -1,9 +1,9 @@ #pragma once -#include - #include +#include + #include #include #include diff --git a/src/test/csf/Tx.h b/src/tests/libxrpl/csf/Tx.h similarity index 100% rename from src/test/csf/Tx.h rename to src/tests/libxrpl/csf/Tx.h diff --git a/src/test/csf/Validation.h b/src/tests/libxrpl/csf/Validation.h similarity index 99% rename from src/test/csf/Validation.h rename to src/tests/libxrpl/csf/Validation.h index 0b9fc94890..4325f96511 100644 --- a/src/test/csf/Validation.h +++ b/src/tests/libxrpl/csf/Validation.h @@ -1,10 +1,10 @@ #pragma once -#include - #include #include +#include + #include #include #include diff --git a/src/test/csf/collectors.h b/src/tests/libxrpl/csf/collectors.h similarity index 99% rename from src/test/csf/collectors.h rename to src/tests/libxrpl/csf/collectors.h index f85854e5dd..05b63f592b 100644 --- a/src/test/csf/collectors.h +++ b/src/tests/libxrpl/csf/collectors.h @@ -1,13 +1,13 @@ #pragma once -#include -#include -#include -#include -#include - #include +#include +#include +#include +#include +#include + #include #include #include diff --git a/src/test/csf/csf_graph.png b/src/tests/libxrpl/csf/csf_graph.png similarity index 100% rename from src/test/csf/csf_graph.png rename to src/tests/libxrpl/csf/csf_graph.png diff --git a/src/test/csf/csf_overview.png b/src/tests/libxrpl/csf/csf_overview.png similarity index 100% rename from src/test/csf/csf_overview.png rename to src/tests/libxrpl/csf/csf_overview.png diff --git a/src/test/csf/events.h b/src/tests/libxrpl/csf/events.h similarity index 96% rename from src/test/csf/events.h rename to src/tests/libxrpl/csf/events.h index 2cf4fd9e9b..4ff15ec987 100644 --- a/src/test/csf/events.h +++ b/src/tests/libxrpl/csf/events.h @@ -1,8 +1,8 @@ #pragma once -#include -#include -#include +#include +#include +#include namespace xrpl::test::csf { diff --git a/src/test/csf/impl/Sim.cpp b/src/tests/libxrpl/csf/impl/Sim.cpp similarity index 93% rename from src/test/csf/impl/Sim.cpp rename to src/tests/libxrpl/csf/impl/Sim.cpp index bf4706927e..28ac5f126a 100644 --- a/src/test/csf/impl/Sim.cpp +++ b/src/tests/libxrpl/csf/impl/Sim.cpp @@ -1,7 +1,7 @@ -#include +#include -#include -#include +#include +#include #include #include diff --git a/src/test/csf/impl/ledgers.cpp b/src/tests/libxrpl/csf/impl/ledgers.cpp similarity index 98% rename from src/test/csf/impl/ledgers.cpp rename to src/tests/libxrpl/csf/impl/ledgers.cpp index 46a3600307..ed1cd927c4 100644 --- a/src/test/csf/impl/ledgers.cpp +++ b/src/tests/libxrpl/csf/impl/ledgers.cpp @@ -1,11 +1,11 @@ -#include - -#include +#include #include #include #include +#include + #include #include #include diff --git a/src/test/csf/ledgers.h b/src/tests/libxrpl/csf/ledgers.h similarity index 99% rename from src/test/csf/ledgers.h rename to src/tests/libxrpl/csf/ledgers.h index 09f5fa54de..ca4e44d5a6 100644 --- a/src/test/csf/ledgers.h +++ b/src/tests/libxrpl/csf/ledgers.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include #include @@ -9,6 +7,8 @@ #include +#include + #include #include #include diff --git a/src/test/csf/random.h b/src/tests/libxrpl/csf/random.h similarity index 98% rename from src/test/csf/random.h rename to src/tests/libxrpl/csf/random.h index f8df253642..007bdecb1b 100644 --- a/src/test/csf/random.h +++ b/src/tests/libxrpl/csf/random.h @@ -135,13 +135,13 @@ class PowerLawDistribution { double xmin_; double a_; - double inv_; + double inv_{1.0 / (1.0 - a_)}; std::uniform_real_distribution uf_{0, 1}; public: using result_type = double; - PowerLawDistribution(double xmin, double a) : xmin_{xmin}, a_{a}, inv_(1.0 / (1.0 - a_)) + PowerLawDistribution(double xmin, double a) : xmin_{xmin}, a_{a} { } diff --git a/src/test/csf/submitters.h b/src/tests/libxrpl/csf/submitters.h similarity index 96% rename from src/test/csf/submitters.h rename to src/tests/libxrpl/csf/submitters.h index 160d0bcd9f..4f27f0f665 100644 --- a/src/test/csf/submitters.h +++ b/src/tests/libxrpl/csf/submitters.h @@ -1,8 +1,8 @@ #pragma once -#include -#include -#include +#include +#include +#include #include #include diff --git a/src/test/csf/timers.h b/src/tests/libxrpl/csf/timers.h similarity index 96% rename from src/test/csf/timers.h rename to src/tests/libxrpl/csf/timers.h index 4f13b21b25..e89ea33698 100644 --- a/src/test/csf/timers.h +++ b/src/tests/libxrpl/csf/timers.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include #include diff --git a/src/tests/libxrpl/helpers/CaptureSink.h b/src/tests/libxrpl/helpers/CaptureSink.h new file mode 100644 index 0000000000..9918d13f9e --- /dev/null +++ b/src/tests/libxrpl/helpers/CaptureSink.h @@ -0,0 +1,50 @@ +#pragma once + +#include + +#include +#include +#include + +namespace xrpl::test { + +class CaptureSink : public beast::Journal::Sink +{ + mutable std::mutex mutex_; + std::stringstream strm_; + +public: + explicit CaptureSink(beast::Severity threshold = beast::Severity::Debug) + : Sink{threshold, false} + { + } + + void + write(beast::Severity level, std::string const& text) override + { + if (level < threshold()) + return; + writeAlways(level, text); + } + + void + writeAlways(beast::Severity /*level*/, std::string const& text) override + { + // Journal sinks may be written to concurrently (e.g. from a backend's background workers), + // so serialize access to strm_. write() funnels into writeAlways(), so the lock lives here + // only: locking in both would self-deadlock on this non-recursive mutex. + std::scoped_lock const lock(mutex_); + strm_ << text << '\n'; + } + + [[nodiscard]] std::string + messages() const + { + // Returns a snapshot of the captured output. Takes the lock so the read is safe even if a + // writer is still active. + std::scoped_lock const lock(mutex_); + return strm_.str(); + } +}; + +} // namespace xrpl::test diff --git a/src/tests/libxrpl/helpers/TestFamily.h b/src/tests/libxrpl/helpers/TestFamily.h index 1a11d3bb68..8a599ab4da 100644 --- a/src/tests/libxrpl/helpers/TestFamily.h +++ b/src/tests/libxrpl/helpers/TestFamily.h @@ -29,11 +29,11 @@ namespace xrpl::test { class TestFamily : public Family { private: - std::unique_ptr db_; + std::unique_ptr db_; TestStopwatch clock_; std::shared_ptr fbCache_; std::shared_ptr tnCache_; - NodeStore::DummyScheduler scheduler_; + node_store::DummyScheduler scheduler_; beast::Journal j_; public: @@ -51,16 +51,16 @@ public: Section config; config.set(Keys::kType, "memory"); config.set(Keys::kPath, "TestFamily"); - db_ = NodeStore::Manager::instance().makeDatabase(megabytes(4), scheduler_, 1, config, j); + db_ = node_store::Manager::instance().makeDatabase(megabytes(4), scheduler_, 1, config, j); } - NodeStore::Database& + node_store::Database& db() override { return *db_; } - [[nodiscard]] NodeStore::Database const& + [[nodiscard]] node_store::Database const& db() const override { return *db_; diff --git a/src/tests/libxrpl/helpers/TestServiceRegistry.h b/src/tests/libxrpl/helpers/TestServiceRegistry.h index d948f8d66f..0e1ff0f2c0 100644 --- a/src/tests/libxrpl/helpers/TestServiceRegistry.h +++ b/src/tests/libxrpl/helpers/TestServiceRegistry.h @@ -221,7 +221,7 @@ public: } // Storage services - NodeStore::Database& + node_store::Database& getNodeStore() override { throw std::logic_error("TestServiceRegistry::getNodeStore() not implemented"); diff --git a/src/tests/libxrpl/main.cpp b/src/tests/libxrpl/main.cpp index 5142bbe08a..f9114bffc4 100644 --- a/src/tests/libxrpl/main.cpp +++ b/src/tests/libxrpl/main.cpp @@ -1,8 +1,9 @@ +#include #include int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); + ::testing::InitGoogleMock(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/src/tests/libxrpl/nodestore/Backend.cpp b/src/tests/libxrpl/nodestore/Backend.cpp new file mode 100644 index 0000000000..eb78851429 --- /dev/null +++ b/src/tests/libxrpl/nodestore/Backend.cpp @@ -0,0 +1,182 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::node_store { + +namespace { + +std::vector +backendTypes() +{ + std::vector types{"nudb"}; +#if XRPL_ROCKSDB_AVAILABLE + types.emplace_back("rocksdb"); +#endif +#ifdef XRPL_ENABLE_SQLITE_BACKEND_TESTS + types.emplace_back("sqlite"); +#endif + return types; +} + +// Run work(i) for every i in [0, n) spread across numThreads threads, handing +// out indices via a shared atomic counter (mirrors the old Timing_test +// parallel-for so the N items are partitioned, not duplicated). +template +void +parallelFor(std::size_t n, std::size_t numThreads, Work work) +{ + std::atomic next{0}; + auto const runner = [&] { + for (std::size_t i = next++; i < n; i = next++) + work(i); + }; + + auto threads = std::views::iota(std::size_t{0}, numThreads) | + std::views::transform([&](std::size_t) { return std::thread{runner}; }) | + std::ranges::to(); + + std::ranges::for_each(threads, &std::thread::join); +} + +} // namespace + +class BackendTypeTest : public ::testing::TestWithParam +{ +protected: + void + SetUp() override + { + params_.set("type", GetParam()); + params_.set("path", tempDir_.path()); + + beast::xor_shift_engine rng(kSeedValue); + batch_ = createPredictableBatch(kNumObjects, rng()); + } + + std::unique_ptr + makeOpenBackend() + { + auto backend = Manager::instance().makeBackend(params_, megabytes(4), scheduler_, journal_); + backend->open(); + return backend; + } + + DummyScheduler scheduler_; + beast::TempDir const tempDir_; + beast::Journal const journal_{TestSink::instance()}; + Section params_; + Batch batch_; +}; + +TEST_P(BackendTypeTest, store_and_fetch) +{ + auto backend = makeOpenBackend(); + storeBatch(*backend, batch_); + + { + SCOPED_TRACE("read in original order"); + auto const copy = fetchCopyOfBatch(*backend, batch_); + EXPECT_EQ(batch_, copy); + } + + { + SCOPED_TRACE("read in shuffled order"); + beast::xor_shift_engine rng(kSeedValue); + std::shuffle(batch_.begin(), batch_.end(), rng); + auto const copy = fetchCopyOfBatch(*backend, batch_); + EXPECT_EQ(batch_, copy); + } +} + +TEST_P(BackendTypeTest, persists_after_reopen) +{ + { + auto backend = makeOpenBackend(); + storeBatch(*backend, batch_); + } + + // re-open a fresh backend instance over the same path + auto backend = makeOpenBackend(); + auto copy = fetchCopyOfBatch(*backend, batch_); + std::ranges::sort(batch_, LessThan{}); + std::ranges::sort(copy, LessThan{}); + EXPECT_EQ(batch_, copy); +} + +// missing-key path. Replaces the correctness half of Timing_test::doMissing +// (and the missing branch of doMixed): every fetch on an empty backend must +// report Status::NotFound. +TEST_P(BackendTypeTest, fetch_missing) +{ + auto backend = makeOpenBackend(); + // deliberately do NOT store batch_ — every key must be absent + fetchMissing(*backend, batch_); +} + +// concurrent store/fetch correctness. Replaces the correctness half of the +// multi-threaded Timing_test workloads (which only ran manually, never in CI): +// many threads store disjoint objects, then many threads fetch and verify each +// round-trips. Doubles as a thread-safety smoke test for the backend. +TEST_P(BackendTypeTest, concurrent_store_and_fetch) +{ + // The SQLite backend is not designed for concurrent writers (and the old + // Timing_test only exercised nudb/rocksdb under threads). + if (GetParam() == "sqlite") + GTEST_SKIP() << "sqlite backend is not exercised under concurrency"; + + for (auto const numThreads : {4uz, 8uz}) + { + SCOPED_TRACE("threads=" + std::to_string(numThreads)); + + auto backend = makeOpenBackend(); + + // concurrent stores of disjoint objects + parallelFor(batch_.size(), numThreads, [&](std::size_t i) { backend->store(batch_[i]); }); + + // concurrent fetches, each verifying its object round-trips. Worker + // threads only touch an atomic counter; the EXPECT runs on the main + // thread after join to avoid relying on cross-thread assertion support. + std::atomic mismatches{0}; + parallelFor(batch_.size(), numThreads, [&](std::size_t i) { + std::shared_ptr result; + if (backend->fetch(batch_[i]->getHash(), &result) != Status::Ok || !result || + !isSame(result, batch_[i])) + { + ++mismatches; + } + }); + EXPECT_EQ(mismatches.load(), 0u); + + backend->close(); + } +} + +INSTANTIATE_TEST_SUITE_P( + BackendTypes, + BackendTypeTest, + ::testing::ValuesIn(backendTypes()), + [](::testing::TestParamInfo const& info) { return info.param; }); + +} // namespace xrpl::node_store diff --git a/src/tests/libxrpl/nodestore/Basics.cpp b/src/tests/libxrpl/nodestore/Basics.cpp new file mode 100644 index 0000000000..5bb902af0d --- /dev/null +++ b/src/tests/libxrpl/nodestore/Basics.cpp @@ -0,0 +1,41 @@ +#include +#include +#include + +#include +#include + +#include +#include +#include + +namespace xrpl::node_store { + +TEST(NodeStoreBasics, predictable_batches) +{ + auto const batch1 = createPredictableBatch(kNumObjectsToTest, kSeedValue); + auto const batch2 = createPredictableBatch(kNumObjectsToTest, kSeedValue); + EXPECT_EQ(batch1, batch2); + + auto const batch3 = createPredictableBatch(kNumObjectsToTest, kSeedValue + 1); + EXPECT_NE(batch1, batch3); +} + +TEST(NodeStoreBasics, blob_encoding) +{ + auto const batch = createPredictableBatch(kNumObjectsToTest, kSeedValue); + for (std::size_t i = 0; i < batch.size(); ++i) + { + SCOPED_TRACE("blob index=" + std::to_string(i)); + EncodedBlob const encoded(batch[i]); + DecodedBlob decoded(encoded.getKey(), encoded.getData(), encoded.getSize()); + EXPECT_TRUE(decoded.wasOk()); + if (decoded.wasOk()) + { + std::shared_ptr const object(decoded.createObject()); + EXPECT_TRUE(isSame(batch[i], object)); + } + } +} + +} // namespace xrpl::node_store diff --git a/src/tests/libxrpl/nodestore/Database.cpp b/src/tests/libxrpl/nodestore/Database.cpp new file mode 100644 index 0000000000..23087a2f84 --- /dev/null +++ b/src/tests/libxrpl/nodestore/Database.cpp @@ -0,0 +1,248 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace xrpl::node_store { + +namespace { + +std::vector +allBackends() +{ + std::vector types{"memory", "nudb"}; +#if XRPL_ROCKSDB_AVAILABLE + types.emplace_back("rocksdb"); +#endif + return types; +} + +std::vector +persistentBackends() +{ + std::vector types{"nudb"}; +#if XRPL_ROCKSDB_AVAILABLE + types.emplace_back("rocksdb"); +#endif + return types; +} + +std::vector +importBackends() +{ + std::vector types{"nudb"}; +#if XRPL_ROCKSDB_AVAILABLE + types.emplace_back("rocksdb"); +#endif +#ifdef XRPL_ENABLE_SQLITE_BACKEND_TESTS + types.emplace_back("sqlite"); +#endif + return types; +} + +} // namespace + +// Shared setup for the parameterized Database tests: builds the node params, +// journal and a predictable batch per test, mirroring Backend.cpp's fixture. +class NodeStoreDatabaseTestBase : public ::testing::TestWithParam +{ +protected: + void + SetUp() override + { + nodeParams_.set("type", GetParam()); + nodeParams_.set("path", nodeDb_.path()); + + beast::xor_shift_engine rng(kSeedValue); + batch_ = createPredictableBatch(kNumObjects, rng()); + } + + std::unique_ptr + makeDatabase() + { + return Manager::instance().makeDatabase(megabytes(4), scheduler_, 2, nodeParams_, journal_); + } + + DummyScheduler scheduler_; + beast::TempDir const nodeDb_; + beast::Journal const journal_{TestSink::instance()}; + Section nodeParams_; + Batch batch_; +}; + +class NodeStoreDatabaseTest : public NodeStoreDatabaseTestBase +{ +}; + +class NodeStoreDatabasePersistenceTest : public NodeStoreDatabaseTestBase +{ +}; + +TEST_P(NodeStoreDatabaseTest, store_and_fetch) +{ + auto db = makeDatabase(); + + storeBatch(*db, batch_); + + { + SCOPED_TRACE("read in original order"); + auto const copy = fetchCopyOfBatch(*db, batch_); + EXPECT_EQ(batch_, copy); + } + + { + SCOPED_TRACE("read in shuffled order"); + beast::xor_shift_engine rng(kSeedValue); + std::shuffle(batch_.begin(), batch_.end(), rng); + auto const copy = fetchCopyOfBatch(*db, batch_); + EXPECT_EQ(batch_, copy); + } +} + +TEST_P(NodeStoreDatabasePersistenceTest, round_trip) +{ + { + auto db = makeDatabase(); + storeBatch(*db, batch_); + } + + // re-open without the ephemeral db + auto db = makeDatabase(); + + auto copy = fetchCopyOfBatch(*db, batch_); + std::ranges::sort(batch_, LessThan{}); + std::ranges::sort(copy, LessThan{}); + EXPECT_EQ(batch_, copy); +} + +// missing-key path at the Database layer. Mirrors Backend's fetch_missing — +// fetching keys that were never stored must return nullptr (NotFound). +TEST_P(NodeStoreDatabaseTest, fetch_missing) +{ + auto db = makeDatabase(); + + // never store: every key must be absent + fetchMissing(*db, batch_); +} + +INSTANTIATE_TEST_SUITE_P( + NodeStoreBackends, + NodeStoreDatabaseTest, + ::testing::ValuesIn(allBackends()), + [](::testing::TestParamInfo const& info) { return info.param; }); + +INSTANTIATE_TEST_SUITE_P( + PersistentBackends, + NodeStoreDatabasePersistenceTest, + ::testing::ValuesIn(persistentBackends()), + [](::testing::TestParamInfo const& info) { return info.param; }); + +TEST(NodeStoreDatabase, memory_earliest_seq) +{ + DummyScheduler scheduler; + beast::TempDir const nodeDb; + Section nodeParams; + nodeParams.set("type", "memory"); + nodeParams.set("path", nodeDb.path()); + + beast::Journal const journal(TestSink::instance()); + + // default earliest ledger sequence + { + auto db = Manager::instance().makeDatabase(megabytes(4), scheduler, 2, nodeParams, journal); + EXPECT_EQ(db->earliestLedgerSeq(), kXrpLedgerEarliestSeq); + } + + // invalid earliest_seq value + { + nodeParams.set("earliest_seq", "0"); + try + { + auto db = + Manager::instance().makeDatabase(megabytes(4), scheduler, 2, nodeParams, journal); + FAIL() << "expected runtime_error for earliest_seq=0"; + } + catch (std::runtime_error const& e) + { + EXPECT_STREQ(e.what(), "Invalid earliest_seq"); + } + } + + // valid earliest_seq value + { + nodeParams.set("earliest_seq", "1"); + auto db = Manager::instance().makeDatabase(megabytes(4), scheduler, 2, nodeParams, journal); + EXPECT_EQ(db->earliestLedgerSeq(), 1u); + } +} + +class DatabaseImportTest : public ::testing::TestWithParam +{ +}; + +TEST_P(DatabaseImportTest, same_backend) +{ + auto const type = GetParam(); + + DummyScheduler scheduler; + beast::Journal const journal(TestSink::instance()); + + beast::TempDir const srcDir; + Section srcParams; + srcParams.set("type", type); + srcParams.set("path", srcDir.path()); + + auto batch = createPredictableBatch(kNumObjects, kSeedValue); + + // write to source db + { + auto src = Manager::instance().makeDatabase(megabytes(4), scheduler, 2, srcParams, journal); + storeBatch(*src, batch); + } + + Batch copy; + { + // re-open source and import into a fresh destination + auto src = Manager::instance().makeDatabase(megabytes(4), scheduler, 2, srcParams, journal); + + beast::TempDir const destDir; + Section destParams; + destParams.set("type", type); + destParams.set("path", destDir.path()); + + auto dest = + Manager::instance().makeDatabase(megabytes(4), scheduler, 2, destParams, journal); + + dest->importDatabase(*src); + copy = fetchCopyOfBatch(*dest, batch); + } + + std::ranges::sort(batch, LessThan{}); + std::ranges::sort(copy, LessThan{}); + EXPECT_EQ(batch, copy); +} + +INSTANTIATE_TEST_SUITE_P( + ImportBackends, + DatabaseImportTest, + ::testing::ValuesIn(importBackends()), + [](::testing::TestParamInfo const& info) { return info.param; }); + +} // namespace xrpl::node_store diff --git a/src/tests/libxrpl/nodestore/NuDBFactory.cpp b/src/tests/libxrpl/nodestore/NuDBFactory.cpp new file mode 100644 index 0000000000..c126984630 --- /dev/null +++ b/src/tests/libxrpl/nodestore/NuDBFactory.cpp @@ -0,0 +1,297 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::node_store { + +namespace { + +Section +makeSection(std::string const& path, std::string const& blockSize = "") +{ + Section params; + params.set("type", "nudb"); + params.set("path", path); + if (!blockSize.empty()) + params.set("nudb_block_size", blockSize); + return params; +} + +void +runRoundTrip(Section const& params, std::size_t expectedBlocksize) +{ + DummyScheduler scheduler; + beast::Journal const journal(TestSink::instance()); + auto backend = Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); + + ASSERT_TRUE(backend); + ASSERT_EQ(backend->getBlockSize(), expectedBlocksize); + backend->open(); + ASSERT_TRUE(backend->isOpen()); + + auto const batch = createPredictableBatch(10, 12345); + storeBatch(*backend, batch); + + auto const copy = fetchCopyOfBatch(*backend, batch); + + backend->close(); + EXPECT_EQ(batch, copy); +} + +} // namespace + +TEST(NuDBFactory, default_block_size) +{ + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path()); + ASSERT_NO_FATAL_FAILURE(runRoundTrip(params, 4096)); +} + +TEST(NuDBFactory, valid_block_sizes) +{ + auto const kValidSizes = std::to_array({4096, 8192, 16384, 32768}); + for (auto const size : kValidSizes) + { + SCOPED_TRACE("size=" + std::to_string(size)); + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), std::to_string(size)); + ASSERT_NO_FATAL_FAILURE(runRoundTrip(params, size)); + } + + // empty value is ignored by config parser; default (4096) is used + { + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), ""); + ASSERT_NO_FATAL_FAILURE(runRoundTrip(params, 4096)); + } +} + +TEST(NuDBFactory, invalid_block_sizes) +{ + std::vector const kInvalidSizes = { + "2048", // too small + "1024", // too small + "65536", // too large + "131072", // too large + "5000", // not power of 2 + "6000", // not power of 2 + "10000", // not power of 2 + "0", // zero + "-1", // negative + "abc", // non-numeric + "4k", // invalid format + "4096.5"}; // decimal + + for (auto const& size : kInvalidSizes) + { + SCOPED_TRACE("size='" + size + "'"); + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), size); + EXPECT_THROW(runRoundTrip(params, 4096), std::exception); + } + + // whitespace handling — lexical_cast may or may not strip; treat as invalid + std::vector const kWhitespaceSizes = {"4096 ", " 4096"}; + for (auto const& size : kWhitespaceSizes) + { + SCOPED_TRACE("size='" + size + "'"); + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), size); + EXPECT_THROW(runRoundTrip(params, 4096), std::exception); + } +} + +TEST(NuDBFactory, log_messages) +{ + // valid custom block size emits info log + { + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), "8192"); + test::CaptureSink sink(beast::Severity::Info); + beast::Journal const journal(sink); + + DummyScheduler scheduler; + [[maybe_unused]] auto backend = + Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); + + EXPECT_TRUE(sink.messages().contains("Using custom NuDB block size: 8192")); + } + + // invalid block size throws with informative message + { + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), "5000"); + test::CaptureSink sink(beast::Severity::Warning); + beast::Journal const journal(sink); + DummyScheduler scheduler; + try + { + auto backend = + Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); + FAIL() << "expected exception for invalid block size 5000"; + } + catch (std::exception const& e) + { + std::string const what{e.what()}; + EXPECT_TRUE(what.contains("Invalid nudb_block_size: 5000")); + EXPECT_TRUE(what.contains("Must be power of 2 between 4096 and 32768")); + } + } + + // non-numeric value throws + { + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), "invalid"); + test::CaptureSink sink(beast::Severity::Warning); + beast::Journal const journal(sink); + DummyScheduler scheduler; + try + { + auto backend = + Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); + FAIL() << "expected exception for non-numeric block size"; + } + catch (std::exception const& e) + { + std::string const what{e.what()}; + EXPECT_TRUE(what.contains("Invalid nudb_block_size value: invalid")); + } + } +} + +TEST(NuDBFactory, power_of_two_validation) +{ + std::vector> const kCASES = { + {"4095", false}, // just below minimum + {"4096", true}, // minimum valid + {"4097", false}, // not power of 2 + {"8192", true}, // valid power of 2 + {"8193", false}, // not power of 2 + {"16384", true}, // valid power of 2 + {"32768", true}, // maximum valid + {"32769", false}, // just above maximum + {"65536", false}}; // power of 2 but too large + + for (auto const& [size, shouldWork] : kCASES) + { + SCOPED_TRACE("size=" + size + " shouldWork=" + (shouldWork ? "true" : "false")); + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), size); + test::CaptureSink sink(beast::Severity::Warning); + beast::Journal const journal(sink); + DummyScheduler scheduler; + try + { + auto backend = + Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); + EXPECT_TRUE(shouldWork); + } + catch (std::exception const& e) + { + // A throw is only expected for sizes that should NOT work; if a + // valid size throws, fail here instead of silently matching the + // message below (which would mask the regression). + EXPECT_FALSE(shouldWork); + std::string const what{e.what()}; + EXPECT_TRUE(what.contains("Invalid nudb_block_size")); + } + } +} + +TEST(NuDBFactory, both_constructor_variants) +{ + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), "16384"); + DummyScheduler scheduler; + beast::Journal const journal(TestSink::instance()); + + auto backend1 = Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); + EXPECT_NE(backend1, nullptr); + ASSERT_NO_FATAL_FAILURE(runRoundTrip(params, 16384)); + + // Test second constructor (with nudb::context) + // Note: This would require access to nudb::context, which might not be + // easily testable without more complex setup. For now, we test that + // the factory can create backends with the first constructor. +} + +TEST(NuDBFactory, configuration_parsing) +{ + // basic valid format emits success log + { + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), "8192"); + test::CaptureSink sink(beast::Severity::Info); + beast::Journal const journal(sink); + DummyScheduler scheduler; + [[maybe_unused]] auto backend = + Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); + EXPECT_TRUE(sink.messages().contains("Using custom NuDB block size")); + } + + // Test whitespace handling separately since lexical_cast behavior may vary + std::vector const kWhitespaceFormats = {" 8192", "8192 "}; + for (auto const& format : kWhitespaceFormats) + { + SCOPED_TRACE("format='" + format + "'"); + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), format); + test::CaptureSink sink(beast::Severity::Debug); + beast::Journal const journal(sink); + DummyScheduler scheduler; + EXPECT_ANY_THROW(Manager::instance().makeBackend(params, megabytes(4), scheduler, journal)); + } +} + +TEST(NuDBFactory, data_persistence) +{ + std::vector const kBlockSizes = {"4096", "8192", "16384", "32768"}; + for (auto const& size : kBlockSizes) + { + SCOPED_TRACE("size=" + size); + beast::TempDir const tempDir; + auto const params = makeSection(tempDir.path(), size); + DummyScheduler scheduler; + beast::Journal const journal(TestSink::instance()); + + // Create test data + auto const batch = createPredictableBatch(50, 54321); + + // Store data + { + auto backend = + Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); + backend->open(); + storeBatch(*backend, batch); + backend->close(); + } + + // Retrieve data in new backend instance + { + auto backend = + Manager::instance().makeBackend(params, megabytes(4), scheduler, journal); + backend->open(); + auto const copy = fetchCopyOfBatch(*backend, batch); + EXPECT_EQ(batch, copy); + backend->close(); + } + } +} + +} // namespace xrpl::node_store diff --git a/src/tests/libxrpl/nodestore/TestBase.h b/src/tests/libxrpl/nodestore/TestBase.h new file mode 100644 index 0000000000..5a262ac7bb --- /dev/null +++ b/src/tests/libxrpl/nodestore/TestBase.h @@ -0,0 +1,169 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace xrpl::node_store { + +constexpr std::size_t kMinPayloadBytes = 1; +constexpr std::size_t kMaxPayloadBytes = 2000; +constexpr int kNumObjectsToTest = 2000; +constexpr int kNumObjects = 2000; +constexpr std::uint64_t kSeedValue = 50; + +struct LessThan +{ + bool + operator()(std::shared_ptr const& lhs, std::shared_ptr const& rhs) + const noexcept + { + return lhs->getHash() < rhs->getHash(); + } +}; + +[[nodiscard]] inline bool +isSame(std::shared_ptr const& lhs, std::shared_ptr const& rhs) +{ + return (lhs->getType() == rhs->getType()) && (lhs->getHash() == rhs->getHash()) && + (lhs->getData() == rhs->getData()); +} + +[[nodiscard]] inline Batch +createPredictableBatch(std::size_t numObjects, std::uint64_t seed) +{ + Batch batch; + batch.reserve(numObjects); + + beast::xor_shift_engine rng(seed); + + for (auto i = 0uz; i < numObjects; ++i) + { + NodeObjectType const type = [&] { + switch (randInt(rng, 3)) + { + case 0: + return NodeObjectType::Ledger; + case 1: + return NodeObjectType::AccountNode; + case 2: + return NodeObjectType::TransactionNode; + case 3: + default: + return NodeObjectType::Unknown; + } + }(); + + uint256 hash; + beast::rngfill(hash.begin(), hash.size(), rng); + + Blob blob(randInt(rng, kMinPayloadBytes, kMaxPayloadBytes)); + beast::rngfill(blob.data(), blob.size(), rng); + + batch.emplace_back(NodeObject::createObject(type, std::move(blob), hash)); + } + + return batch; +} + +inline void +storeBatch(Backend& backend, Batch const& batch) +{ + for (auto const& obj : batch) + backend.store(obj); +} + +[[nodiscard]] inline Batch +fetchCopyOfBatch(Backend& backend, Batch const& batch) +{ + Batch copy; + copy.reserve(batch.size()); + + for (auto i = 0uz; i < batch.size(); ++i) + { + SCOPED_TRACE("fetchCopyOfBatch index=" + std::to_string(i)); + std::shared_ptr object; + Status const status = backend.fetch(batch[i]->getHash(), &object); + EXPECT_EQ(status, Status::Ok); + if (status == Status::Ok) + { + EXPECT_NE(object, nullptr); + copy.emplace_back(object); + } + } + return copy; +} + +inline void +fetchMissing(Backend& backend, Batch const& batch) +{ + for (auto i = 0uz; i < batch.size(); ++i) + { + SCOPED_TRACE("fetchMissing index=" + std::to_string(i)); + std::shared_ptr object; + Status const status = backend.fetch(batch[i]->getHash(), &object); + EXPECT_EQ(status, Status::NotFound); + } +} + +inline void +storeBatch(Database& db, Batch const& batch) +{ + for (auto const& obj : batch) + { + Blob data(obj->getData()); + db.store(obj->getType(), std::move(data), obj->getHash(), db.earliestLedgerSeq()); + } +} + +[[nodiscard]] inline Batch +fetchCopyOfBatch(Database& db, Batch const& batch) +{ + Batch copy; + copy.reserve(batch.size()); + + for (auto const& obj : batch) + { + std::shared_ptr const result = db.fetchNodeObject(obj->getHash(), 0); + if (result != nullptr) + copy.emplace_back(result); + } + return copy; +} + +inline void +fetchMissing(Database& db, Batch const& batch) +{ + for (auto i = 0uz; i < batch.size(); ++i) + { + SCOPED_TRACE("fetchMissing(Database) index=" + std::to_string(i)); + EXPECT_EQ(db.fetchNodeObject(batch[i]->getHash(), 0), nullptr); + } +} + +} // namespace xrpl::node_store + +namespace xrpl { + +[[nodiscard]] inline bool +operator==(node_store::Batch const& lhs, node_store::Batch const& rhs) +{ + return std::ranges::equal(lhs, rhs, node_store::isSame); +} + +} // namespace xrpl diff --git a/src/tests/libxrpl/nodestore/varint.cpp b/src/tests/libxrpl/nodestore/varint.cpp new file mode 100644 index 0000000000..fee96f314b --- /dev/null +++ b/src/tests/libxrpl/nodestore/varint.cpp @@ -0,0 +1,46 @@ +#include + +#include + +#include +#include +#include +#include +#include + +using namespace xrpl::node_store; + +TEST(varint, encode_decode) +{ + std::vector const kVALUES = { + 0, + 1, + 2, + 126, + 127, + 128, + 253, + 254, + 255, + 16127, + 16128, + 16129, + 0xff, + 0xffff, + 0xffffffff, + 0xffffffffffffUL, + 0xffffffffffffffffUL}; + + for (auto const v : kVALUES) + { + SCOPED_TRACE("value=" + std::to_string(v)); + std::array::kMax> vi{}; + auto const n0 = writeVarint(vi.data(), v); + EXPECT_GT(n0, 0u) << "write error"; + EXPECT_EQ(n0, sizeVarint(v)) << "size error"; + std::size_t v1 = 0; + auto const n1 = readVarint(vi.data(), n0, v1); + EXPECT_EQ(n1, n0) << "read error"; + EXPECT_EQ(v1, v) << "wrong value"; + } +} diff --git a/src/tests/libxrpl/peerfinder/Livecache.cpp b/src/tests/libxrpl/peerfinder/Livecache.cpp new file mode 100644 index 0000000000..464ec0e5da --- /dev/null +++ b/src/tests/libxrpl/peerfinder/Livecache.cpp @@ -0,0 +1,294 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::PeerFinder { +namespace { + +class LivecacheTest : public ::testing::Test +{ +protected: + static beast::Journal + journal() + { + return beast::Journal{TestSink::instance()}; + } + + static beast::IP::Endpoint + endpoint(std::uint16_t index, bool v4 = true) + { + auto const port = static_cast(10000 + index); + + if (v4) + { + auto bytes = beast::IP::AddressV4::bytes_type{ + {54, + static_cast((index / 256) % 256), + static_cast(index % 256), + 1}}; + return beast::IP::Endpoint{beast::IP::Address{beast::IP::AddressV4{bytes}}, port}; + } + + auto bytes = beast::IP::AddressV6::bytes_type{ + {0x20, + 0x01, + 0x0d, + 0xb8, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + static_cast((index / 256) % 256), + static_cast(index % 256), + 1}}; + return beast::IP::Endpoint{beast::IP::Address{beast::IP::AddressV6{bytes}}, port}; + } + + void + addEndpoint(beast::IP::Endpoint const& ep, std::uint32_t hops = 0) + { + cache_.insert(Endpoint{ep, hops}); + } + + TestStopwatch clock_; + Livecache<> cache_{clock_, journal()}; +}; + +} // namespace + +TEST_F(LivecacheTest, basic_insert) +{ + EXPECT_TRUE(cache_.empty()); + + for (auto i = 0; i < 10; ++i) + addEndpoint(endpoint(i, true)); + + EXPECT_FALSE(cache_.empty()); + EXPECT_EQ(cache_.size(), 10u); + + for (auto i = 10; i < 20; ++i) + addEndpoint(endpoint(i, false)); + + EXPECT_FALSE(cache_.empty()); + EXPECT_EQ(cache_.size(), 20u); +} + +TEST_F(LivecacheTest, insert_update_keeps_lowest_hop_count) +{ + auto const ep1 = Endpoint{endpoint(1), 2}; + cache_.insert(ep1); + ASSERT_EQ(cache_.size(), 1u); + EXPECT_EQ((cache_.hops.begin() + 2)->begin()->hops, 2u); + + auto const ep2 = Endpoint{ep1.address, 4}; + cache_.insert(ep2); + EXPECT_EQ(cache_.size(), 1u); + EXPECT_EQ((cache_.hops.begin() + 2)->begin()->hops, 2u); + + auto const ep3 = Endpoint{ep1.address, 2}; + cache_.insert(ep3); + EXPECT_EQ(cache_.size(), 1u); + EXPECT_EQ((cache_.hops.begin() + 2)->begin()->hops, 2u); + + auto const ep4 = Endpoint{ep1.address, 1}; + cache_.insert(ep4); + EXPECT_EQ(cache_.size(), 1u); + EXPECT_EQ((cache_.hops.begin() + 1)->begin()->hops, 1u); +} + +TEST_F(LivecacheTest, hop_iterators_support_const_reverse_and_move_back) +{ + auto const ep1 = Endpoint{endpoint(1), 1}; + auto const ep2 = Endpoint{endpoint(2), 1}; + cache_.insert(ep1); + cache_.insert(ep2); + + auto hop = *(cache_.hops.begin() + 1); + ASSERT_NE(hop.begin(), hop.end()); + ASSERT_NE(hop.cbegin(), hop.cend()); + ASSERT_NE(hop.rbegin(), hop.rend()); + ASSERT_NE(hop.crbegin(), hop.crend()); + + auto const firstAddress = hop.begin()->address; + hop.moveBack(hop.begin()); + EXPECT_EQ(hop.rbegin()->address, firstAddress); + + auto const& constHops = cache_.hops; + EXPECT_NE(constHops.begin(), constHops.end()); + EXPECT_NE(constHops.cbegin(), constHops.cend()); + EXPECT_NE(constHops.rbegin(), constHops.rend()); + EXPECT_NE(constHops.crbegin(), constHops.crend()); + + auto const constHop = *(constHops.cbegin() + 1); + EXPECT_EQ(std::distance(constHop.begin(), constHop.end()), 2); + EXPECT_EQ(std::distance(constHop.cbegin(), constHop.cend()), 2); + EXPECT_EQ(std::distance(constHop.rbegin(), constHop.rend()), 2); + EXPECT_EQ(std::distance(constHop.crbegin(), constHop.crend()), 2); +} + +TEST_F(LivecacheTest, on_write_reports_entries_and_expiration) +{ + cache_.insert(Endpoint{endpoint(1), 1}); + cache_.insert(Endpoint{endpoint(2), Tuning::kMaxHops + 1}); + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + cache_.onWrite(map); + } + + auto const& top = stream.top(); + EXPECT_EQ(top["size"].asUInt(), 2u); + EXPECT_FALSE(top["hist"].asString().empty()); + ASSERT_TRUE(top.isMember("entries")); + ASSERT_EQ(top["entries"].size(), 2u); + auto const& entry = top["entries"][json::UInt{0}]; + EXPECT_TRUE(entry.isMember("hops")); + EXPECT_TRUE(entry.isMember("address")); + EXPECT_TRUE(entry.isMember("expires")); +} + +TEST_F(LivecacheTest, expire_removes_entries_after_ttl) +{ + using namespace std::chrono_literals; + + cache_.insert(Endpoint{endpoint(1), 1}); + ASSERT_EQ(cache_.size(), 1u); + + cache_.expire(); + EXPECT_EQ(cache_.size(), 1u); + + clock_.advance(Tuning::kLiveCacheSecondsToLive - 1s); + cache_.expire(); + EXPECT_EQ(cache_.size(), 1u); + + clock_.advance(1s); + cache_.expire(); + EXPECT_TRUE(cache_.empty()); +} + +TEST_F(LivecacheTest, expire_removes_multiple_entries_after_ttl) +{ + using namespace std::chrono_literals; + + cache_.insert(Endpoint{endpoint(1), 1}); + cache_.insert(Endpoint{endpoint(2), 2}); + + clock_.advance(Tuning::kLiveCacheSecondsToLive); + cache_.expire(); + EXPECT_TRUE(cache_.empty()); +} + +TEST_F(LivecacheTest, histogram_counts_all_entries) +{ + constexpr auto kNumEndpoints = 40; + + for (auto i = 0; i < kNumEndpoints; ++i) + { + addEndpoint(endpoint(static_cast(i)), xrpl::randInt()); + } + + auto const histogram = cache_.hops.histogram(); + ASSERT_FALSE(histogram.empty()); + + std::vector values; + boost::split(values, histogram, boost::algorithm::is_any_of(",")); + + auto sum = 0; + for (auto const& value : values) + { + auto const count = boost::lexical_cast(boost::trim_copy(value)); + sum += count; + EXPECT_GE(count, 0); + } + EXPECT_EQ(sum, kNumEndpoints); +} + +TEST_F(LivecacheTest, shuffle_preserves_bucket_contents) +{ + for (auto i = 0; i < 100; ++i) + { + addEndpoint(endpoint(static_cast(i)), xrpl::randInt(Tuning::kMaxHops + 1)); + } + + using AtHop = std::vector; + using AllHops = std::array; + + auto const compareEndpoint = [](Endpoint const& lhs, Endpoint const& rhs) { + return rhs.hops < lhs.hops || (rhs.hops == lhs.hops && rhs.address < lhs.address); + }; + auto const sameEndpoint = [](Endpoint const& lhs, Endpoint const& rhs) { + return lhs.hops == rhs.hops && lhs.address == rhs.address; + }; + auto const sameEndpoints = + [&sameEndpoint](std::vector const& lhs, std::vector const& rhs) { + return lhs.size() == rhs.size() && + std::equal(lhs.begin(), lhs.end(), rhs.begin(), sameEndpoint); + }; + + AllHops before; + AllHops beforeSorted; + for (auto i = std::make_pair(0, cache_.hops.begin()); i.second != cache_.hops.end(); + ++i.first, ++i.second) + { + std::ranges::copy(*i.second, std::back_inserter(before[i.first])); + std::ranges::copy(*i.second, std::back_inserter(beforeSorted[i.first])); + std::ranges::sort(beforeSorted[i.first], compareEndpoint); + } + + cache_.hops.shuffle(); + + AllHops after; + AllHops afterSorted; + for (auto i = std::make_pair(0, cache_.hops.begin()); i.second != cache_.hops.end(); + ++i.first, ++i.second) + { + std::ranges::copy(*i.second, std::back_inserter(after[i.first])); + std::ranges::copy(*i.second, std::back_inserter(afterSorted[i.first])); + std::ranges::sort(afterSorted[i.first], compareEndpoint); + } + + auto allBucketsKeptOriginalOrder = true; + for (auto i = 0u; i < before.size(); ++i) + { + EXPECT_EQ(before[i].size(), after[i].size()); + allBucketsKeptOriginalOrder = + allBucketsKeptOriginalOrder && sameEndpoints(before[i], after[i]); + EXPECT_TRUE(sameEndpoints(beforeSorted[i], afterSorted[i])); + } + EXPECT_FALSE(allBucketsKeptOriginalOrder); +} + +} // namespace xrpl::PeerFinder diff --git a/src/tests/libxrpl/peerfinder/PeerFinder.cpp b/src/tests/libxrpl/peerfinder/PeerFinder.cpp new file mode 100644 index 0000000000..31fa59d1ce --- /dev/null +++ b/src/tests/libxrpl/peerfinder/PeerFinder.cpp @@ -0,0 +1,1270 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace xrpl::PeerFinder { +namespace { + +using ::testing::_; +using ::testing::NiceMock; +using ::testing::Return; + +beast::Journal +journal() +{ + return beast::Journal{TestSink::instance()}; +} + +beast::IP::Endpoint +endpoint(std::string const& value) +{ + return beast::IP::Endpoint::fromString(value); +} + +class MockStore : public Store +{ +public: + MOCK_METHOD(std::size_t, load, (Store::load_callback const& cb), (override)); + MOCK_METHOD(void, save, (std::vector const& entries), (override)); +}; + +class CapturingStore : public Store +{ +public: + std::vector entriesToLoad; + std::vector> saves; + + std::size_t + load(Store::load_callback const& cb) override + { + for (auto const& entry : entriesToLoad) + cb(entry.endpoint, entry.valence); + return entriesToLoad.size(); + } + + void + save(std::vector const& entries) override + { + saves.push_back(entries); + } +}; + +Store::Entry +storeEntry(beast::IP::Endpoint const& endpoint, int valence) +{ + Store::Entry entry; + entry.endpoint = endpoint; + entry.valence = valence; + return entry; +} + +void +allowEmptyStore(MockStore& store) +{ + ON_CALL(store, load(_)).WillByDefault(Return(0)); + ON_CALL(store, save(_)).WillByDefault([](std::vector const&) {}); +} + +class MockChecker +{ +public: + MOCK_METHOD(void, stop, ()); + MOCK_METHOD(void, wait, ()); + MOCK_METHOD(void, recordAsyncConnect, (beast::IP::Endpoint const& ep)); + + boost::system::error_code nextError; + bool completeAsync = true; + std::vector asyncConnects; + + template + void + asyncConnect(beast::IP::Endpoint const& ep, Handler&& handler) + { + asyncConnects.push_back(ep); + recordAsyncConnect(ep); + if (completeAsync) + std::forward(handler)(nextError); + } +}; + +class TestSource : public Source +{ +public: + explicit TestSource(std::string name) : name_(std::move(name)) + { + } + + std::string const& + name() override + { + return name_; + } + + void + cancel() override + { + ++cancelCount; + } + + void + fetch(Results& results, beast::Journal) override + { + ++fetchCount; + results = resultsToFetch; + } + + Results resultsToFetch; + int fetchCount = 0; + int cancelCount = 0; + +private: + std::string name_; +}; + +class DefaultCancelSource : public Source +{ +public: + std::string const& + name() override + { + return name_; + } + + void + fetch(Results& results, beast::Journal) override + { + results = resultsToFetch; + } + + Results resultsToFetch; + +private: + std::string name_{"default"}; +}; + +class PeerFinderTest : public ::testing::Test +{ +public: + PeerFinderTest() + { + allowEmptyStore(store_); + } + +protected: + void + configure(std::size_t ipLimit = 2) + { + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = static_cast(ipLimit); + logic_.config(config); + } + + NiceMock store_; + NiceMock checker_; + TestStopwatch clock_; + Logic> logic_{clock_, store_, checker_, journal()}; +}; + +int +savedValence(std::vector const& entries, beast::IP::Endpoint const& endpoint) +{ + for (auto const& entry : entries) + { + if (entry.endpoint == endpoint) + return entry.valence; + } + + ADD_FAILURE() << "missing saved endpoint " << endpoint.toString(); + return 0; +} + +TEST_F(PeerFinderTest, backoff_limits_repeated_connection_attempts) +{ + auto constexpr kSECONDS = 10000; + + logic_.addFixedPeer("test", endpoint("65.0.0.1:5")); + configure(); + + std::size_t attempts = 0; + for (std::size_t i = 0; i < kSECONDS; ++i) + { + auto const list = logic_.autoconnect(); + if (!list.empty()) + { + ASSERT_EQ(list.size(), 1u); + auto const [slot, result] = logic_.newOutboundSlot(list.front()); + ASSERT_NE(slot, nullptr); + ASSERT_EQ(result, Result::Success); + EXPECT_TRUE(logic_.onConnected(slot, endpoint("65.0.0.2:5"))); + logic_.onClosed(slot); + ++attempts; + } + clock_.advance(std::chrono::seconds(1)); + logic_.oncePerSecond(); + } + + EXPECT_LT(attempts, 20u); +} + +TEST_F(PeerFinderTest, activated_peer_backoff_allows_at_most_one_attempt_per_minute) +{ + auto constexpr kSECONDS = 10000; + + logic_.addFixedPeer("test", endpoint("65.0.0.1:5")); + configure(); + + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + + std::size_t attempts = 0; + for (std::size_t i = 0; i < kSECONDS; ++i) + { + auto const list = logic_.autoconnect(); + if (!list.empty()) + { + ASSERT_EQ(list.size(), 1u); + auto const [slot, result] = logic_.newOutboundSlot(list.front()); + ASSERT_NE(slot, nullptr); + ASSERT_EQ(result, Result::Success); + ASSERT_TRUE(logic_.onConnected(slot, endpoint("65.0.0.2:5"))); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + logic_.onClosed(slot); + ++attempts; + } + clock_.advance(std::chrono::seconds(1)); + logic_.oncePerSecond(); + } + + EXPECT_LE(attempts, (kSECONDS + 59u) / 60u); +} + +TEST_F(PeerFinderTest, duplicate_inbound_slot_is_rejected_for_existing_outbound_slot) +{ + configure(); + + auto const remote = endpoint("65.0.0.1:5"); + auto const [slot1, result1] = logic_.newOutboundSlot(remote); + ASSERT_NE(slot1, nullptr); + EXPECT_EQ(result1, Result::Success); + EXPECT_EQ(logic_.connectedAddresses.count(remote.address()), 1u); + + auto const local = endpoint("65.0.0.2:1024"); + auto const [slot2, result2] = logic_.newInboundSlot(local, remote); + EXPECT_EQ(logic_.connectedAddresses.count(remote.address()), 1u); + EXPECT_EQ(result2, Result::DuplicatePeer); + EXPECT_EQ(slot2, nullptr); + + if (slot2) + logic_.onClosed(slot2); + logic_.onClosed(slot1); +} + +TEST_F(PeerFinderTest, duplicate_outbound_slot_is_rejected_for_existing_inbound_slot) +{ + configure(); + + auto const remote = endpoint("65.0.0.1:5"); + auto const local = endpoint("65.0.0.2:1024"); + + auto const [slot1, result1] = logic_.newInboundSlot(local, remote); + ASSERT_NE(slot1, nullptr); + EXPECT_EQ(result1, Result::Success); + EXPECT_EQ(logic_.connectedAddresses.count(remote.address()), 1u); + + auto const [slot2, result2] = logic_.newOutboundSlot(remote); + EXPECT_EQ(result2, Result::DuplicatePeer); + EXPECT_EQ(logic_.connectedAddresses.count(remote.address()), 1u); + EXPECT_EQ(slot2, nullptr); + + if (slot2) + logic_.onClosed(slot2); + logic_.onClosed(slot1); +} + +TEST_F(PeerFinderTest, peer_limit_exceeded_rejects_additional_inbound_slot) +{ + configure(); + + auto const local = endpoint("65.0.0.2:1024"); + auto const [slot, result] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1025")); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + + auto const [slot1, result1] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1026")); + ASSERT_NE(slot1, nullptr); + EXPECT_EQ(result1, Result::Success); + + auto const [slot2, result2] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1027")); + EXPECT_EQ(result2, Result::IpLimitExceeded); + EXPECT_EQ(slot2, nullptr); + + if (slot2) + logic_.onClosed(slot2); + logic_.onClosed(slot1); + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, activate_rejects_duplicate_public_key) +{ + configure(); + + auto const local = endpoint("65.0.0.2:1024"); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + + auto const [slot, result] = logic_.newOutboundSlot(endpoint("55.104.0.2:1025")); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + + auto const [slot2, result2] = logic_.newOutboundSlot(endpoint("55.104.0.2:1026")); + ASSERT_NE(slot2, nullptr); + EXPECT_EQ(result2, Result::Success); + + EXPECT_TRUE(logic_.onConnected(slot, local)); + EXPECT_TRUE(logic_.onConnected(slot2, local)); + + EXPECT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + EXPECT_EQ(logic_.activate(slot2, publicKey, false), Result::DuplicatePeer); + + logic_.onClosed(slot); + + EXPECT_EQ(logic_.activate(slot2, publicKey, false), Result::Success); + logic_.onClosed(slot2); +} + +TEST_F(PeerFinderTest, activate_rejects_inbound_when_inbound_connections_are_disabled) +{ + configure(); + + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + auto const local = endpoint("65.0.0.2:1024"); + + auto const [slot, result] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1025")); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + + EXPECT_EQ(logic_.activate(slot, publicKey, false), Result::InboundDisabled); + + { + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.inPeers = 1; + logic_.config(config); + } + + EXPECT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + auto const [slot2, result2] = logic_.newInboundSlot(local, endpoint("55.104.0.2:1026")); + ASSERT_NE(slot2, nullptr); + EXPECT_EQ(result2, Result::Success); + + PublicKey const publicKey2(randomKeyPair(KeyType::Secp256k1).first); + EXPECT_EQ(logic_.activate(slot2, publicKey2, false), Result::Full); + + logic_.onClosed(slot2); + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, add_fixed_peer_rejects_endpoint_without_port) +{ + EXPECT_THROW(logic_.addFixedPeer("test", endpoint("65.0.0.2")), std::runtime_error); +} + +TEST_F(PeerFinderTest, on_connected_rejects_self_connection) +{ + auto const local = endpoint("65.0.0.2:1234"); + auto const [slot, result] = logic_.newOutboundSlot(local); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + + EXPECT_FALSE(logic_.onConnected(slot, local)); + logic_.onClosed(slot); +} + +TEST(PeerFinderResult, converts_all_result_values_to_strings) +{ + EXPECT_EQ(to_string(Result::InboundDisabled), "inbound disabled"); + EXPECT_EQ(to_string(Result::DuplicatePeer), "peer already connected"); + EXPECT_EQ(to_string(Result::IpLimitExceeded), "ip limit exceeded"); + EXPECT_EQ(to_string(Result::Full), "slots full"); + EXPECT_EQ(to_string(Result::Success), "success"); + EXPECT_EQ(to_string(static_cast(-1)), "unknown"); +} + +TEST(PeerFinderEndpoint, orders_by_address) +{ + Endpoint const high{endpoint("65.0.0.2:10002"), 1}; + Endpoint const low{endpoint("65.0.0.1:10001"), 2}; + std::vector endpoints{high, low}; + + std::ranges::sort( + endpoints, [](Endpoint const& lhs, Endpoint const& rhs) { return lhs < rhs; }); + + EXPECT_EQ(endpoints.front().address, low.address); + EXPECT_EQ(endpoints.back().address, high.address); +} + +TEST(PeerFinderCounts, tracks_slot_states_and_capacity) +{ + TestStopwatch clock; + Counts counts; + Config config; + config.outPeers = 1; + config.inPeers = 1; + config.wantIncoming = true; + counts.onConfig(config); + + EXPECT_EQ(counts.outMax(), 1); + EXPECT_EQ(counts.inMax(), 1); + EXPECT_EQ(counts.inboundSlotsFree(), 1); + EXPECT_EQ(counts.outboundSlotsFree(), 1); + EXPECT_EQ(counts.totalActive(), 0); + EXPECT_FALSE(counts.isConnectedToNetwork()); + EXPECT_EQ(counts.attemptsNeeded(), Tuning::kMaxConnectAttempts); + EXPECT_EQ(counts.stateString(), "0/1 out, 0/1 in, 0 connecting, 0 closing"); + + SlotImp inbound(endpoint("65.0.0.1:10001"), endpoint("65.0.0.2:10002"), false, clock); + counts.add(inbound); + EXPECT_EQ(counts.acceptCount(), 1); + EXPECT_TRUE(counts.canActivate(inbound)); + counts.remove(inbound); + EXPECT_EQ(counts.acceptCount(), 0); + + inbound.activate(clock.now()); + counts.add(inbound); + EXPECT_EQ(counts.inboundActive(), 1); + EXPECT_EQ(counts.totalActive(), 1); + EXPECT_EQ(counts.inboundSlotsFree(), 0); + + SlotImp const extraInbound( + endpoint("65.0.0.3:10003"), endpoint("65.0.0.4:10004"), false, clock); + EXPECT_FALSE(counts.canActivate(extraInbound)); + counts.remove(inbound); + + SlotImp outbound(endpoint("65.0.0.5:10005"), false, clock); + counts.add(outbound); + EXPECT_EQ(counts.attempts(), 1); + EXPECT_EQ(counts.connectCount(), 1); + EXPECT_EQ(counts.attemptsNeeded(), Tuning::kMaxConnectAttempts - 1); + counts.remove(outbound); + + outbound.state(Slot::State::Connected); + EXPECT_TRUE(counts.canActivate(outbound)); + outbound.activate(clock.now()); + counts.add(outbound); + EXPECT_EQ(counts.outActive(), 1); + EXPECT_EQ(counts.outboundSlotsFree(), 0); + + SlotImp extraOutbound(endpoint("65.0.0.6:10006"), false, clock); + extraOutbound.state(Slot::State::Connected); + EXPECT_FALSE(counts.canActivate(extraOutbound)); + + SlotImp fixedOutbound(endpoint("65.0.0.7:10007"), true, clock); + fixedOutbound.state(Slot::State::Connected); + EXPECT_TRUE(counts.canActivate(fixedOutbound)); + fixedOutbound.activate(clock.now()); + counts.add(fixedOutbound); + EXPECT_EQ(counts.fixed(), 1u); + EXPECT_EQ(counts.fixedActive(), 1u); + counts.remove(fixedOutbound); + + SlotImp reservedOutbound(endpoint("65.0.0.8:10008"), false, clock); + reservedOutbound.reserved(true); + reservedOutbound.state(Slot::State::Connected); + EXPECT_TRUE(counts.canActivate(reservedOutbound)); + reservedOutbound.activate(clock.now()); + counts.add(reservedOutbound); + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + counts.onWrite(map); + } + EXPECT_TRUE(stream.top().isMember("accept")); + EXPECT_TRUE(stream.top().isMember("connect")); + EXPECT_TRUE(stream.top().isMember("close")); + EXPECT_TRUE(stream.top().isMember("reserved")); + EXPECT_TRUE(stream.top().isMember("total")); + counts.remove(reservedOutbound); + counts.remove(outbound); + + SlotImp closing(endpoint("65.0.0.9:10009"), endpoint("65.0.0.10:10010"), false, clock); + closing.state(Slot::State::Closing); + counts.add(closing); + EXPECT_EQ(counts.closingCount(), 1); + counts.remove(closing); + + Counts saturatedAttempts; + saturatedAttempts.onConfig(config); + std::vector> attempts; + for (int i = 0; i < Tuning::kMaxConnectAttempts; ++i) + { + attempts.push_back( + std::make_unique( + endpoint("65.1.0." + std::to_string(i + 1) + ":" + std::to_string(11000 + i)), + false, + clock)); + saturatedAttempts.add(*attempts.back()); + } + EXPECT_EQ(saturatedAttempts.attempts(), Tuning::kMaxConnectAttempts); + EXPECT_EQ(saturatedAttempts.attemptsNeeded(), 0u); + + Config disconnected; + disconnected.outPeers = 0; + counts.onConfig(disconnected); + EXPECT_TRUE(counts.isConnectedToNetwork()); +} + +TEST(PeerFinderHandouts, filters_redirect_slot_and_connect_targets) +{ + TestStopwatch clock; + auto const remote = endpoint("65.0.0.2:10002"); + auto const slot = std::make_shared(endpoint("65.0.0.1:10001"), remote, false, clock); + + RedirectHandouts redirects(slot); + EXPECT_EQ(redirects.slot(), slot); + EXPECT_TRUE(redirects.list().empty()); + EXPECT_FALSE(redirects.full()); + EXPECT_FALSE(redirects.tryInsert(Endpoint{endpoint("65.0.0.3:10003"), Tuning::kMaxHops + 1})); + EXPECT_FALSE(redirects.tryInsert(Endpoint{endpoint("65.0.0.3:10003"), 0})); + EXPECT_FALSE(redirects.tryInsert(Endpoint{remote.atPort(12000), 1})); + EXPECT_TRUE(redirects.tryInsert(Endpoint{endpoint("65.0.0.3:10003"), 1})); + EXPECT_FALSE(redirects.tryInsert(Endpoint{endpoint("65.0.0.3:12000"), 1})); + EXPECT_EQ(redirects.list().size(), 1u); + + SlotHandouts slotHandouts(slot); + EXPECT_EQ(slotHandouts.slot(), slot); + EXPECT_FALSE(slotHandouts.full()); + EXPECT_FALSE( + slotHandouts.tryInsert(Endpoint{endpoint("65.0.0.4:10004"), Tuning::kMaxHops + 1})); + EXPECT_FALSE(slotHandouts.tryInsert(Endpoint{remote.atPort(12001), 1})); + + auto const recent = endpoint("65.0.0.5:10005"); + slot->recent.insert(recent, 2); + EXPECT_FALSE(slotHandouts.tryInsert(Endpoint{recent, 2})); + EXPECT_TRUE(slotHandouts.tryInsert(Endpoint{endpoint("65.0.0.6:10006"), 2})); + EXPECT_FALSE(slotHandouts.tryInsert(Endpoint{endpoint("65.0.0.6:12000"), 2})); + slotHandouts.insert(Endpoint{endpoint("65.0.0.7:10007"), 1}); + EXPECT_EQ(slotHandouts.list().size(), 2u); + + ConnectHandouts::Squelches squelches(clock); + ConnectHandouts connects(2, squelches); + EXPECT_TRUE(connects.empty()); + EXPECT_TRUE(connects.tryInsert(endpoint("65.0.0.8:10008"))); + EXPECT_FALSE(connects.empty()); + EXPECT_FALSE(connects.tryInsert(endpoint("65.0.0.8:12000"))); + EXPECT_TRUE(connects.tryInsert(Endpoint{endpoint("65.0.0.9:10009"), 1})); + EXPECT_TRUE(connects.full()); + EXPECT_FALSE(connects.tryInsert(endpoint("65.0.0.10:10010"))); + EXPECT_EQ(connects.list().size(), 2u); + + ConnectHandouts squelched(1, squelches); + EXPECT_FALSE(squelched.tryInsert(endpoint("65.0.0.9:12000"))); +} + +TEST(PeerFinderHandouts, distributes_livecache_entries) +{ + TestStopwatch clock; + Livecache<> cache(clock, journal()); + cache.insert(Endpoint{endpoint("65.0.0.10:10010"), 1}); + cache.insert(Endpoint{endpoint("65.0.0.11:10011"), 2}); + + auto const slot1 = std::make_shared( + endpoint("65.0.0.1:10001"), endpoint("65.0.0.2:10002"), false, clock); + auto const slot2 = std::make_shared( + endpoint("65.0.0.3:10003"), endpoint("65.0.0.4:10004"), false, clock); + std::vector targets; + targets.emplace_back(slot1); + targets.emplace_back(slot2); + + handout(targets.begin(), targets.end(), cache.hops.begin(), cache.hops.end()); + + EXPECT_FALSE(targets.front().list().empty()); + EXPECT_FALSE(targets.back().list().empty()); + + for (std::uint32_t i = 0; i < Tuning::kNumberOfEndpoints; ++i) + targets.front().insert(Endpoint{endpoint("65.1.0." + std::to_string(i + 1) + ":12000"), 1}); + + handout(targets.begin(), targets.begin() + 1, cache.hops.begin(), cache.hops.end()); + EXPECT_TRUE(targets.front().full()); +} + +TEST_F(PeerFinderTest, preprocess_filters_invalid_duplicate_and_extra_self_endpoints) +{ + auto const local = endpoint("65.0.0.1:10001"); + auto const remote = endpoint("65.0.0.2:10002"); + auto const slot = std::make_shared(local, remote, false, clock_); + Endpoints endpoints{ + Endpoint{endpoint("65.0.0.3:10003"), Tuning::kMaxHops + 1}, + Endpoint{endpoint("0.0.0.0:2459"), 0}, + Endpoint{endpoint("0.0.0.0:2460"), 0}, + Endpoint{endpoint("10.0.0.1:10004"), 1}, + Endpoint{endpoint("65.0.0.5"), 1}, + Endpoint{endpoint("65.0.0.6:10006"), 1}, + Endpoint{endpoint("65.0.0.6:10006"), 2}}; + + logic_.preprocess(slot, endpoints); + + ASSERT_EQ(endpoints.size(), 2u); + EXPECT_EQ(endpoints.front().address, remote.atPort(2459)); + EXPECT_EQ(endpoints.front().hops, 1u); + EXPECT_EQ(endpoints.back().address, endpoint("65.0.0.6:10006")); + EXPECT_EQ(endpoints.back().hops, 2u); +} + +TEST_F(PeerFinderTest, on_endpoints_checks_neighbor_before_caching_it) +{ + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.inPeers = 1; + logic_.config(config); + + auto const local = endpoint("65.0.0.1:10001"); + auto const remote = endpoint("55.104.0.2:1025"); + auto const [slot, result] = logic_.newInboundSlot(local, remote); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + Endpoints const advertised{Endpoint{endpoint("0.0.0.0:2459"), 0}}; + logic_.onEndpoints(slot, advertised); + + ASSERT_EQ(checker_.asyncConnects.size(), 1u); + EXPECT_EQ(checker_.asyncConnects.front(), remote.atPort(2459)); + EXPECT_EQ(slot->listeningPort(), std::optional{2459}); + EXPECT_TRUE(slot->checked); + EXPECT_TRUE(slot->canAccept); + EXPECT_TRUE(logic_.livecache.empty()); + + clock_.advance(Tuning::kSecondsPerMessage); + logic_.onEndpoints(slot, advertised); + EXPECT_EQ(logic_.livecache.size(), 1u); + EXPECT_EQ(logic_.bootcache.size(), 1u); + + logic_.onEndpoints(slot, Endpoints{Endpoint{endpoint("65.0.0.9:10009"), 1}}); + EXPECT_EQ(logic_.livecache.size(), 1u); + + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, on_endpoints_skips_failed_neighbor_connectivity_checks) +{ + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.inPeers = 1; + logic_.config(config); + + checker_.nextError = boost::asio::error::host_unreachable; + auto const local = endpoint("65.0.0.1:10001"); + auto const remote = endpoint("55.104.0.3:1025"); + auto const [slot, result] = logic_.newInboundSlot(local, remote); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + Endpoints const advertised{Endpoint{endpoint("0.0.0.0:2459"), 0}}; + logic_.onEndpoints(slot, advertised); + EXPECT_TRUE(slot->checked); + EXPECT_FALSE(slot->canAccept); + + clock_.advance(Tuning::kSecondsPerMessage); + logic_.onEndpoints(slot, advertised); + EXPECT_TRUE(logic_.livecache.empty()); + + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, on_endpoints_waits_for_pending_connectivity_check) +{ + Config config; + config.autoConnect = false; + config.listeningPort = 1024; + config.ipLimit = 2; + config.inPeers = 1; + logic_.config(config); + + checker_.completeAsync = false; + auto const local = endpoint("65.0.0.1:10001"); + auto const remote = endpoint("55.104.0.4:1025"); + auto const [slot, result] = logic_.newInboundSlot(local, remote); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + Endpoints const advertised{Endpoint{endpoint("0.0.0.0:2459"), 0}}; + logic_.onEndpoints(slot, advertised); + EXPECT_TRUE(slot->connectivityCheckInProgress); + + clock_.advance(Tuning::kSecondsPerMessage); + logic_.onEndpoints(slot, advertised); + EXPECT_EQ(checker_.asyncConnects.size(), 1u); + EXPECT_TRUE(logic_.livecache.empty()); + + checker_.completeAsync = true; + logic_.checkComplete(remote, remote.atPort(2459), boost::asio::error::operation_aborted); + slot->connectivityCheckInProgress = false; + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, builds_endpoint_messages_and_redirects_from_livecache) +{ + Config config; + config.autoConnect = false; + config.wantIncoming = true; + config.listeningPort = 2459; + config.inPeers = 2; + config.outPeers = 2; + config.ipLimit = 2; + logic_.config(config); + + auto const remote = endpoint("55.104.0.5:1025"); + auto const live = endpoint("65.0.0.10:10010"); + logic_.livecache.insert(Endpoint{live, 1}); + + auto const [slot, result] = logic_.newOutboundSlot(remote); + ASSERT_NE(slot, nullptr); + EXPECT_EQ(result, Result::Success); + ASSERT_TRUE(logic_.onConnected(slot, endpoint("65.0.0.1:10001"))); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + ASSERT_EQ(logic_.activate(slot, publicKey, false), Result::Success); + + auto const messages = logic_.buildEndpointsForPeers(); + ASSERT_EQ(messages.size(), 1u); + auto const& sent = messages.front().second; + EXPECT_TRUE(std::ranges::any_of(sent, [](Endpoint const& ep) { return ep.hops == 0; })); + EXPECT_TRUE( + std::ranges::any_of(sent, [&live](Endpoint const& ep) { return ep.address == live; })); + EXPECT_TRUE(logic_.buildEndpointsForPeers().empty()); + + auto const redirects = logic_.redirect(slot); + EXPECT_FALSE(redirects.empty()); + + logic_.onClosed(slot); +} + +TEST_F(PeerFinderTest, autoconnect_uses_livecache_then_bootcache) +{ + Config config; + config.autoConnect = true; + config.wantIncoming = false; + config.outPeers = 1; + config.inPeers = 0; + config.ipLimit = 1; + logic_.config(config); + + auto const live = endpoint("65.0.0.11:10011"); + logic_.livecache.insert(Endpoint{live, 1}); + auto const liveAddresses = logic_.autoconnect(); + ASSERT_EQ(liveAddresses.size(), 1u); + EXPECT_EQ(liveAddresses.front(), live); + + auto const boot = endpoint("65.0.0.12:10012"); + EXPECT_TRUE(logic_.bootcache.insertStatic(boot)); + auto const bootAddresses = logic_.autoconnect(); + ASSERT_EQ(bootAddresses.size(), 1u); + EXPECT_EQ(bootAddresses.front(), boot); +} + +TEST_F(PeerFinderTest, sources_redirects_status_and_validation_paths_are_exercised) +{ + auto const source = std::make_shared("static"); + source->resultsToFetch.addresses = {endpoint("65.0.0.13:10013")}; + logic_.addStaticSource(source); + EXPECT_EQ(source->fetchCount, 1); + EXPECT_EQ(logic_.bootcache.size(), 1u); + + auto const failing = std::make_shared("failing"); + failing->resultsToFetch.error = boost::asio::error::host_unreachable; + logic_.fetch(failing); + EXPECT_EQ(failing->fetchCount, 1); + + auto const dynamic = std::make_shared("dynamic"); + logic_.addSource(dynamic); + ASSERT_EQ(logic_.sources.size(), 1u); + EXPECT_EQ(logic_.sources.front(), dynamic); + + std::vector redirects{ + {boost::asio::ip::make_address("65.0.0.14"), 10014}, + {boost::asio::ip::make_address("65.0.0.15"), 10015}}; + logic_.onRedirects(redirects.begin(), redirects.end(), redirects.front()); + EXPECT_EQ(logic_.bootcache.size(), 3u); + + EXPECT_FALSE(logic_.isValidAddress(endpoint("0.0.0.0:10016"))); + EXPECT_FALSE(logic_.isValidAddress(endpoint("10.0.0.1:10017"))); + EXPECT_FALSE(logic_.isValidAddress(endpoint("65.0.0.16"))); + EXPECT_TRUE(logic_.isValidAddress(endpoint("65.0.0.16:10016"))); + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + logic_.onWrite(map); + } + EXPECT_TRUE(stream.top().isMember("peers")); + EXPECT_TRUE(stream.top().isMember("counts")); + EXPECT_TRUE(stream.top().isMember("config")); + EXPECT_TRUE(stream.top().isMember("livecache")); + EXPECT_TRUE(stream.top().isMember("bootcache")); + + DefaultCancelSource defaultCancel; + Source::Results results; + EXPECT_TRUE(results.addresses.empty()); + defaultCancel.cancel(); + defaultCancel.fetch(results, journal()); + + logic_.fetchSource = dynamic; + logic_.stop(); + EXPECT_TRUE(logic_.stopping); + EXPECT_EQ(dynamic->cancelCount, 1); + + auto const ignored = std::make_shared("ignored"); + logic_.fetch(ignored); + EXPECT_EQ(ignored->fetchCount, 0); + + logic_.checkComplete( + endpoint("65.0.0.18:10018"), endpoint("65.0.0.19:10019"), boost::system::error_code{}); +} + +TEST(PeerFinderBootcache, loads_unique_entries_and_clears_cache) +{ + CapturingStore store; + TestStopwatch clock; + auto const ep1 = endpoint("65.0.0.1:10001"); + auto const ep2 = endpoint("65.0.0.2:10002"); + store.entriesToLoad = {storeEntry(ep1, 3), storeEntry(ep2, -2), storeEntry(ep1, 4)}; + + Bootcache cache(store, clock, journal()); + cache.load(); + + EXPECT_FALSE(cache.empty()); + EXPECT_EQ(cache.size(), 2u); + EXPECT_EQ(*cache.begin(), ep1); + EXPECT_EQ(*cache.cbegin(), ep1); + EXPECT_NE(cache.begin(), cache.end()); + EXPECT_NE(cache.cbegin(), cache.cend()); + + cache.clear(); + EXPECT_TRUE(cache.empty()); + EXPECT_EQ(cache.begin(), cache.end()); +} + +TEST(PeerFinderBootcache, records_connection_outcomes_and_persists_pending_updates) +{ + CapturingStore store; + TestStopwatch clock; + auto const ep1 = endpoint("65.0.0.1:10001"); + auto const ep2 = endpoint("65.0.0.2:10002"); + auto const ep3 = endpoint("65.0.0.3:10003"); + auto const ep4 = endpoint("65.0.0.4:10004"); + + { + Bootcache cache(store, clock, journal()); + + EXPECT_TRUE(cache.insert(ep1)); + EXPECT_FALSE(cache.insert(ep1)); + + cache.onSuccess(ep1); + EXPECT_TRUE(cache.insertStatic(ep1)); + EXPECT_FALSE(cache.insertStatic(ep1)); + + EXPECT_TRUE(cache.insertStatic(ep2)); + cache.onSuccess(ep3); + cache.onFailure(ep3); + cache.onFailure(ep4); + + EXPECT_EQ(cache.size(), 4u); + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + cache.onWrite(map); + } + EXPECT_TRUE(stream.top().isMember("entries")); + EXPECT_EQ(stream.top()["entries"].size(), 4u); + } + + ASSERT_EQ(store.saves.size(), 1u); + auto const& saved = store.saves.front(); + ASSERT_EQ(saved.size(), 4u); + EXPECT_EQ(savedValence(saved, ep1), Bootcache::kStaticValence); + EXPECT_EQ(savedValence(saved, ep2), Bootcache::kStaticValence); + EXPECT_EQ(savedValence(saved, ep3), -1); + EXPECT_EQ(savedValence(saved, ep4), -1); +} + +TEST(PeerFinderBootcache, periodic_activity_saves_after_cooldown) +{ + using namespace std::chrono_literals; + + CapturingStore store; + TestStopwatch clock; + + { + Bootcache cache(store, clock, journal()); + EXPECT_TRUE(cache.insert(endpoint("65.0.0.1:10001"))); + + cache.periodicActivity(); + EXPECT_TRUE(store.saves.empty()); + + clock.advance(Tuning::kBootcacheCooldownTime + 1s); + cache.periodicActivity(); + ASSERT_EQ(store.saves.size(), 1u); + + cache.periodicActivity(); + EXPECT_EQ(store.saves.size(), 1u); + } + + EXPECT_EQ(store.saves.size(), 1u); +} + +TEST(PeerFinderBootcache, prunes_when_cache_exceeds_limit) +{ + CapturingStore store; + TestStopwatch clock; + Bootcache cache(store, clock, journal()); + + for (std::uint16_t i = 0; i <= Tuning::kBootcacheSize; ++i) + { + EXPECT_TRUE(cache.insert(endpoint( + "65.0." + std::to_string((i / 256) % 256) + "." + std::to_string(i % 256) + ":" + + std::to_string(10000 + i)))); + } + + EXPECT_LE(cache.size(), Tuning::kBootcacheSize); +} + +TEST(PeerFinderEndpoint, clamps_hops_to_overflow_bucket) +{ + auto const address = endpoint("65.0.0.1:10001"); + Endpoint const ep(address, Tuning::kMaxHops + 10); + + EXPECT_EQ(ep.address, address); + EXPECT_EQ(ep.hops, Tuning::kMaxHops + 1); +} + +TEST(PeerFinderSlotImp, tracks_state_and_recent_endpoints) +{ + using State = Slot::State; + using namespace std::chrono_literals; + + TestStopwatch clock; + auto const local = endpoint("65.0.0.1:10000"); + auto const remote = endpoint("65.0.0.2:10001"); + SlotImp inbound(local, remote, true, clock); + + EXPECT_TRUE(inbound.inbound()); + EXPECT_TRUE(inbound.fixed()); + EXPECT_FALSE(inbound.reserved()); + EXPECT_EQ(inbound.state(), State::Accept); + EXPECT_EQ(inbound.remoteEndpoint(), remote); + EXPECT_EQ(inbound.localEndpoint(), std::optional{local}); + EXPECT_FALSE(inbound.publicKey()); + EXPECT_FALSE(inbound.listeningPort()); + EXPECT_FALSE(inbound.checked); + EXPECT_FALSE(inbound.canAccept); + EXPECT_FALSE(inbound.connectivityCheckInProgress); + + auto const newLocal = endpoint("65.0.0.3:10002"); + auto const newRemote = endpoint("65.0.0.4:10003"); + PublicKey const publicKey(randomKeyPair(KeyType::Secp256k1).first); + + inbound.localEndpoint(newLocal); + inbound.remoteEndpoint(newRemote); + inbound.publicKey(publicKey); + inbound.reserved(true); + inbound.setListeningPort(2459); + + EXPECT_EQ(inbound.localEndpoint(), std::optional{newLocal}); + EXPECT_EQ(inbound.remoteEndpoint(), newRemote); + EXPECT_EQ(inbound.publicKey(), std::optional{publicKey}); + EXPECT_TRUE(inbound.reserved()); + EXPECT_EQ(inbound.listeningPort(), std::optional{2459}); + EXPECT_FALSE(inbound.prefix().empty()); + + inbound.state(State::Closing); + EXPECT_EQ(inbound.state(), State::Closing); + + SlotImp outbound(remote, false, clock); + EXPECT_FALSE(outbound.inbound()); + EXPECT_FALSE(outbound.fixed()); + EXPECT_EQ(outbound.state(), State::Connect); + EXPECT_TRUE(outbound.checked); + EXPECT_TRUE(outbound.canAccept); + + outbound.state(State::Connected); + outbound.activate(clock.now()); + EXPECT_EQ(outbound.state(), State::Active); + EXPECT_EQ(outbound.whenAcceptEndpoints, clock.now()); + + auto const recent = endpoint("65.0.0.5:10004"); + EXPECT_FALSE(outbound.recent.filter(recent, 2)); + + outbound.recent.insert(recent, 2); + EXPECT_TRUE(outbound.recent.filter(recent, 2)); + EXPECT_TRUE(outbound.recent.filter(recent, 3)); + EXPECT_FALSE(outbound.recent.filter(recent, 1)); + + outbound.recent.insert(recent, 4); + EXPECT_FALSE(outbound.recent.filter(recent, 1)); + + outbound.recent.insert(recent, 1); + EXPECT_TRUE(outbound.recent.filter(recent, 1)); + EXPECT_FALSE(outbound.recent.filter(recent, 0)); + + clock.advance(Tuning::kLiveCacheSecondsToLive + 1s); + outbound.expire(); + EXPECT_FALSE(outbound.recent.filter(recent, 1)); +} + +TEST(PeerFinderConfig, writes_property_stream_and_compares_verify_endpoints) +{ + Config config; + config.maxPeers = 42; + config.outPeers = 12; + config.inPeers = 30; + config.peerPrivate = false; + config.wantIncoming = true; + config.autoConnect = false; + config.listeningPort = 2459; + config.features = "feature"; + config.ipLimit = 4; + config.verifyEndpoints = false; + + JsonPropertyStream stream; + { + beast::PropertyStream::Map map(stream); + config.onWrite(map); + } + + auto const& json = stream.top(); + EXPECT_EQ(json["max_peers"].asUInt(), config.maxPeers); + EXPECT_EQ(json["out_peers"].asUInt(), config.outPeers); + EXPECT_TRUE(json.isMember("want_incoming")); + EXPECT_TRUE(json.isMember("auto_connect")); + EXPECT_EQ(json["port"].asUInt(), config.listeningPort); + EXPECT_EQ(json["features"].asString(), config.features); + EXPECT_EQ(json["ip_limit"].asInt(), config.ipLimit); + EXPECT_TRUE(json.isMember("verify_endpoints")); + + Config same = config; + EXPECT_EQ(config, same); + same.verifyEndpoints = true; + EXPECT_NE(config, same); +} + +TEST(PeerFinderConfig, validator_and_standalone_settings_disable_auto_connect) +{ + PeerLimitConfig const limits{.maxPeers = 50, .inPeers = {}, .outPeers = {}}; + + Config const config = Config::makeConfig(false, true, limits, 2459, true, 7, false); + + EXPECT_TRUE(config.peerPrivate); + EXPECT_FALSE(config.autoConnect); + EXPECT_FALSE(config.verifyEndpoints); + EXPECT_EQ(config.ipLimit, 7); +} + +TEST(PeerFinderConfig, calculates_outbound_peers_and_clamps_ip_limits) +{ + Config config; + config.maxPeers = 1; + EXPECT_EQ(config.calcOutPeers(), Tuning::kMinOutCount); + + config.maxPeers = 100; + EXPECT_EQ(config.calcOutPeers(), 15u); + + config.inPeers = 1; + config.ipLimit = 0; + config.applyTuning(); + EXPECT_EQ(config.ipLimit, 1); + + Config explicitLimit; + explicitLimit.inPeers = 8; + explicitLimit.ipLimit = 99; + explicitLimit.applyTuning(); + EXPECT_EQ(explicitLimit.ipLimit, 4); + + Config largeInbound; + largeInbound.inPeers = 200; + largeInbound.ipLimit = 0; + largeInbound.applyTuning(); + EXPECT_EQ(largeInbound.ipLimit, 7); +} + +TEST(PeerFinderConfig, applies_legacy_and_explicit_peer_limits) +{ + struct ConfigCase + { + std::string name; + std::optional maxPeers; + std::optional maxIn; + std::optional maxOut; + std::uint16_t port; + std::uint16_t expectedOut; + std::uint16_t expectedIn; + std::uint16_t expectedIpLimit; + }; + + std::vector const cases{ + {.name = "legacy no config", + .maxPeers = {}, + .maxIn = {}, + .maxOut = {}, + .port = 4000, + .expectedOut = 10, + .expectedIn = 11, + .expectedIpLimit = 2}, + {.name = "legacy max_peers 0", + .maxPeers = 0, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 11, + .expectedIpLimit = 2}, + {.name = "legacy max_peers 5", + .maxPeers = 5, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 0, + .expectedIpLimit = 1}, + {.name = "legacy max_peers 20", + .maxPeers = 20, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 10, + .expectedIpLimit = 2}, + {.name = "legacy max_peers 100", + .maxPeers = 100, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 15, + .expectedIn = 85, + .expectedIpLimit = 6}, + {.name = "legacy max_peers 20, private", + .maxPeers = 20, + .maxIn = 100, + .maxOut = 10, + .port = 0, + .expectedOut = 20, + .expectedIn = 0, + .expectedIpLimit = 1}, + {.name = "new in 100/out 10", + .maxPeers = {}, + .maxIn = 100, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 100, + .expectedIpLimit = 6}, + {.name = "new in 0/out 10", + .maxPeers = {}, + .maxIn = 0, + .maxOut = 10, + .port = 4000, + .expectedOut = 10, + .expectedIn = 0, + .expectedIpLimit = 1}, + {.name = "new in 100/out 10, private", + .maxPeers = {}, + .maxIn = 100, + .maxOut = 10, + .port = 0, + .expectedOut = 10, + .expectedIn = 0, + .expectedIpLimit = 6}}; + + for (auto const& testCase : cases) + { + SCOPED_TRACE(testCase.name); + + PeerLimitConfig const limits{ + .maxPeers = testCase.maxPeers, .inPeers = testCase.maxIn, .outPeers = testCase.maxOut}; + + Config const config = + Config::makeConfig(false, false, limits, testCase.port, false, 0, true); + + Counts counts; + counts.onConfig(config); + EXPECT_EQ(counts.outMax(), testCase.expectedOut); + EXPECT_EQ(counts.inMax(), testCase.expectedIn); + EXPECT_EQ(config.ipLimit, testCase.expectedIpLimit); + + NiceMock store; + allowEmptyStore(store); + NiceMock checker; + TestStopwatch clock; + Logic> logic(clock, store, checker, journal()); + logic.config(config); + + EXPECT_EQ(logic.config(), config); + } +} + +TEST(PeerFinderConfig, rejects_incomplete_or_out_of_range_peer_limits) +{ + std::vector const configs{ + {.maxPeers = {}, .inPeers = 100, .outPeers = {}}, + {.maxPeers = {}, .inPeers = {}, .outPeers = 100}, + {.maxPeers = {}, .inPeers = 100, .outPeers = 5}, + {.maxPeers = {}, .inPeers = 1001, .outPeers = 10}, + {.maxPeers = {}, .inPeers = 10, .outPeers = 1001}}; + + for (auto const& limits : configs) + { + EXPECT_THROW( + Config::makeConfig(false, false, limits, 4000, false, 0, true), std::exception); + } +} + +} // namespace +} // namespace xrpl::PeerFinder diff --git a/src/tests/libxrpl/shamap/common.h b/src/tests/libxrpl/shamap/common.h index 5b44f2b251..91401d2973 100644 --- a/src/tests/libxrpl/shamap/common.h +++ b/src/tests/libxrpl/shamap/common.h @@ -24,13 +24,13 @@ namespace xrpl::tests { class TestNodeFamily : public Family { private: - std::unique_ptr db_; + std::unique_ptr db_; std::shared_ptr fbCache_; std::shared_ptr tnCache_; TestStopwatch clock_; - NodeStore::DummyScheduler scheduler_; + node_store::DummyScheduler scheduler_; beast::Journal const j_; @@ -49,17 +49,17 @@ public: Section testSection; testSection.set(Keys::kType, "memory"); testSection.set(Keys::kPath, "SHAMap_test"); - db_ = NodeStore::Manager::instance().makeDatabase( + db_ = node_store::Manager::instance().makeDatabase( megabytes(4), scheduler_, 1, testSection, j); } - NodeStore::Database& + node_store::Database& db() override { return *db_; } - [[nodiscard]] NodeStore::Database const& + [[nodiscard]] node_store::Database const& db() const override { return *db_; diff --git a/src/xrpld/app/consensus/RCLConsensus.cpp b/src/xrpld/app/consensus/RCLConsensus.cpp index 4abf77f578..20421ab916 100644 --- a/src/xrpld/app/consensus/RCLConsensus.cpp +++ b/src/xrpld/app/consensus/RCLConsensus.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -17,8 +16,6 @@ #include #include #include -#include -#include #include #include @@ -32,6 +29,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -389,7 +389,7 @@ RCLConsensus::Adaptor::onClose( if (!wrongLCL) { LedgerIndex const seq = prevLedger->header().seq + 1; - RCLCensorshipDetector::TxIDSeqVec proposed; + CensorshipDetector::TxIDSeqVec proposed; initialSet->visitLeaves( [&proposed, seq](boost::intrusive_ptr const& item) { diff --git a/src/xrpld/app/consensus/RCLConsensus.h b/src/xrpld/app/consensus/RCLConsensus.h index 4ffe18a7a8..4c07e7e646 100644 --- a/src/xrpld/app/consensus/RCLConsensus.h +++ b/src/xrpld/app/consensus/RCLConsensus.h @@ -1,20 +1,20 @@ #pragma once -#include #include #include #include #include #include #include -#include -#include -#include #include #include #include #include +#include +#include +#include +#include #include #include #include @@ -83,7 +83,7 @@ class RCLConsensus std::atomic prevRoundTime_{std::chrono::milliseconds{0}}; std::atomic mode_{ConsensusMode::Observing}; - RCLCensorshipDetector censorshipDetector_; + CensorshipDetector censorshipDetector_; NegativeUNLVote nUnlVote_; public: diff --git a/src/xrpld/app/consensus/RCLCxPeerPos.h b/src/xrpld/app/consensus/RCLCxPeerPos.h index 050bdf6d36..078556dea8 100644 --- a/src/xrpld/app/consensus/RCLCxPeerPos.h +++ b/src/xrpld/app/consensus/RCLCxPeerPos.h @@ -1,11 +1,10 @@ #pragma once -#include - #include #include #include #include +#include #include #include #include diff --git a/src/xrpld/app/consensus/RCLValidations.cpp b/src/xrpld/app/consensus/RCLValidations.cpp index 9d40e60b00..c587a04cf0 100644 --- a/src/xrpld/app/consensus/RCLValidations.cpp +++ b/src/xrpld/app/consensus/RCLValidations.cpp @@ -5,12 +5,12 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include diff --git a/src/xrpld/app/consensus/RCLValidations.h b/src/xrpld/app/consensus/RCLValidations.h index 7eadaf0dff..963b6c150e 100644 --- a/src/xrpld/app/consensus/RCLValidations.h +++ b/src/xrpld/app/consensus/RCLValidations.h @@ -1,10 +1,9 @@ #pragma once -#include - #include #include #include +#include #include #include #include diff --git a/src/xrpld/app/ledger/AccountStateSF.h b/src/xrpld/app/ledger/AccountStateSF.h index f5117db4d4..5c1d260c9a 100644 --- a/src/xrpld/app/ledger/AccountStateSF.h +++ b/src/xrpld/app/ledger/AccountStateSF.h @@ -18,7 +18,7 @@ namespace xrpl { class AccountStateSF : public SHAMapSyncFilter { public: - AccountStateSF(NodeStore::Database& db, AbstractFetchPackContainer& fp) : db_(db), fp_(fp) + AccountStateSF(node_store::Database& db, AbstractFetchPackContainer& fp) : db_(db), fp_(fp) { } @@ -34,7 +34,7 @@ public: getNode(SHAMapHash const& nodeHash) const override; private: - NodeStore::Database& db_; + node_store::Database& db_; AbstractFetchPackContainer& fp_; }; diff --git a/src/xrpld/app/ledger/InboundLedger.h b/src/xrpld/app/ledger/InboundLedger.h index d8a9ddf46b..31ca4169ce 100644 --- a/src/xrpld/app/ledger/InboundLedger.h +++ b/src/xrpld/app/ledger/InboundLedger.h @@ -136,7 +136,7 @@ private: addPeers(); void - tryDB(NodeStore::Database& srcDB); + tryDB(node_store::Database& srcDB); void done(); diff --git a/src/xrpld/app/ledger/TransactionStateSF.h b/src/xrpld/app/ledger/TransactionStateSF.h index a3f7e7f55a..b8c1b2c835 100644 --- a/src/xrpld/app/ledger/TransactionStateSF.h +++ b/src/xrpld/app/ledger/TransactionStateSF.h @@ -18,7 +18,7 @@ namespace xrpl { class TransactionStateSF : public SHAMapSyncFilter { public: - TransactionStateSF(NodeStore::Database& db, AbstractFetchPackContainer& fp) : db_(db), fp_(fp) + TransactionStateSF(node_store::Database& db, AbstractFetchPackContainer& fp) : db_(db), fp_(fp) { } @@ -34,7 +34,7 @@ public: getNode(SHAMapHash const& nodeHash) const override; private: - NodeStore::Database& db_; + node_store::Database& db_; AbstractFetchPackContainer& fp_; }; diff --git a/src/xrpld/app/ledger/detail/InboundLedger.cpp b/src/xrpld/app/ledger/detail/InboundLedger.cpp index 627a5d574f..55a2a9d283 100644 --- a/src/xrpld/app/ledger/detail/InboundLedger.cpp +++ b/src/xrpld/app/ledger/detail/InboundLedger.cpp @@ -224,7 +224,7 @@ InboundLedger::neededStateHashes(int max, SHAMapSyncFilter const* filter) const // See how much of the ledger data is stored locally // Data found in a fetch pack will be stored void -InboundLedger::tryDB(NodeStore::Database& srcDB) +InboundLedger::tryDB(node_store::Database& srcDB) { if (!haveHeader_) { diff --git a/src/xrpld/app/ledger/detail/LedgerMaster.cpp b/src/xrpld/app/ledger/detail/LedgerMaster.cpp index 2bd83b0f18..9a0335fc3c 100644 --- a/src/xrpld/app/ledger/detail/LedgerMaster.cpp +++ b/src/xrpld/app/ledger/detail/LedgerMaster.cpp @@ -653,7 +653,7 @@ LedgerMaster::tryFill(std::shared_ptr ledger) std::uint32_t minHas = seq; std::uint32_t maxHas = seq; - NodeStore::Database& nodeStore{app_.getNodeStore()}; + node_store::Database& nodeStore{app_.getNodeStore()}; while (!app_.getJobQueue().isStopping() && seq > 0) { { diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index abe39212be..c8de63f914 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -236,7 +236,7 @@ public: std::unique_ptr resourceManager_; - std::unique_ptr nodeStore_; + std::unique_ptr nodeStore_; NodeFamily nodeFamily_; std::unique_ptr orderBookDB_; std::unique_ptr pathRequestManager_; @@ -674,7 +674,7 @@ public: return tempNodeCache_; } - NodeStore::Database& + node_store::Database& getNodeStore() override { return *nodeStore_; @@ -879,9 +879,9 @@ public: if (config_->doImport) { auto j = logs_->journal("NodeObject"); - NodeStore::DummyScheduler dummyScheduler; - std::unique_ptr source = - NodeStore::Manager::instance().makeDatabase( + node_store::DummyScheduler dummyScheduler; + std::unique_ptr source = + node_store::Manager::instance().makeDatabase( megabytes(config_->getValueFor(SizedItem::BurstSize, std::nullopt)), dummyScheduler, 0, diff --git a/src/xrpld/app/main/NodeStoreScheduler.cpp b/src/xrpld/app/main/NodeStoreScheduler.cpp index 7892503f90..c3ac5d78cf 100644 --- a/src/xrpld/app/main/NodeStoreScheduler.cpp +++ b/src/xrpld/app/main/NodeStoreScheduler.cpp @@ -12,7 +12,7 @@ NodeStoreScheduler::NodeStoreScheduler(JobQueue& jobQueue) : jobQueue_(jobQueue) } void -NodeStoreScheduler::scheduleTask(NodeStore::Task& task) +NodeStoreScheduler::scheduleTask(node_store::Task& task) { if (jobQueue_.isStopped()) return; @@ -26,19 +26,19 @@ NodeStoreScheduler::scheduleTask(NodeStore::Task& task) } void -NodeStoreScheduler::onFetch(NodeStore::FetchReport const& report) +NodeStoreScheduler::onFetch(node_store::FetchReport const& report) { if (jobQueue_.isStopped()) return; jobQueue_.addLoadEvents( - report.fetchType == NodeStore::FetchType::Async ? JtNsAsyncRead : JtNsSyncRead, + report.fetchType == node_store::FetchType::Async ? JtNsAsyncRead : JtNsSyncRead, 1, report.elapsed); } void -NodeStoreScheduler::onBatchWrite(NodeStore::BatchWriteReport const& report) +NodeStoreScheduler::onBatchWrite(node_store::BatchWriteReport const& report) { if (jobQueue_.isStopped()) return; diff --git a/src/xrpld/app/main/NodeStoreScheduler.h b/src/xrpld/app/main/NodeStoreScheduler.h index 8bfd1607ae..09a48d5be1 100644 --- a/src/xrpld/app/main/NodeStoreScheduler.h +++ b/src/xrpld/app/main/NodeStoreScheduler.h @@ -7,19 +7,19 @@ namespace xrpl { /** - * A NodeStore::Scheduler which uses the JobQueue. + * A node_store::Scheduler which uses the JobQueue. */ -class NodeStoreScheduler : public NodeStore::Scheduler +class NodeStoreScheduler : public node_store::Scheduler { public: explicit NodeStoreScheduler(JobQueue& jobQueue); void - scheduleTask(NodeStore::Task& task) override; + scheduleTask(node_store::Task& task) override; void - onFetch(NodeStore::FetchReport const& report) override; + onFetch(node_store::FetchReport const& report) override; void - onBatchWrite(NodeStore::BatchWriteReport const& report) override; + onBatchWrite(node_store::BatchWriteReport const& report) override; private: JobQueue& jobQueue_; diff --git a/src/xrpld/app/misc/NetworkOPs.cpp b/src/xrpld/app/misc/NetworkOPs.cpp index 4b0091dff6..47cbebb901 100644 --- a/src/xrpld/app/misc/NetworkOPs.cpp +++ b/src/xrpld/app/misc/NetworkOPs.cpp @@ -21,8 +21,6 @@ #include #include #include -#include -#include #include #include #include @@ -54,6 +52,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/xrpld/app/misc/SHAMapStore.h b/src/xrpld/app/misc/SHAMapStore.h index df696c685f..eeb04df53d 100644 --- a/src/xrpld/app/misc/SHAMapStore.h +++ b/src/xrpld/app/misc/SHAMapStore.h @@ -43,7 +43,7 @@ public: [[nodiscard]] virtual std::uint32_t clampFetchDepth(std::uint32_t fetchDepth) const = 0; - virtual std::unique_ptr + virtual std::unique_ptr makeNodeStore(int readThreads) = 0; /** @@ -101,5 +101,5 @@ public: //------------------------------------------------------------------------------ std::unique_ptr -makeSHAMapStore(Application& app, NodeStore::Scheduler& scheduler, beast::Journal journal); +makeSHAMapStore(Application& app, node_store::Scheduler& scheduler, beast::Journal journal); } // namespace xrpl diff --git a/src/xrpld/app/misc/SHAMapStoreImp.cpp b/src/xrpld/app/misc/SHAMapStoreImp.cpp index 9b5f412fc5..e41837d206 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.cpp +++ b/src/xrpld/app/misc/SHAMapStoreImp.cpp @@ -94,7 +94,7 @@ SHAMapStoreImp::SavedStateDB::setLastRotated(LedgerIndex seq) SHAMapStoreImp::SHAMapStoreImp( Application& app, - NodeStore::Scheduler& scheduler, + node_store::Scheduler& scheduler, beast::Journal journal) : app_(app) , scheduler_(scheduler) @@ -165,7 +165,7 @@ SHAMapStoreImp::SHAMapStoreImp( } } -std::unique_ptr +std::unique_ptr SHAMapStoreImp::makeNodeStore(int readThreads) { auto nscfg = app_.config().section(Sections::kNodeDatabase); @@ -185,7 +185,7 @@ SHAMapStoreImp::makeNodeStore(int readThreads) std::to_string(app_.config().getValueFor(SizedItem::TreeCacheAge, std::nullopt))); } - std::unique_ptr db; + std::unique_ptr db; if (deleteInterval_ != 0u) { @@ -201,7 +201,7 @@ SHAMapStoreImp::makeNodeStore(int readThreads) // Create NodeStore with two backends to allow online deletion of // data - auto dbr = std::make_unique( + auto dbr = std::make_unique( scheduler_, readThreads, std::move(writableBackend), @@ -210,11 +210,11 @@ SHAMapStoreImp::makeNodeStore(int readThreads) app_.getJournal(kNodeStoreName)); fdRequired_ += dbr->fdRequired(); dbRotating_ = dbr.get(); - db.reset(dynamic_cast(dbr.release())); + db.reset(dynamic_cast(dbr.release())); } else { - db = NodeStore::Manager::instance().makeDatabase( + db = node_store::Manager::instance().makeDatabase( megabytes(app_.config().getValueFor(SizedItem::BurstSize, std::nullopt)), scheduler_, readThreads, @@ -257,7 +257,7 @@ SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node) { // Copy a single record from node to dbRotating_ auto obj = dbRotating_->fetchNodeObject( - node.getHash().asUInt256(), 0, NodeStore::FetchType::Synchronous, true); + node.getHash().asUInt256(), 0, node_store::FetchType::Synchronous, true); if (!obj) { XRPL_ASSERT(node.cowid() == 0, "SHAMapStoreImp::copyNode : rescued node must be clean"); @@ -374,7 +374,7 @@ SHAMapStoreImp::run() // exception) also clear the flag. struct RotationExposureGuard { - NodeStore::DatabaseRotating& db; + node_store::DatabaseRotating& db; ~RotationExposureGuard() { db.setRotationInFlight(false); @@ -516,7 +516,7 @@ SHAMapStoreImp::dbPaths() boost::filesystem::remove_all(p); } -std::unique_ptr +std::unique_ptr SHAMapStoreImp::makeBackendRotating(std::string path) { Section section{app_.config().section(Sections::kNodeDatabase)}; @@ -535,7 +535,7 @@ SHAMapStoreImp::makeBackendRotating(std::string path) } section.set(Keys::kPath, newPath.string()); - auto backend{NodeStore::Manager::instance().makeBackend( + auto backend{node_store::Manager::instance().makeBackend( section, megabytes(app_.config().getValueFor(SizedItem::BurstSize, std::nullopt)), scheduler_, @@ -702,7 +702,7 @@ SHAMapStoreImp::minimumOnline() const //------------------------------------------------------------------------------ std::unique_ptr -makeSHAMapStore(Application& app, NodeStore::Scheduler& scheduler, beast::Journal journal) +makeSHAMapStore(Application& app, node_store::Scheduler& scheduler, beast::Journal journal) { return std::make_unique(app, scheduler, journal); } diff --git a/src/xrpld/app/misc/SHAMapStoreImp.h b/src/xrpld/app/misc/SHAMapStoreImp.h index a0ca59ecc8..8a1b7504b9 100644 --- a/src/xrpld/app/misc/SHAMapStoreImp.h +++ b/src/xrpld/app/misc/SHAMapStoreImp.h @@ -81,9 +81,9 @@ private: // minimum ledger to maintain online. std::atomic minimumOnline_; - NodeStore::Scheduler& scheduler_; + node_store::Scheduler& scheduler_; beast::Journal const journal_; - NodeStore::DatabaseRotating* dbRotating_ = nullptr; + node_store::DatabaseRotating* dbRotating_ = nullptr; SavedStateDB stateDb_; std::thread thread_; bool stop_ = false; @@ -119,7 +119,7 @@ private: static constexpr auto kNodeStoreName = "NodeStore"; public: - SHAMapStoreImp(Application& app, NodeStore::Scheduler& scheduler, beast::Journal journal); + SHAMapStoreImp(Application& app, node_store::Scheduler& scheduler, beast::Journal journal); std::uint32_t clampFetchDepth(std::uint32_t fetchDepth) const override @@ -127,7 +127,7 @@ public: return (deleteInterval_ != 0u) ? std::min(fetchDepth, deleteInterval_) : fetchDepth; } - std::unique_ptr + std::unique_ptr makeNodeStore(int readThreads) override; LedgerIndex @@ -180,7 +180,7 @@ private: void dbPaths(); - std::unique_ptr + std::unique_ptr makeBackendRotating(std::string path = std::string()); template @@ -191,7 +191,7 @@ private: for (auto const& key : cache.getKeys()) { - dbRotating_->fetchNodeObject(key, 0, NodeStore::FetchType::Synchronous, true); + dbRotating_->fetchNodeObject(key, 0, node_store::FetchType::Synchronous, true); if (!(++check % checkHealthInterval_) && healthWait() == HealthResult::Stopping) return true; } diff --git a/src/xrpld/app/rdb/PeerFinder.h b/src/xrpld/app/rdb/PeerFinder.h index 4f186ff7e2..5d916000a3 100644 --- a/src/xrpld/app/rdb/PeerFinder.h +++ b/src/xrpld/app/rdb/PeerFinder.h @@ -1,9 +1,8 @@ #pragma once -#include - #include #include +#include #include #include diff --git a/src/xrpld/app/rdb/detail/PeerFinder.cpp b/src/xrpld/app/rdb/detail/PeerFinder.cpp index abdcd1c61a..72a275c7cd 100644 --- a/src/xrpld/app/rdb/detail/PeerFinder.cpp +++ b/src/xrpld/app/rdb/detail/PeerFinder.cpp @@ -1,12 +1,11 @@ #include -#include - #include #include #include #include #include +#include #include #include // IWYU pragma: keep diff --git a/src/xrpld/overlay/detail/ConnectAttempt.cpp b/src/xrpld/overlay/detail/ConnectAttempt.cpp index 064b4ecd3e..0f0b3242de 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.cpp +++ b/src/xrpld/overlay/detail/ConnectAttempt.cpp @@ -7,8 +7,6 @@ #include #include #include -#include -#include #include #include @@ -16,6 +14,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/xrpld/overlay/detail/ConnectAttempt.h b/src/xrpld/overlay/detail/ConnectAttempt.h index d7836e3c84..f9ba33571f 100644 --- a/src/xrpld/overlay/detail/ConnectAttempt.h +++ b/src/xrpld/overlay/detail/ConnectAttempt.h @@ -3,12 +3,12 @@ #include #include #include -#include #include #include #include #include +#include #include #include diff --git a/src/xrpld/overlay/detail/OverlayImpl.cpp b/src/xrpld/overlay/detail/OverlayImpl.cpp index 6a6a6edace..f9972548d1 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.cpp +++ b/src/xrpld/overlay/detail/OverlayImpl.cpp @@ -10,8 +10,6 @@ #include #include #include -#include -#include #include #include #include @@ -39,6 +37,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -179,12 +180,13 @@ OverlayImpl::OverlayImpl( , journal_(app_.getJournal("Overlay")) , serverHandler_(serverHandler) , resourceManager_(resourceManager) + , store_(app_.getJournal("PeerFinder")) , peerFinder_( PeerFinder::makeManager( ioContext, stopwatch(), app_.getJournal("PeerFinder"), - config, + store_, collector)) , resolver_(resolver) , nextId_(1) @@ -201,6 +203,7 @@ OverlayImpl::OverlayImpl( return ret; }()) { + store_.open(config); beast::PropertyStream::Source::add(peerFinder_.get()); } @@ -505,7 +508,7 @@ OverlayImpl::remove(std::shared_ptr const& slot) void OverlayImpl::start() { - PeerFinder::Config const config = PeerFinder::Config::makeConfig( + PeerFinder::Config const config = PeerFinder::makeConfig( app_.config(), serverHandler_.setup().overlay.port(), app_.getValidationPublicKey().has_value(), diff --git a/src/xrpld/overlay/detail/OverlayImpl.h b/src/xrpld/overlay/detail/OverlayImpl.h index 092ac86a6d..cd2c7d630b 100644 --- a/src/xrpld/overlay/detail/OverlayImpl.h +++ b/src/xrpld/overlay/detail/OverlayImpl.h @@ -8,8 +8,7 @@ #include #include #include -#include -#include +#include #include #include @@ -24,6 +23,8 @@ #include #include #include +#include +#include #include #include #include @@ -109,6 +110,7 @@ private: beast::Journal const journal_; ServerHandler& serverHandler_; Resource::Manager& resourceManager_; + PeerFinder::StoreSqdb store_; std::unique_ptr peerFinder_; TrafficCount traffic_; hash_map, std::weak_ptr> peers_; diff --git a/src/xrpld/overlay/detail/PeerImp.cpp b/src/xrpld/overlay/detail/PeerImp.cpp index 8838970b5f..d7a9a9e449 100644 --- a/src/xrpld/overlay/detail/PeerImp.cpp +++ b/src/xrpld/overlay/detail/PeerImp.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -19,8 +18,6 @@ #include #include #include -#include -#include #include #include @@ -37,12 +34,15 @@ #include #include #include +#include #include #include #include #include #include #include +#include +#include #include #include #include diff --git a/src/xrpld/overlay/detail/PeerImp.h b/src/xrpld/overlay/detail/PeerImp.h index ea6eccd656..90f8a917f4 100644 --- a/src/xrpld/overlay/detail/PeerImp.h +++ b/src/xrpld/overlay/detail/PeerImp.h @@ -9,8 +9,6 @@ #include #include #include -#include -#include #include #include @@ -24,6 +22,8 @@ #include #include #include +#include +#include #include #include #include diff --git a/src/xrpld/peerfinder/PeerfinderManager.h b/src/xrpld/peerfinder/PeerfinderManager.h index 0530343641..f96ea31943 100644 --- a/src/xrpld/peerfinder/PeerfinderManager.h +++ b/src/xrpld/peerfinder/PeerfinderManager.h @@ -1,368 +1,19 @@ #pragma once #include -#include -#include -#include -#include -#include -#include +#include -#include - -#include -#include #include -#include -#include -#include -#include -#include namespace xrpl::PeerFinder { -using clock_type = beast::AbstractClock; - -/** - * Represents a set of addresses. - */ -using IPAddresses = std::vector; - -//------------------------------------------------------------------------------ - -/** - * PeerFinder configuration settings. - */ -struct Config -{ - /** - * The largest number of public peer slots to allow. - * This includes both inbound and outbound, but does not include - * fixed peers. - */ - std::size_t maxPeers{Tuning::kDefaultMaxPeers}; - - /** - * The number of automatic outbound connections to maintain. - * Outbound connections are only maintained if autoConnect - * is `true`. - */ - std::size_t outPeers; - - /** - * The number of automatic inbound connections to maintain. - * Inbound connections are only maintained if wantIncoming - * is `true`. - */ - std::size_t inPeers{0}; - - /** - * `true` if we want our IP address kept private. - */ - bool peerPrivate = true; - - /** - * `true` if we want to accept incoming connections. - */ - bool wantIncoming{true}; - - /** - * `true` if we want to establish connections automatically - */ - bool autoConnect{true}; - - /** - * The listening port number. - */ - std::uint16_t listeningPort{0}; - - /** - * The set of features we advertise. - */ - std::string features; - - /** - * Limit how many incoming connections we allow per IP - */ - int ipLimit{0}; - - /** - * `true` if we want to verify endpoints in TMEndpoints messages - */ - bool verifyEndpoints = true; - - //-------------------------------------------------------------------------- - - /** - * Create a configuration with default values. - */ - Config(); - - /** - * Returns a suitable value for outPeers according to the rules. - */ - [[nodiscard]] std::size_t - calcOutPeers() const; - - /** - * Adjusts the values so they follow the business rules. - */ - void - applyTuning(); - - /** - * Write the configuration into a property stream - */ - void - onWrite(beast::PropertyStream::Map& map) const; - - /** - * Make PeerFinder::Config from configuration parameters - * @param config server's configuration - * @param port server's listening port - * @param validationPublicKey true if validation public key is not empty - * @param ipLimit limit of incoming connections per IP - * @param verifyEndpoints `true` if we want to verify endpoints in - * TMEndpoints messages - * @return PeerFinder::Config - */ - static Config - makeConfig( - xrpl::Config const& config, - std::uint16_t port, - bool validationPublicKey, - int ipLimit, - bool verifyEndpoints); - - friend bool - operator==(Config const& lhs, Config const& rhs) = default; -}; - -//------------------------------------------------------------------------------ - -/** - * Describes a connectable peer address along with some metadata. - */ -struct Endpoint -{ - Endpoint() = default; - - Endpoint(beast::IP::Endpoint ep, std::uint32_t hops); - - std::uint32_t hops = 0; - beast::IP::Endpoint address; -}; - -inline bool -operator<(Endpoint const& lhs, Endpoint const& rhs) -{ - return lhs.address < rhs.address; -} - -/** - * A set of Endpoint used for connecting. - */ -using Endpoints = std::vector; - -//------------------------------------------------------------------------------ - -/** - * Possible results from activating a slot. - */ -enum class Result { InboundDisabled, DuplicatePeer, IpLimitExceeded, Full, Success }; - -/** - * @brief Converts a `Result` enum value to its string representation. - * - * This function provides a human-readable string for a given `Result` enum, - * which is useful for logging, debugging, or displaying status messages. - * - * @param result The `Result` enum value to convert. - * @return A `std::string_view` representing the enum value. Returns "unknown" - * if the enum value is not explicitly handled. - * - * @note This function returns a `std::string_view` for performance. - * A `std::string` would need to allocate memory on the heap and copy the - * string literal into it every time the function is called. - */ -inline std::string_view -to_string(Result result) noexcept -{ - switch (result) - { - case Result::InboundDisabled: - return "inbound disabled"; - case Result::DuplicatePeer: - return "peer already connected"; - case Result::IpLimitExceeded: - return "ip limit exceeded"; - case Result::Full: - return "slots full"; - case Result::Success: - return "success"; - } - - return "unknown"; -} - -/** - * Maintains a set of IP addresses used for getting into the network. - */ -class Manager : public beast::PropertyStream::Source -{ -protected: - Manager() noexcept; - -public: - /** - * Destroy the object. - * Any pending source fetch operations are aborted. - * There may be some listener calls made before the - * destructor returns. - */ - ~Manager() override = default; - - /** - * Set the configuration for the manager. - * The new settings will be applied asynchronously. - * Thread safety: - * Can be called from any threads at any time. - */ - virtual void - setConfig(Config const& config) = 0; - - /** - * Transition to the started state, synchronously. - */ - virtual void - start() = 0; - - /** - * Transition to the stopped state, synchronously. - */ - virtual void - stop() = 0; - - /** - * Returns the configuration for the manager. - */ - virtual Config - config() = 0; - - /** - * Add a peer that should always be connected. - * This is useful for maintaining a private cluster of peers. - * The string is the name as specified in the configuration - * file, along with the set of corresponding IP addresses. - */ - virtual void - addFixedPeer(std::string_view name, std::vector const& addresses) = 0; - - /** - * Add a set of strings as fallback IP::Endpoint sources. - * @param name A label used for diagnostics. - */ - virtual void - addFallbackStrings(std::string const& name, std::vector const& strings) = 0; - - /** - * Add a URL as a fallback location to obtain IP::Endpoint sources. - * @param name A label used for diagnostics. - */ - /* VFALCO NOTE Unimplemented - virtual void addFallbackURL (std::string const& name, - std::string const& url) = 0; - */ - - //-------------------------------------------------------------------------- - - /** - * Create a new inbound slot with the specified remote endpoint. - * If nullptr is returned, then the slot could not be assigned. - * Usually this is because of a detected self-connection. - */ - virtual std::pair, Result> - newInboundSlot( - beast::IP::Endpoint const& localEndpoint, - beast::IP::Endpoint const& remoteEndpoint) = 0; - - /** - * Create a new outbound slot with the specified remote endpoint. - * If nullptr is returned, then the slot could not be assigned. - * Usually this is because of a duplicate connection. - */ - virtual std::pair, Result> - newOutboundSlot(beast::IP::Endpoint const& remoteEndpoint) = 0; - - /** - * Called when mtENDPOINTS is received. - */ - virtual void - onEndpoints(std::shared_ptr const& slot, Endpoints const& endpoints) = 0; - - /** - * Called when the slot is closed. - * This always happens when the socket is closed, unless the socket - * was canceled. - */ - virtual void - onClosed(std::shared_ptr const& slot) = 0; - - /** - * Called when an outbound connection is deemed to have failed - */ - virtual void - onFailure(std::shared_ptr const& slot) = 0; - - /** - * Called when we received redirect IPs from a busy peer. - */ - virtual void - onRedirects( - boost::asio::ip::tcp::endpoint const& remoteAddress, - std::vector const& eps) = 0; - - //-------------------------------------------------------------------------- - - /** - * Called when an outbound connection attempt succeeds. - * The local endpoint must be valid. If the caller receives an error - * when retrieving the local endpoint from the socket, it should - * proceed as if the connection attempt failed by calling on_closed - * instead of on_connected. - * @return `true` if the connection should be kept - */ - virtual bool - onConnected(std::shared_ptr const& slot, beast::IP::Endpoint const& localEndpoint) = 0; - - /** - * Request an active slot type. - */ - virtual Result - activate(std::shared_ptr const& slot, PublicKey const& key, bool reserved) = 0; - - /** - * Returns a set of endpoints suitable for redirection. - */ - virtual std::vector - redirect(std::shared_ptr const& slot) = 0; - - /** - * Return a set of addresses we should connect to. - */ - virtual std::vector - autoconnect() = 0; - - virtual std::vector, std::vector>> - buildEndpointsForPeers() = 0; - - /** - * Perform periodic activity. - * This should be called once per second. - */ - virtual void - oncePerSecond() = 0; -}; +Config +makeConfig( + xrpl::Config const& config, + std::uint16_t port, + bool validationPublicKey, + int ipLimit, + bool verifyEndpoints); } // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/PeerfinderConfig.cpp b/src/xrpld/peerfinder/detail/PeerfinderConfig.cpp index 5d276dc9c5..a893b969e0 100644 --- a/src/xrpld/peerfinder/detail/PeerfinderConfig.cpp +++ b/src/xrpld/peerfinder/detail/PeerfinderConfig.cpp @@ -1,124 +1,39 @@ #include #include -#include -#include +#include -#include -#include #include namespace xrpl::PeerFinder { -Config::Config() : outPeers(calcOutPeers()) - -{ -} - -std::size_t -Config::calcOutPeers() const -{ - return std::max( - ((maxPeers * Tuning::kOutPercent) + 50) / 100, std::size_t(Tuning::kMinOutCount)); -} - -void -Config::applyTuning() -{ - if (ipLimit == 0) - { - // Unless a limit is explicitly set, we allow between - // 2 and 5 connections from non RFC-1918 "private" - // IP addresses. - ipLimit = 2; - - if (inPeers > Tuning::kDefaultMaxPeers) - ipLimit += std::min(5, static_cast(inPeers / Tuning::kDefaultMaxPeers)); - } - - // We don't allow a single IP to consume all incoming slots, - // unless we only have one incoming slot available. - ipLimit = std::max(1, std::min(ipLimit, static_cast(inPeers / 2))); -} - -void -Config::onWrite(beast::PropertyStream::Map& map) const -{ - map["max_peers"] = maxPeers; - map["out_peers"] = outPeers; - map["want_incoming"] = wantIncoming; - map["auto_connect"] = autoConnect; - map["port"] = listeningPort; - map["features"] = features; - map["ip_limit"] = ipLimit; - map["verify_endpoints"] = verifyEndpoints; -} - Config -Config::makeConfig( +makeConfig( xrpl::Config const& cfg, std::uint16_t port, bool validationPublicKey, int ipLimit, bool verifyEndpoints) { - PeerFinder::Config config; - - config.peerPrivate = cfg.peerPrivate; - - // Servers with peer privacy don't want to allow incoming connections - config.wantIncoming = (!config.peerPrivate) && (port != 0); - + PeerLimitConfig limits; if ((cfg.peersOutMax == 0u) && (cfg.peersInMax == 0u)) { - if (cfg.peersMax != 0) - config.maxPeers = cfg.peersMax; - - config.maxPeers = std::max(config.maxPeers, Tuning::kMinOutCount); - config.outPeers = config.calcOutPeers(); - - // Calculate the number of outbound peers we want. If we dont want - // or can't accept incoming, this will simply be equal to maxPeers. - if (!config.wantIncoming) - config.outPeers = config.maxPeers; - - // Calculate the largest number of inbound connections we could - // take. - if (config.maxPeers >= config.outPeers) - { - config.inPeers = config.maxPeers - config.outPeers; - } - else - { - config.inPeers = 0; - } + limits.maxPeers = cfg.peersMax; } else { - config.outPeers = cfg.peersOutMax; - config.inPeers = cfg.peersInMax; - config.maxPeers = 0; + limits.inPeers = cfg.peersInMax; + limits.outPeers = cfg.peersOutMax; } - // This will cause servers configured as validators to request that - // peers they connect to never report their IP address. We set this - // after we set the 'wantIncoming' because we want a "soft" version - // of peer privacy unless the operator explicitly asks for it. - if (validationPublicKey) - config.peerPrivate = true; - - // if it's a private peer or we are running as standalone - // automatic connections would defeat the purpose. - config.autoConnect = !cfg.standalone() && !cfg.peerPrivate; - config.listeningPort = port; - config.features = ""; - config.ipLimit = ipLimit; - config.verifyEndpoints = verifyEndpoints; - - // Enforce business rules - config.applyTuning(); - - return config; + return Config::makeConfig( + cfg.peerPrivate, + cfg.standalone(), + limits, + port, + validationPublicKey, + ipLimit, + verifyEndpoints); } } // namespace xrpl::PeerFinder diff --git a/src/xrpld/peerfinder/detail/StoreSqdb.h b/src/xrpld/peerfinder/detail/StoreSqdb.h index b17d2fdc5b..b0973a42b3 100644 --- a/src/xrpld/peerfinder/detail/StoreSqdb.h +++ b/src/xrpld/peerfinder/detail/StoreSqdb.h @@ -1,11 +1,11 @@ #pragma once #include -#include #include #include #include +#include #include #include diff --git a/src/xrpld/peerfinder/detail/iosformat.h b/src/xrpld/peerfinder/detail/iosformat.h deleted file mode 100644 index 46c69ef602..0000000000 --- a/src/xrpld/peerfinder/detail/iosformat.h +++ /dev/null @@ -1,201 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace beast { - -// A collection of handy stream manipulators and -// functions to produce nice looking log output. - -/** - * Left justifies a field at the specified width. - */ -struct Leftw -{ - explicit Leftw(int width) : width(width) - { - } - int const width; - template - friend std::basic_ios& - operator<<(std::basic_ios& ios, Leftw const& p) - { - ios.setf(std::ios_base::left, std::ios_base::adjustfield); - ios.width(p.width); - return ios; - } -}; - -/** - * Produce a section heading and fill the rest of the line with dashes. - */ -template -std::basic_string -heading(std::basic_string title, int width = 80, CharT fill = CharT('-')) -{ - title.reserve(width); - title.push_back(CharT(' ')); - title.resize(width, fill); - return title; -} - -/** - * Produce a dashed line separator, with a specified or default size. - */ -struct Divider -{ - using CharT = char; - explicit Divider(int width = 80, CharT fill = CharT('-')) : width(width), fill(fill) - { - } - int const width; - CharT const fill; - template - friend std::basic_ostream& - operator<<(std::basic_ostream& os, Divider const& d) - { - os << std::basic_string(d.width, d.fill); - return os; - } -}; - -/** - * Creates a padded field with an optional fill character. - */ -struct Fpad -{ - explicit Fpad(int width, int pad = 0, char fill = ' ') : width(width + pad), fill(fill) - { - } - int const width; - char const fill; - template - friend std::basic_ostream& - operator<<(std::basic_ostream& os, Fpad const& f) - { - os << std::basic_string(f.width, f.fill); - return os; - } -}; - -//------------------------------------------------------------------------------ - -namespace detail { - -template -std::string -to_string(T const& t) -{ - std::stringstream ss; - ss << t; - return ss.str(); -} - -} // namespace detail - -/** - * Justifies a field at the specified width. - */ -/** @{ */ -template < - class CharT, - class Traits = std::char_traits, - class Allocator = std::allocator> -class FieldT -{ -public: - using string_t = std::basic_string; - FieldT(string_t const& text, int width, int pad, bool right) - : text(text), width(width), pad(pad), right(right) - { - } - string_t const text; - int const width; - int const pad; - bool const right; - template - friend std::basic_ostream& - operator<<(std::basic_ostream& os, FieldT const& f) - { - std::size_t const length(f.text.length()); - if (f.right) - { - if (length < f.width) - os << std::basic_string(f.width - length, CharT2(' ')); - os << f.text; - } - else - { - os << f.text; - if (length < f.width) - os << std::basic_string(f.width - length, CharT2(' ')); - } - if (f.pad != 0) - os << string_t(f.pad, CharT(' ')); - return os; - } -}; - -template -FieldT -field( - std::basic_string const& text, - int width = 8, - int pad = 0, - bool right = false) -{ - return FieldT(text, width, pad, right); -} - -template -FieldT -field(CharT const* text, int width = 8, int pad = 0, bool right = false) -{ - return FieldT, std::allocator>( - std::basic_string, std::allocator>(text), - width, - pad, - right); -} - -template -FieldT -field(T const& t, int width = 8, int pad = 0, bool right = false) -{ - std::string const text(detail::to_string(t)); - return field(text, width, pad, right); -} - -template -FieldT -rField(std::basic_string const& text, int width = 8, int pad = 0) -{ - return FieldT(text, width, pad, true); -} - -template -FieldT -rField(CharT const* text, int width = 8, int pad = 0) -{ - return FieldT, std::allocator>( - std::basic_string, std::allocator>(text), - width, - pad, - true); -} - -template -FieldT -rField(T const& t, int width = 8, int pad = 0) -{ - std::string const text(detail::to_string(t)); - return field(text, width, pad, true); -} -/** @} */ - -} // namespace beast diff --git a/src/xrpld/peerfinder/make_Manager.h b/src/xrpld/peerfinder/make_Manager.h deleted file mode 100644 index 1c13d7a4ca..0000000000 --- a/src/xrpld/peerfinder/make_Manager.h +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -#include -#include -#include - -#include - -#include - -namespace xrpl::PeerFinder { - -/** - * Create a new Manager. - */ -std::unique_ptr -makeManager( - boost::asio::io_context& ioContext, - clock_type& clock, - beast::Journal journal, - BasicConfig const& config, - beast::insight::Collector::ptr const& collector); - -} // namespace xrpl::PeerFinder diff --git a/src/xrpld/shamap/NodeFamily.h b/src/xrpld/shamap/NodeFamily.h index d532f13ecc..1307d76886 100644 --- a/src/xrpld/shamap/NodeFamily.h +++ b/src/xrpld/shamap/NodeFamily.h @@ -33,13 +33,13 @@ public: NodeFamily(Application& app, CollectorManager& cm); - NodeStore::Database& + node_store::Database& db() override { return db_; } - [[nodiscard]] NodeStore::Database const& + [[nodiscard]] node_store::Database const& db() const override { return db_; @@ -80,7 +80,7 @@ public: private: Application& app_; - NodeStore::Database& db_; + node_store::Database& db_; beast::Journal const j_; std::shared_ptr fbCache_;