diff --git a/.github/actions/build-clio/action.yml b/.github/actions/build-clio/action.yml index 86f7ff05e..6fa94eef2 100644 --- a/.github/actions/build-clio/action.yml +++ b/.github/actions/build-clio/action.yml @@ -14,7 +14,7 @@ runs: using: composite steps: - name: Get number of processors - uses: XRPLF/actions/.github/actions/get-nproc@046b1620f6bfd6cd0985dc82c3df02786801fe0a + uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf id: nproc with: subtract: ${{ inputs.nproc_subtract }} diff --git a/.github/actions/code-coverage/action.yml b/.github/actions/code-coverage/action.yml index 6a89a0a24..fa9f67a4e 100644 --- a/.github/actions/code-coverage/action.yml +++ b/.github/actions/code-coverage/action.yml @@ -24,7 +24,7 @@ runs: -j8 --exclude-throw-branches - name: Archive coverage report - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: coverage-report.xml path: build/coverage_report.xml diff --git a/.github/actions/conan/action.yml b/.github/actions/conan/action.yml index 5c6a0ddd6..897c3e548 100644 --- a/.github/actions/conan/action.yml +++ b/.github/actions/conan/action.yml @@ -21,10 +21,6 @@ inputs: runs: using: composite steps: - - name: Create build directory - shell: bash - run: mkdir -p "${{ inputs.build_dir }}" - - name: Run conan shell: bash env: diff --git a/.github/scripts/conan/regenerate_lockfile.sh b/.github/scripts/conan/regenerate_lockfile.sh new file mode 100755 index 000000000..2024c80d6 --- /dev/null +++ b/.github/scripts/conan/regenerate_lockfile.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -ex + +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +echo "Using temporary CONAN_HOME: $TEMP_DIR" + +# We use a temporary Conan home to avoid polluting the user's existing Conan +# configuration and to not use local cache (which leads to non-reproducible lockfiles). +export CONAN_HOME="$TEMP_DIR" + +# Ensure that the xrplf remote is the first to be consulted, so any recipes we +# patched are used. We also add it there to not created huge diff when the +# official Conan Center Index is updated. +conan remote add --force --index 0 xrplf https://conan.ripplex.io + +# Delete any existing lockfile. +rm -f conan.lock + +# Create a new lockfile that is compatible with macOS. +# It should also work on Linux. +conan lock create . \ + --profile:all=.github/scripts/conan/apple-clang-17.profile diff --git a/.github/workflows/build-clio-docker-image.yml b/.github/workflows/build-clio-docker-image.yml index 0427b7663..d73c6bec9 100644 --- a/.github/workflows/build-clio-docker-image.yml +++ b/.github/workflows/build-clio-docker-image.yml @@ -52,7 +52,7 @@ jobs: - name: Download Clio binary from artifact if: ${{ inputs.artifact_name != null }} - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: ${{ inputs.artifact_name }} path: ./docker/clio/artifact/ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd5d08699..10e64db0c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,24 +92,6 @@ jobs: secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - package: - name: Build packages - - uses: ./.github/workflows/reusable-build.yml - with: - runs_on: heavy - container: '{ "image": "ghcr.io/xrplf/clio-ci:067449c3f8ae6755ea84752ea2962b589fe56c8f" }' - conan_profile: gcc - build_type: Release - download_ccache: true - upload_ccache: false - code_coverage: false - static: true - upload_clio_server: false - package: true - targets: package - analyze_build_time: false - check_config: name: Check Config Description needs: build-and-test @@ -120,7 +102,7 @@ jobs: steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: clio_server_Linux_Release_gcc diff --git a/.github/workflows/check-libxrpl.yml b/.github/workflows/check-libxrpl.yml index 51b389e4c..4e5b744ae 100644 --- a/.github/workflows/check-libxrpl.yml +++ b/.github/workflows/check-libxrpl.yml @@ -29,7 +29,7 @@ jobs: fetch-depth: 0 - name: Prepare runner - uses: XRPLF/actions/.github/actions/prepare-runner@8abb0722cbff83a9a2dc7d06c473f7a4964b7382 + uses: XRPLF/actions/prepare-runner@2ece4ec6ab7de266859a6f053571425b2bd684b6 with: disable_ccache: true @@ -59,7 +59,7 @@ jobs: run: strip build/clio_tests - name: Upload clio_tests - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: clio_tests_check_libxrpl path: build/clio_tests @@ -72,7 +72,7 @@ jobs: image: ghcr.io/xrplf/clio-ci:067449c3f8ae6755ea84752ea2962b589fe56c8f steps: - - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: clio_tests_check_libxrpl diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 565d25221..db75ab300 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -44,7 +44,7 @@ jobs: fetch-depth: 0 - name: Prepare runner - uses: XRPLF/actions/.github/actions/prepare-runner@8abb0722cbff83a9a2dc7d06c473f7a4964b7382 + uses: XRPLF/actions/prepare-runner@2ece4ec6ab7de266859a6f053571425b2bd684b6 with: disable_ccache: true @@ -59,7 +59,7 @@ jobs: conan_profile: ${{ env.CONAN_PROFILE }} - name: Get number of processors - uses: XRPLF/actions/.github/actions/get-nproc@046b1620f6bfd6cd0985dc82c3df02786801fe0a + uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf id: nproc - name: Run clang-tidy (several times) @@ -107,7 +107,7 @@ jobs: - name: Create PR with fixes if: ${{ steps.files_changed.outcome != 'success' && github.event_name != 'pull_request' }} - uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7.0.11 + uses: peter-evans/create-pull-request@98357b18bf14b5342f975ff684046ec3b2a07725 # v8.0.0 env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c3347330c..f35b872bd 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,7 +27,7 @@ jobs: lfs: true - name: Prepare runner - uses: XRPLF/actions/.github/actions/prepare-runner@8abb0722cbff83a9a2dc7d06c473f7a4964b7382 + uses: XRPLF/actions/prepare-runner@2ece4ec6ab7de266859a6f053571425b2bd684b6 with: disable_ccache: true diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 49e060c00..b1529190c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -68,6 +68,24 @@ jobs: download_ccache: false upload_ccache: false + package: + name: Build debian package + + uses: ./.github/workflows/reusable-build.yml + with: + runs_on: heavy + container: '{ "image": "ghcr.io/xrplf/clio-ci:067449c3f8ae6755ea84752ea2962b589fe56c8f" }' + conan_profile: gcc + build_type: Release + download_ccache: false + upload_ccache: false + code_coverage: false + static: true + upload_clio_server: false + package: true + targets: package + analyze_build_time: false + analyze_build_time: name: Analyze Build Time @@ -109,7 +127,7 @@ jobs: echo "date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT nightly_release: - needs: [build-and-test, get_date] + needs: [build-and-test, package, get_date] uses: ./.github/workflows/reusable-release.yml with: delete_pattern: "nightly-*" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 76bc0172a..158107a42 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,8 +45,26 @@ jobs: upload_ccache: false expected_version: ${{ github.event_name == 'push' && github.ref_name || '' }} + package: + name: Build debian package + + uses: ./.github/workflows/reusable-build.yml + with: + runs_on: heavy + container: '{ "image": "ghcr.io/xrplf/clio-ci:067449c3f8ae6755ea84752ea2962b589fe56c8f" }' + conan_profile: gcc + build_type: Release + download_ccache: false + upload_ccache: false + code_coverage: false + static: true + upload_clio_server: false + package: true + targets: package + analyze_build_time: false + release: - needs: build-and-test + needs: [build-and-test, package] uses: ./.github/workflows/reusable-release.yml with: delete_pattern: "" diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index e4c9f188c..4586097e8 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -88,7 +88,7 @@ jobs: steps: - name: Cleanup workspace if: ${{ runner.os == 'macOS' }} - uses: XRPLF/actions/.github/actions/cleanup-workspace@ea9970b7c211b18f4c8bcdb28c29f5711752029f + uses: XRPLF/actions/cleanup-workspace@cf0433aa74563aead044a1e395610c96d65a37cf - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: @@ -99,7 +99,7 @@ jobs: ref: ${{ github.ref }} - name: Prepare runner - uses: XRPLF/actions/.github/actions/prepare-runner@8abb0722cbff83a9a2dc7d06c473f7a4964b7382 + uses: XRPLF/actions/prepare-runner@2ece4ec6ab7de266859a6f053571425b2bd684b6 with: disable_ccache: ${{ !inputs.download_ccache }} @@ -154,7 +154,7 @@ jobs: - name: Upload build time analyze report if: ${{ inputs.analyze_build_time }} - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: build_time_report_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} path: build_time_report.txt @@ -182,28 +182,28 @@ jobs: - name: Upload clio_server if: ${{ inputs.upload_clio_server && !inputs.code_coverage && !inputs.analyze_build_time }} - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: clio_server_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} path: build/clio_server - name: Upload clio_tests if: ${{ !inputs.code_coverage && !inputs.analyze_build_time && !inputs.package }} - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} path: build/clio_tests - name: Upload clio_integration_tests if: ${{ !inputs.code_coverage && !inputs.analyze_build_time && !inputs.package }} - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} path: build/clio_integration_tests - name: Upload Clio Linux package if: ${{ inputs.package }} - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: clio_deb_package_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} path: build/*.deb diff --git a/.github/workflows/reusable-release.yml b/.github/workflows/reusable-release.yml index 757774c39..c29e26518 100644 --- a/.github/workflows/reusable-release.yml +++ b/.github/workflows/reusable-release.yml @@ -60,15 +60,23 @@ jobs: fetch-depth: 0 - name: Prepare runner - uses: XRPLF/actions/.github/actions/prepare-runner@8abb0722cbff83a9a2dc7d06c473f7a4964b7382 + uses: XRPLF/actions/prepare-runner@2ece4ec6ab7de266859a6f053571425b2bd684b6 with: disable_ccache: true - - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: release_artifacts pattern: clio_server_* + - name: Prepare release artifacts + run: .github/scripts/prepare-release-artifacts.sh release_artifacts + + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + path: release_artifacts + pattern: clio_deb_package_* + - name: Create release notes env: RELEASE_HEADER: ${{ inputs.header }} @@ -86,11 +94,8 @@ jobs: git-cliff "${BASE_COMMIT}..HEAD" --ignore-tags "nightly|-b|-rc" cat CHANGELOG.md >> "${RUNNER_TEMP}/release_notes.md" - - name: Prepare release artifacts - run: .github/scripts/prepare-release-artifacts.sh release_artifacts - - name: Upload release notes - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: release_notes_${{ inputs.version }} path: "${RUNNER_TEMP}/release_notes.md" @@ -122,4 +127,4 @@ jobs: --target "${GITHUB_SHA}" \ ${DRAFT_OPTION} \ --notes-file "${RUNNER_TEMP}/release_notes.md" \ - ./release_artifacts/clio_server* + ./release_artifacts/clio_* diff --git a/.github/workflows/reusable-test.yml b/.github/workflows/reusable-test.yml index 757c9e8f8..f7ca51926 100644 --- a/.github/workflows/reusable-test.yml +++ b/.github/workflows/reusable-test.yml @@ -52,13 +52,13 @@ jobs: steps: - name: Cleanup workspace if: ${{ runner.os == 'macOS' }} - uses: XRPLF/actions/.github/actions/cleanup-workspace@ea9970b7c211b18f4c8bcdb28c29f5711752029f + uses: XRPLF/actions/cleanup-workspace@cf0433aa74563aead044a1e395610c96d65a37cf - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 - - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} @@ -85,7 +85,7 @@ jobs: - name: Upload sanitizer report if: ${{ env.SANITIZER_IGNORE_ERRORS == 'true' && steps.check_report.outputs.found_report == 'true' }} - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: sanitizer_report_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} path: .sanitizer-report/* @@ -124,7 +124,7 @@ jobs: steps: - name: Cleanup workspace if: ${{ runner.os == 'macOS' }} - uses: XRPLF/actions/.github/actions/cleanup-workspace@ea9970b7c211b18f4c8bcdb28c29f5711752029f + uses: XRPLF/actions/cleanup-workspace@cf0433aa74563aead044a1e395610c96d65a37cf - name: Spin up scylladb if: ${{ runner.os == 'macOS' }} @@ -146,7 +146,7 @@ jobs: sleep 5 done - - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} diff --git a/.github/workflows/reusable-upload-coverage-report.yml b/.github/workflows/reusable-upload-coverage-report.yml index a223c2744..61befa38b 100644 --- a/.github/workflows/reusable-upload-coverage-report.yml +++ b/.github/workflows/reusable-upload-coverage-report.yml @@ -21,14 +21,14 @@ jobs: fetch-depth: 0 - name: Download report artifact - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: name: coverage-report.xml path: build - name: Upload coverage report if: ${{ hashFiles('build/coverage_report.xml') != '' }} - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: files: build/coverage_report.xml fail_ci_if_error: true diff --git a/.github/workflows/update-docker-ci.yml b/.github/workflows/update-docker-ci.yml index eee543f74..745ac5e0f 100644 --- a/.github/workflows/update-docker-ci.yml +++ b/.github/workflows/update-docker-ci.yml @@ -60,7 +60,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: "docker/compilers/gcc/**" @@ -98,7 +98,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: "docker/compilers/gcc/**" @@ -136,7 +136,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: "docker/compilers/gcc/**" @@ -187,7 +187,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: "docker/compilers/clang/**" @@ -223,7 +223,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: "docker/tools/**" @@ -254,7 +254,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: "docker/tools/**" @@ -285,7 +285,7 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@e0021407031f5be11a464abee9a0776171c79891 # v47.0.1 with: files: "docker/tools/**" diff --git a/.github/workflows/upload-conan-deps.yml b/.github/workflows/upload-conan-deps.yml index 178e80d6c..07e05751e 100644 --- a/.github/workflows/upload-conan-deps.yml +++ b/.github/workflows/upload-conan-deps.yml @@ -22,6 +22,7 @@ on: - .github/actions/conan/action.yml - ".github/scripts/conan/**" + - "!.github/scripts/conan/regenerate_lockfile.sh" - conanfile.py - conan.lock @@ -32,6 +33,7 @@ on: - .github/actions/conan/action.yml - ".github/scripts/conan/**" + - "!.github/scripts/conan/regenerate_lockfile.sh" - conanfile.py - conan.lock @@ -76,7 +78,7 @@ jobs: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 - name: Prepare runner - uses: XRPLF/actions/.github/actions/prepare-runner@8abb0722cbff83a9a2dc7d06c473f7a4964b7382 + uses: XRPLF/actions/prepare-runner@2ece4ec6ab7de266859a6f053571425b2bd684b6 with: disable_ccache: true diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index ec2d91eb9..e70ee2bb4 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -9,10 +9,12 @@ target_sources( util/async/ExecutionContextBenchmarks.cpp # Logger util/log/LoggerBenchmark.cpp + # WorkQueue + rpc/WorkQueueBenchmarks.cpp ) include(deps/gbench) target_include_directories(clio_benchmark PRIVATE .) -target_link_libraries(clio_benchmark PUBLIC clio_util benchmark::benchmark_main spdlog::spdlog) +target_link_libraries(clio_benchmark PUBLIC clio_util clio_rpc benchmark::benchmark_main spdlog::spdlog) set_target_properties(clio_benchmark PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) diff --git a/benchmarks/rpc/WorkQueueBenchmarks.cpp b/benchmarks/rpc/WorkQueueBenchmarks.cpp new file mode 100644 index 000000000..9b9ce6947 --- /dev/null +++ b/benchmarks/rpc/WorkQueueBenchmarks.cpp @@ -0,0 +1,127 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "rpc/WorkQueue.hpp" +#include "util/Assert.hpp" +#include "util/config/Array.hpp" +#include "util/config/ConfigConstraints.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ConfigValue.hpp" +#include "util/config/Types.hpp" +#include "util/log/Logger.hpp" +#include "util/prometheus/Prometheus.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace rpc; +using namespace util::config; + +namespace { + +auto const kCONFIG = ClioConfigDefinition{ + {"prometheus.compress_reply", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, + {"prometheus.enabled", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, + {"log.channels.[].channel", Array{ConfigValue{ConfigType::String}}}, + {"log.channels.[].level", Array{ConfigValue{ConfigType::String}}}, + {"log.level", ConfigValue{ConfigType::String}.defaultValue("info")}, + {"log.format", ConfigValue{ConfigType::String}.defaultValue(R"(%Y-%m-%d %H:%M:%S.%f %^%3!l:%n%$ - %v)")}, + {"log.is_async", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"log.enable_console", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"log.directory", ConfigValue{ConfigType::String}.optional()}, + {"log.rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048).withConstraint(gValidateUint32)}, + {"log.directory_max_files", ConfigValue{ConfigType::Integer}.defaultValue(25).withConstraint(gValidateUint32)}, + {"log.tag_style", ConfigValue{ConfigType::String}.defaultValue("none")}, +}; + +// this should be a fixture but it did not work with Args very well +void +init() +{ + static std::once_flag kONCE; + std::call_once(kONCE, [] { + PrometheusService::init(kCONFIG); + (void)util::LogService::init(kCONFIG); + }); +} + +} // namespace + +static void +benchmarkWorkQueue(benchmark::State& state) +{ + init(); + + auto const total = static_cast(state.range(0)); + auto const numThreads = static_cast(state.range(1)); + auto const maxSize = static_cast(state.range(2)); + auto const delayMs = static_cast(state.range(3)); + + for (auto _ : state) { + std::atomic_size_t totalExecuted = 0uz; + std::atomic_size_t totalQueued = 0uz; + + state.PauseTiming(); + WorkQueue queue(numThreads, maxSize); + state.ResumeTiming(); + + for (auto i = 0uz; i < total; ++i) { + totalQueued += static_cast(queue.postCoro( + [&delayMs, &totalExecuted](auto yield) { + ++totalExecuted; + + boost::asio::steady_timer timer(yield.get_executor(), std::chrono::milliseconds{delayMs}); + timer.async_wait(yield); + }, + /* isWhiteListed = */ false + )); + } + + queue.stop(); + + ASSERT(totalExecuted == totalQueued, "Totals don't match"); + ASSERT(totalQueued <= total, "Queued more than requested"); + ASSERT(totalQueued >= maxSize, "Queued less than maxSize"); + } +} + +// Usage example: +/* + ./clio_benchmark \ + --benchmark_repetitions=10 \ + --benchmark_display_aggregates_only=true \ + --benchmark_min_time=1x \ + --benchmark_filter="WorkQueue" +*/ +// TODO: figure out what happens on 1 thread +BENCHMARK(benchmarkWorkQueue) + ->ArgsProduct({{1'000, 10'000, 100'000}, {2, 4, 8}, {0, 5'000}, {10, 100, 250}}) + ->Unit(benchmark::kMillisecond); diff --git a/docs/build-clio.md b/docs/build-clio.md index 51c6822b5..791bc49d3 100644 --- a/docs/build-clio.md +++ b/docs/build-clio.md @@ -97,30 +97,14 @@ Now you should be able to download the prebuilt dependencies (including `xrpl` p #### Conan lockfile -To achieve reproducible dependencies, we use [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html). +To achieve reproducible dependencies, we use a [Conan lockfile](https://docs.conan.io/2/tutorial/versioning/lockfiles.html). The `conan.lock` file in the repository contains a "snapshot" of the current dependencies. It is implicitly used when running `conan` commands, you don't need to specify it. You have to update this file every time you add a new dependency or change a revision or version of an existing dependency. -> [!NOTE] -> Conan uses local cache by default when creating a lockfile. -> -> To ensure, that lockfile creation works the same way on all developer machines, you should clear the local cache before creating a new lockfile. - -To create a new lockfile, run the following commands in the repository root: - -```bash -conan remove '*' --confirm -rm conan.lock -# This ensure that xrplf remote is the first to be consulted -conan remote add --force --index 0 xrplf https://conan.ripplex.io -conan lock create . -``` - -> [!NOTE] -> If some dependencies are exclusive for some OS, you may need to run the last command for them adding `--profile:all `. +To update a lockfile, run from the repository root: `./.github/scripts/conan/regenerate_lockfile.sh` ## Building Clio diff --git a/src/rpc/WorkQueue.cpp b/src/rpc/WorkQueue.cpp index 34617dec3..b676fc64b 100644 --- a/src/rpc/WorkQueue.cpp +++ b/src/rpc/WorkQueue.cpp @@ -34,8 +34,8 @@ #include #include #include +#include #include -#include namespace rpc { @@ -122,7 +122,7 @@ WorkQueue::dispatcherLoop(boost::asio::yield_context yield) // all ongoing tasks must be completed before stopping fully while (not stopping_ or size() > 0) { - std::vector batch; + std::optional task; { auto state = dispatcherState_.lock(); @@ -130,43 +130,31 @@ WorkQueue::dispatcherLoop(boost::asio::yield_context yield) if (state->empty()) { state->isIdle = true; } else { - for (auto count = 0uz; count < kTAKE_HIGH_PRIO and not state->high.empty(); ++count) { - batch.push_back(std::move(state->high.front())); - state->high.pop(); - } - - if (not state->normal.empty()) { - batch.push_back(std::move(state->normal.front())); - state->normal.pop(); - } + task = state->popNext(); } } - if (not stopping_ and batch.empty()) { + if (not stopping_ and not task.has_value()) { waitTimer_.expires_at(std::chrono::steady_clock::time_point::max()); boost::system::error_code ec; waitTimer_.async_wait(yield[ec]); - } else { - for (auto task : std::move(batch)) { - util::spawn( - ioc_, - [this, spawnedAt = std::chrono::system_clock::now(), task = std::move(task)](auto yield) mutable { - auto const takenAt = std::chrono::system_clock::now(); - auto const waited = - std::chrono::duration_cast(takenAt - spawnedAt).count(); + } else if (task.has_value()) { + util::spawn( + ioc_, + [this, spawnedAt = std::chrono::system_clock::now(), task = std::move(*task)](auto yield) mutable { + auto const takenAt = std::chrono::system_clock::now(); + auto const waited = + std::chrono::duration_cast(takenAt - spawnedAt).count(); - ++queued_.get(); - durationUs_.get() += waited; - LOG(log_.info()) << "WorkQueue wait time: " << waited << ", queue size: " << size(); + ++queued_.get(); + durationUs_.get() += waited; + LOG(log_.info()) << "WorkQueue wait time: " << waited << ", queue size: " << size(); - task(yield); + task(yield); - --curSize_.get(); - } - ); - } - - boost::asio::post(ioc_.get_executor(), yield); // yield back to avoid hijacking the thread + --curSize_.get(); + } + ); } } diff --git a/src/rpc/WorkQueue.hpp b/src/rpc/WorkQueue.hpp index 702a173df..8fa466501 100644 --- a/src/rpc/WorkQueue.hpp +++ b/src/rpc/WorkQueue.hpp @@ -38,7 +38,9 @@ #include #include #include +#include #include +#include namespace rpc { @@ -79,6 +81,7 @@ private: QueueType normal; bool isIdle = false; + size_t highPriorityCounter = 0; void push(Priority priority, auto&& task) @@ -96,6 +99,26 @@ private: { return high.empty() and normal.empty(); } + + [[nodiscard]] std::optional + popNext() + { + if (not high.empty() and (highPriorityCounter < kTAKE_HIGH_PRIO or normal.empty())) { + auto task = std::move(high.front()); + high.pop(); + ++highPriorityCounter; + return task; + } + + if (not normal.empty()) { + auto task = std::move(normal.front()); + normal.pop(); + highPriorityCounter = 0; + return task; + } + + return std::nullopt; + } }; private: diff --git a/tests/unit/rpc/handlers/AccountMPTokenIssuancesTests.cpp b/tests/unit/rpc/handlers/AccountMPTokenIssuancesTests.cpp index 09af77b63..5e1a045e9 100644 --- a/tests/unit/rpc/handlers/AccountMPTokenIssuancesTests.cpp +++ b/tests/unit/rpc/handlers/AccountMPTokenIssuancesTests.cpp @@ -985,13 +985,13 @@ static auto generateSingleFlagTests() { return std::vector{ - {"Locked", ripple::lsfMPTLocked, "mpt_locked"}, - {"CanLock", ripple::lsfMPTCanLock, "mpt_can_lock"}, - {"RequireAuth", ripple::lsfMPTRequireAuth, "mpt_require_auth"}, - {"CanEscrow", ripple::lsfMPTCanEscrow, "mpt_can_escrow"}, - {"CanTrade", ripple::lsfMPTCanTrade, "mpt_can_trade"}, - {"CanTransfer", ripple::lsfMPTCanTransfer, "mpt_can_transfer"}, - {"CanClawback", ripple::lsfMPTCanClawback, "mpt_can_clawback"}, + {.testName = "Locked", .flag = ripple::lsfMPTLocked, .expectedJsonKey = "mpt_locked"}, + {.testName = "CanLock", .flag = ripple::lsfMPTCanLock, .expectedJsonKey = "mpt_can_lock"}, + {.testName = "RequireAuth", .flag = ripple::lsfMPTRequireAuth, .expectedJsonKey = "mpt_require_auth"}, + {.testName = "CanEscrow", .flag = ripple::lsfMPTCanEscrow, .expectedJsonKey = "mpt_can_escrow"}, + {.testName = "CanTrade", .flag = ripple::lsfMPTCanTrade, .expectedJsonKey = "mpt_can_trade"}, + {.testName = "CanTransfer", .flag = ripple::lsfMPTCanTransfer, .expectedJsonKey = "mpt_can_transfer"}, + {.testName = "CanClawback", .flag = ripple::lsfMPTCanClawback, .expectedJsonKey = "mpt_can_clawback"}, }; } @@ -1059,14 +1059,30 @@ static auto generateSingleMutableFlagTests() { return std::vector{ - {"CanMutateCanLock", ripple::lsmfMPTCanMutateCanLock, "mpt_can_mutate_can_lock"}, - {"CanMutateRequireAuth", ripple::lsmfMPTCanMutateRequireAuth, "mpt_can_mutate_require_auth"}, - {"CanMutateCanEscrow", ripple::lsmfMPTCanMutateCanEscrow, "mpt_can_mutate_can_escrow"}, - {"CanMutateCanTrade", ripple::lsmfMPTCanMutateCanTrade, "mpt_can_mutate_can_trade"}, - {"CanMutateCanTransfer", ripple::lsmfMPTCanMutateCanTransfer, "mpt_can_mutate_can_transfer"}, - {"CanMutateCanClawback", ripple::lsmfMPTCanMutateCanClawback, "mpt_can_mutate_can_clawback"}, - {"CanMutateMetadata", ripple::lsmfMPTCanMutateMetadata, "mpt_can_mutate_metadata"}, - {"CanMutateTransferFee", ripple::lsmfMPTCanMutateTransferFee, "mpt_can_mutate_transfer_fee"}, + {.testName = "CanMutateCanLock", + .mutableFlag = ripple::lsmfMPTCanMutateCanLock, + .expectedJsonKey = "mpt_can_mutate_can_lock"}, + {.testName = "CanMutateRequireAuth", + .mutableFlag = ripple::lsmfMPTCanMutateRequireAuth, + .expectedJsonKey = "mpt_can_mutate_require_auth"}, + {.testName = "CanMutateCanEscrow", + .mutableFlag = ripple::lsmfMPTCanMutateCanEscrow, + .expectedJsonKey = "mpt_can_mutate_can_escrow"}, + {.testName = "CanMutateCanTrade", + .mutableFlag = ripple::lsmfMPTCanMutateCanTrade, + .expectedJsonKey = "mpt_can_mutate_can_trade"}, + {.testName = "CanMutateCanTransfer", + .mutableFlag = ripple::lsmfMPTCanMutateCanTransfer, + .expectedJsonKey = "mpt_can_mutate_can_transfer"}, + {.testName = "CanMutateCanClawback", + .mutableFlag = ripple::lsmfMPTCanMutateCanClawback, + .expectedJsonKey = "mpt_can_mutate_can_clawback"}, + {.testName = "CanMutateMetadata", + .mutableFlag = ripple::lsmfMPTCanMutateMetadata, + .expectedJsonKey = "mpt_can_mutate_metadata"}, + {.testName = "CanMutateTransferFee", + .mutableFlag = ripple::lsmfMPTCanMutateTransferFee, + .expectedJsonKey = "mpt_can_mutate_transfer_fee"}, }; }