diff --git a/.clang-tidy b/.clang-tidy index f7009c4666..5f4187b008 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,105 +1,143 @@ --- Checks: "-*, - bugprone-argument-comment + bugprone-argument-comment, + bugprone-assert-side-effect, + bugprone-bad-signal-to-kill-thread, + bugprone-bool-pointer-implicit-conversion, + bugprone-casting-through-void, + bugprone-chained-comparison, + bugprone-compare-pointer-to-member-virtual-function, + bugprone-copy-constructor-init, + bugprone-dangling-handle, + bugprone-dynamic-static-initializers, + bugprone-fold-init-type, + bugprone-forward-declaration-namespace, + bugprone-inaccurate-erase, + bugprone-incorrect-enable-if, + bugprone-incorrect-roundings, + bugprone-infinite-loop, + bugprone-integer-division, + bugprone-lambda-function-name, + bugprone-macro-parentheses, + bugprone-macro-repeated-side-effects, + bugprone-misplaced-operator-in-strlen-in-alloc, + bugprone-misplaced-pointer-arithmetic-in-alloc, + bugprone-misplaced-widening-cast, + bugprone-multi-level-implicit-pointer-conversion, + bugprone-multiple-new-in-one-expression, + bugprone-multiple-statement-macro, + bugprone-no-escape, + bugprone-non-zero-enum-to-bool-conversion, + bugprone-parent-virtual-call, + bugprone-posix-return, + bugprone-redundant-branch-condition, + bugprone-shared-ptr-array-mismatch, + bugprone-signal-handler, + bugprone-signed-char-misuse, + bugprone-sizeof-container, + bugprone-spuriously-wake-up-functions, + bugprone-standalone-empty, + bugprone-string-constructor, + bugprone-string-integer-assignment, + bugprone-string-literal-with-embedded-nul, + bugprone-stringview-nullptr, + bugprone-suspicious-enum-usage, + bugprone-suspicious-include, + bugprone-suspicious-memory-comparison, + bugprone-suspicious-memset-usage, + bugprone-suspicious-realloc-usage, + bugprone-suspicious-semicolon, + bugprone-suspicious-string-compare, + bugprone-swapped-arguments, + bugprone-terminating-continue, + bugprone-throw-keyword-missing, + bugprone-undefined-memory-manipulation, + bugprone-undelegated-constructor, + bugprone-unhandled-exception-at-new, + bugprone-unique-ptr-array-mismatch, + bugprone-unsafe-functions, + bugprone-virtual-near-miss, + cppcoreguidelines-no-suspend-with-lock, + cppcoreguidelines-virtual-class-destructor, + hicpp-ignored-remove-result, + misc-definitions-in-headers, + misc-header-include-cycle, + misc-misplaced-const, + misc-static-assert, + misc-throw-by-value-catch-by-reference, + misc-unused-alias-decls, + misc-unused-using-decls, + readability-duplicate-include, + readability-enum-initial-value, + readability-misleading-indentation, + readability-non-const-parameter, + readability-redundant-declaration, + readability-reference-to-constructed-temporary, + modernize-deprecated-headers, + modernize-make-shared, + modernize-make-unique, + performance-implicit-conversion-in-loop, + performance-move-constructor-init, + performance-trivially-destructible " -# bugprone-assert-side-effect, -# bugprone-bad-signal-to-kill-thread, -# bugprone-bool-pointer-implicit-conversion, -# bugprone-casting-through-void, -# bugprone-chained-comparison, -# bugprone-compare-pointer-to-member-virtual-function, -# bugprone-copy-constructor-init, -# bugprone-crtp-constructor-accessibility, -# bugprone-dangling-handle, -# bugprone-dynamic-static-initializers, +# --- +# checks that have some issues that need to be resolved: +# # bugprone-empty-catch, -# bugprone-fold-init-type, -# bugprone-forward-declaration-namespace, -# bugprone-inaccurate-erase, +# bugprone-crtp-constructor-accessibility, # bugprone-inc-dec-in-conditions, -# bugprone-incorrect-enable-if, -# bugprone-incorrect-roundings, -# bugprone-infinite-loop, -# bugprone-integer-division, -# bugprone-lambda-function-name, -# bugprone-macro-parentheses, -# bugprone-macro-repeated-side-effects, -# bugprone-misplaced-operator-in-strlen-in-alloc, -# bugprone-misplaced-pointer-arithmetic-in-alloc, -# bugprone-misplaced-widening-cast, -# bugprone-move-forwarding-reference, -# bugprone-multi-level-implicit-pointer-conversion, -# bugprone-multiple-new-in-one-expression, -# bugprone-multiple-statement-macro, -# bugprone-no-escape, -# bugprone-non-zero-enum-to-bool-conversion, -# bugprone-optional-value-conversion, -# bugprone-parent-virtual-call, -# bugprone-pointer-arithmetic-on-polymorphic-object, -# bugprone-posix-return, -# bugprone-redundant-branch-condition, # bugprone-reserved-identifier, -# bugprone-return-const-ref-from-parameter, -# bugprone-shared-ptr-array-mismatch, -# bugprone-signal-handler, -# bugprone-signed-char-misuse, -# bugprone-sizeof-container, -# bugprone-sizeof-expression, -# bugprone-spuriously-wake-up-functions, -# bugprone-standalone-empty, -# bugprone-string-constructor, -# bugprone-string-integer-assignment, -# bugprone-string-literal-with-embedded-nul, -# bugprone-stringview-nullptr, -# bugprone-suspicious-enum-usage, -# bugprone-suspicious-include, -# bugprone-suspicious-memory-comparison, -# bugprone-suspicious-memset-usage, -# bugprone-suspicious-missing-comma, -# bugprone-suspicious-realloc-usage, -# bugprone-suspicious-semicolon, -# bugprone-suspicious-string-compare, -# bugprone-suspicious-stringview-data-usage, -# bugprone-swapped-arguments, -# bugprone-switch-missing-default-case, -# bugprone-terminating-continue, -# bugprone-throw-keyword-missing, -# bugprone-too-small-loop-variable, -# bugprone-undefined-memory-manipulation, -# bugprone-undelegated-constructor, -# bugprone-unhandled-exception-at-new, -# bugprone-unhandled-self-assignment, -# bugprone-unique-ptr-array-mismatch, -# bugprone-unsafe-functions, +# bugprone-move-forwarding-reference, # bugprone-unused-local-non-trivial-variable, -# bugprone-unused-raii, +# bugprone-return-const-ref-from-parameter, +# bugprone-switch-missing-default-case, +# bugprone-sizeof-expression, +# bugprone-suspicious-stringview-data-usage, +# bugprone-suspicious-missing-comma, +# bugprone-pointer-arithmetic-on-polymorphic-object, +# bugprone-optional-value-conversion, +# bugprone-too-small-loop-variable, # bugprone-unused-return-value, # bugprone-use-after-move, -# bugprone-virtual-near-miss, -# cppcoreguidelines-init-variables, +# bugprone-unhandled-self-assignment, +# bugprone-unused-raii, +# # cppcoreguidelines-misleading-capture-default-by-value, -# cppcoreguidelines-no-suspend-with-lock, +# cppcoreguidelines-init-variables, # cppcoreguidelines-pro-type-member-init, # cppcoreguidelines-pro-type-static-cast-downcast, -# cppcoreguidelines-rvalue-reference-param-not-moved, # cppcoreguidelines-use-default-member-init, -# cppcoreguidelines-virtual-class-destructor, -# hicpp-ignored-remove-result, +# cppcoreguidelines-rvalue-reference-param-not-moved, +# # llvm-namespace-comment, # misc-const-correctness, -# misc-definitions-in-headers, -# misc-header-include-cycle, # misc-include-cleaner, -# misc-misplaced-const, # misc-redundant-expression, -# misc-static-assert, -# misc-throw-by-value-catch-by-reference, -# misc-unused-alias-decls, -# misc-unused-using-decls, +# +# readability-avoid-nested-conditional-operator, +# readability-avoid-return-with-void-value, +# readability-braces-around-statements, +# readability-container-contains, +# readability-container-size-empty, +# readability-convert-member-functions-to-static, +# readability-const-return-type, +# readability-else-after-return, +# readability-implicit-bool-conversion, +# readability-inconsistent-declaration-parameter-name, +# readability-identifier-naming, +# readability-make-member-function-const, +# readability-math-missing-parentheses, +# readability-redundant-inline-specifier, +# readability-redundant-member-init, +# readability-redundant-casting, +# readability-redundant-string-init, +# readability-simplify-boolean-expr, +# readability-static-definition-in-anonymous-namespace, +# readability-suspicious-call-argument, +# readability-use-std-min-max, +# readability-static-accessed-through-instance, +# # modernize-concat-nested-namespaces, -# modernize-deprecated-headers, -# modernize-make-shared, -# modernize-make-unique, # modernize-pass-by-value, # modernize-type-traits, # modernize-use-designated-initializers, @@ -111,79 +149,50 @@ Checks: "-*, # modernize-use-starts-ends-with, # modernize-use-std-numbers, # modernize-use-using, +# # performance-faster-string-find, # performance-for-range-copy, -# performance-implicit-conversion-in-loop, # performance-inefficient-vector-operation, # performance-move-const-arg, -# performance-move-constructor-init, # performance-no-automatic-move, -# performance-trivially-destructible, -# readability-avoid-nested-conditional-operator, -# readability-avoid-return-with-void-value, -# readability-braces-around-statements, -# readability-const-return-type, -# readability-container-contains, -# readability-container-size-empty, -# readability-convert-member-functions-to-static, -# readability-duplicate-include, -# readability-else-after-return, -# readability-enum-initial-value, -# readability-implicit-bool-conversion, -# readability-inconsistent-declaration-parameter-name, -# readability-identifier-naming, -# readability-make-member-function-const, -# readability-math-missing-parentheses, -# readability-misleading-indentation, -# readability-non-const-parameter, -# readability-redundant-casting, -# readability-redundant-declaration, -# readability-redundant-inline-specifier, -# readability-redundant-member-init, -# readability-redundant-string-init, -# readability-reference-to-constructed-temporary, -# readability-simplify-boolean-expr, -# readability-static-accessed-through-instance, -# readability-static-definition-in-anonymous-namespace, -# readability-suspicious-call-argument, -# readability-use-std-min-max +# --- # -# CheckOptions: -# readability-braces-around-statements.ShortStatementLines: 2 -# readability-identifier-naming.MacroDefinitionCase: UPPER_CASE -# readability-identifier-naming.ClassCase: CamelCase -# readability-identifier-naming.StructCase: CamelCase -# readability-identifier-naming.UnionCase: CamelCase -# readability-identifier-naming.EnumCase: CamelCase -# readability-identifier-naming.EnumConstantCase: CamelCase -# readability-identifier-naming.ScopedEnumConstantCase: CamelCase -# readability-identifier-naming.GlobalConstantCase: UPPER_CASE -# readability-identifier-naming.GlobalConstantPrefix: "k" -# readability-identifier-naming.GlobalVariableCase: CamelCase -# readability-identifier-naming.GlobalVariablePrefix: "g" -# readability-identifier-naming.ConstexprFunctionCase: camelBack -# readability-identifier-naming.ConstexprMethodCase: camelBack -# readability-identifier-naming.ClassMethodCase: camelBack -# readability-identifier-naming.ClassMemberCase: camelBack -# readability-identifier-naming.ClassConstantCase: UPPER_CASE -# readability-identifier-naming.ClassConstantPrefix: "k" -# readability-identifier-naming.StaticConstantCase: UPPER_CASE -# readability-identifier-naming.StaticConstantPrefix: "k" -# readability-identifier-naming.StaticVariableCase: UPPER_CASE -# readability-identifier-naming.StaticVariablePrefix: "k" -# readability-identifier-naming.ConstexprVariableCase: UPPER_CASE -# readability-identifier-naming.ConstexprVariablePrefix: "k" -# readability-identifier-naming.LocalConstantCase: camelBack -# readability-identifier-naming.LocalVariableCase: camelBack -# readability-identifier-naming.TemplateParameterCase: CamelCase -# readability-identifier-naming.ParameterCase: camelBack -# readability-identifier-naming.FunctionCase: camelBack -# readability-identifier-naming.MemberCase: camelBack -# readability-identifier-naming.PrivateMemberSuffix: _ -# readability-identifier-naming.ProtectedMemberSuffix: _ -# readability-identifier-naming.PublicMemberSuffix: "" -# readability-identifier-naming.FunctionIgnoredRegexp: ".*tag_invoke.*" -# bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true +CheckOptions: + # readability-braces-around-statements.ShortStatementLines: 2 + # readability-identifier-naming.MacroDefinitionCase: UPPER_CASE + # readability-identifier-naming.ClassCase: CamelCase + # readability-identifier-naming.StructCase: CamelCase + # readability-identifier-naming.UnionCase: CamelCase + # readability-identifier-naming.EnumCase: CamelCase + # readability-identifier-naming.EnumConstantCase: CamelCase + # readability-identifier-naming.ScopedEnumConstantCase: CamelCase + # readability-identifier-naming.GlobalConstantCase: UPPER_CASE + # readability-identifier-naming.GlobalConstantPrefix: "k" + # readability-identifier-naming.GlobalVariableCase: CamelCase + # readability-identifier-naming.GlobalVariablePrefix: "g" + # readability-identifier-naming.ConstexprFunctionCase: camelBack + # readability-identifier-naming.ConstexprMethodCase: camelBack + # readability-identifier-naming.ClassMethodCase: camelBack + # readability-identifier-naming.ClassMemberCase: camelBack + # readability-identifier-naming.ClassConstantCase: UPPER_CASE + # readability-identifier-naming.ClassConstantPrefix: "k" + # readability-identifier-naming.StaticConstantCase: UPPER_CASE + # readability-identifier-naming.StaticConstantPrefix: "k" + # readability-identifier-naming.StaticVariableCase: UPPER_CASE + # readability-identifier-naming.StaticVariablePrefix: "k" + # readability-identifier-naming.ConstexprVariableCase: UPPER_CASE + # readability-identifier-naming.ConstexprVariablePrefix: "k" + # readability-identifier-naming.LocalConstantCase: camelBack + # readability-identifier-naming.LocalVariableCase: camelBack + # readability-identifier-naming.TemplateParameterCase: CamelCase + # readability-identifier-naming.ParameterCase: camelBack + # readability-identifier-naming.FunctionCase: camelBack + # readability-identifier-naming.MemberCase: camelBack + # readability-identifier-naming.PrivateMemberSuffix: _ + # readability-identifier-naming.ProtectedMemberSuffix: _ + # readability-identifier-naming.PublicMemberSuffix: "" + # readability-identifier-naming.FunctionIgnoredRegexp: ".*tag_invoke.*" + bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true # bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc # misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*;.*ranges_lower_bound\.h;time.h;stdlib.h;__chrono/.*;fmt/chrono.h;boost/uuid/uuid_hash.hpp' # diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..66e319e0e7 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,56 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/build-deps/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/generate-version/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/print-env/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/setup-conan/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index b0710303a0..be6a92eea2 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Determine changed files # This step checks whether any files have changed that should # cause the next jobs to run. We do it this way rather than @@ -46,7 +46,7 @@ jobs: # that Github considers any skipped jobs to have passed, and in # turn the required checks as well. id: changes - uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5 + uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 with: files: | # These paths are unique to `on-pr.yml`. diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index f43275201c..7793d1e3ab 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -14,4 +14,4 @@ jobs: uses: XRPLF/actions/.github/workflows/pre-commit.yml@320be44621ca2a080f05aeb15817c44b84518108 with: runs_on: ubuntu-latest - container: '{ "image": "ghcr.io/xrplf/ci/tools-rippled-pre-commit:sha-ab4d1f0" }' + container: '{ "image": "ghcr.io/xrplf/ci/tools-rippled-pre-commit:sha-41ec7c1" }' diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index f61559d6d3..acde57fd91 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -4,6 +4,18 @@ name: Build and publish documentation on: push: + branches: + - "develop" + - "release*" + paths: + - ".github/workflows/publish-docs.yml" + - "*.md" + - "**/*.md" + - "docs/**" + - "include/**" + - "src/libxrpl/**" + - "src/xrpld/**" + pull_request: paths: - ".github/workflows/publish-docs.yml" - "*.md" @@ -23,7 +35,9 @@ defaults: env: BUILD_DIR: build - NPROC_SUBTRACT: 2 + # ubuntu-latest has only 2 CPUs for private repositories + # https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for--private-repositories + NPROC_SUBTRACT: ${{ github.event.repository.private && '1' || '2' }} jobs: publish: @@ -33,7 +47,7 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Get number of processors uses: XRPLF/actions/get-nproc@cf0433aa74563aead044a1e395610c96d65a37cf @@ -65,7 +79,7 @@ jobs: cmake --build . --target docs --parallel ${BUILD_NPROC} - name: Publish documentation - if: ${{ github.ref_type == 'branch' && github.ref_name == github.event.repository.default_branch }} + if: ${{ github.event_name == 'push' }} uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/reusable-build-test-config.yml b/.github/workflows/reusable-build-test-config.yml index 85b973ea0c..6060a208fe 100644 --- a/.github/workflows/reusable-build-test-config.yml +++ b/.github/workflows/reusable-build-test-config.yml @@ -101,10 +101,10 @@ jobs: steps: - name: Cleanup workspace (macOS and Windows) if: ${{ runner.os == 'macOS' || runner.os == 'Windows' }} - uses: XRPLF/actions/cleanup-workspace@cf0433aa74563aead044a1e395610c96d65a37cf + uses: XRPLF/actions/cleanup-workspace@c7d9ce5ebb03c752a354889ecd870cadfc2b1cd4 - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Prepare runner uses: XRPLF/actions/prepare-runner@2cbf481018d930656e9276fcc20dc0e3a0be5b6d @@ -177,7 +177,7 @@ jobs: - name: Upload the binary (Linux) if: ${{ github.repository_owner == 'XRPLF' && runner.os == 'Linux' }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: xrpld-${{ inputs.config_name }} path: ${{ env.BUILD_DIR }}/xrpld @@ -254,7 +254,7 @@ jobs: - name: Upload coverage report if: ${{ github.repository_owner == 'XRPLF' && !inputs.build_only && env.COVERAGE_ENABLED == 'true' }} - uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3 + uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5.5.2 with: disable_search: true disable_telem: true diff --git a/.github/workflows/reusable-check-levelization.yml b/.github/workflows/reusable-check-levelization.yml index 29a1dc1480..ae3eed0f16 100644 --- a/.github/workflows/reusable-check-levelization.yml +++ b/.github/workflows/reusable-check-levelization.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Check levelization run: .github/scripts/levelization/generate.sh - name: Check for differences diff --git a/.github/workflows/reusable-check-rename.yml b/.github/workflows/reusable-check-rename.yml index a73ac49b7d..0e335ab9ca 100644 --- a/.github/workflows/reusable-check-rename.yml +++ b/.github/workflows/reusable-check-rename.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Check definitions run: .github/scripts/rename/definitions.sh . - name: Check copyright notices diff --git a/.github/workflows/reusable-clang-tidy-files.yml b/.github/workflows/reusable-clang-tidy-files.yml index c410a2e55d..d36dea747c 100644 --- a/.github/workflows/reusable-clang-tidy-files.yml +++ b/.github/workflows/reusable-clang-tidy-files.yml @@ -32,7 +32,7 @@ jobs: contents: read steps: - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Prepare runner uses: XRPLF/actions/prepare-runner@2cbf481018d930656e9276fcc20dc0e3a0be5b6d @@ -78,13 +78,13 @@ jobs: id: run_clang_tidy continue-on-error: true env: - FILES: ${{ inputs.files }} + TARGETS: ${{ inputs.files != '' && inputs.files || 'src tests' }} run: | - run-clang-tidy -j ${{ steps.nproc.outputs.nproc }} -p "$BUILD_DIR" $FILES 2>&1 | tee clang-tidy-output.txt + run-clang-tidy -j ${{ steps.nproc.outputs.nproc }} -p "${BUILD_DIR}" ${TARGETS} 2>&1 | tee clang-tidy-output.txt - name: Upload clang-tidy output if: steps.run_clang_tidy.outcome != 'success' - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: clang-tidy-results path: clang-tidy-output.txt diff --git a/.github/workflows/reusable-clang-tidy.yml b/.github/workflows/reusable-clang-tidy.yml index f2f1aeee78..7050d3509f 100644 --- a/.github/workflows/reusable-clang-tidy.yml +++ b/.github/workflows/reusable-clang-tidy.yml @@ -22,7 +22,8 @@ jobs: if: ${{ inputs.check_only_changed }} runs-on: ubuntu-latest outputs: - any_changed: ${{ steps.changed_files.outputs.any_changed }} + clang_tidy_config_changed: ${{ steps.changed_clang_tidy.outputs.any_changed }} + any_cpp_changed: ${{ steps.changed_files.outputs.any_changed }} all_changed_files: ${{ steps.changed_files.outputs.all_changed_files }} steps: - name: Checkout repository @@ -30,7 +31,7 @@ jobs: - name: Get changed C++ files id: changed_files - uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5 + uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 with: files: | **/*.cpp @@ -38,10 +39,17 @@ jobs: **/*.ipp separator: " " + - name: Get changed clang-tidy configuration + id: changed_clang_tidy + uses: tj-actions/changed-files@7dee1b0c1557f278e5c7dc244927139d78c0e22a # v47.0.4 + with: + files: | + .clang-tidy + run-clang-tidy: needs: [determine-files] - if: ${{ always() && !cancelled() && (!inputs.check_only_changed || needs.determine-files.outputs.any_changed == 'true') }} + if: ${{ always() && !cancelled() && (!inputs.check_only_changed || needs.determine-files.outputs.any_cpp_changed == 'true' || needs.determine-files.outputs.clang_tidy_config_changed == 'true') }} uses: ./.github/workflows/reusable-clang-tidy-files.yml with: - files: ${{ inputs.check_only_changed && needs.determine-files.outputs.all_changed_files || '' }} + files: ${{ (needs.determine-files.outputs.clang_tidy_config_changed == 'true' && '') || (inputs.check_only_changed && needs.determine-files.outputs.all_changed_files || '') }} create_issue_on_failure: ${{ inputs.create_issue_on_failure }} diff --git a/.github/workflows/reusable-strategy-matrix.yml b/.github/workflows/reusable-strategy-matrix.yml index c975347307..b1232a138f 100644 --- a/.github/workflows/reusable-strategy-matrix.yml +++ b/.github/workflows/reusable-strategy-matrix.yml @@ -29,10 +29,10 @@ jobs: matrix: ${{ steps.generate.outputs.matrix }} steps: - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: 3.13 diff --git a/.github/workflows/reusable-upload-recipe.yml b/.github/workflows/reusable-upload-recipe.yml index 79af516fb3..6245fd06e1 100644 --- a/.github/workflows/reusable-upload-recipe.yml +++ b/.github/workflows/reusable-upload-recipe.yml @@ -43,7 +43,7 @@ jobs: container: ghcr.io/xrplf/ci/ubuntu-noble:gcc-13-sha-5dd7158 steps: - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Generate build version number id: version diff --git a/.github/workflows/upload-conan-deps.yml b/.github/workflows/upload-conan-deps.yml index 60696a9769..df8aa43a18 100644 --- a/.github/workflows/upload-conan-deps.yml +++ b/.github/workflows/upload-conan-deps.yml @@ -64,10 +64,10 @@ jobs: steps: - name: Cleanup workspace (macOS and Windows) if: ${{ runner.os == 'macOS' || runner.os == 'Windows' }} - uses: XRPLF/actions/cleanup-workspace@cf0433aa74563aead044a1e395610c96d65a37cf + uses: XRPLF/actions/cleanup-workspace@c7d9ce5ebb03c752a354889ecd870cadfc2b1cd4 - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Prepare runner uses: XRPLF/actions/prepare-runner@2cbf481018d930656e9276fcc20dc0e3a0be5b6d diff --git a/.gitignore b/.gitignore index a1c2f034d1..60e8fef56c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,9 @@ gmon.out # Locally patched Conan recipes external/conan-center-index/ +# Local conan directory +.conan + # XCode IDE. *.pbxuser !default.pbxuser @@ -72,5 +75,8 @@ DerivedData /.claude /CLAUDE.md +# Direnv's directory +/.direnv + # clangd cache /.cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9117fe0d3e..6e04c752e9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -57,6 +57,16 @@ repos: - .git/COMMIT_EDITMSG stages: [commit-msg] + - repo: local + hooks: + - id: nix-fmt + name: Format Nix files + entry: nix --extra-experimental-features 'nix-command flakes' fmt + language: system + types: + - nix + pass_filenames: true + exclude: | (?x)^( external/.*| diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a928065ef2..4bb1db8689 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -251,6 +251,29 @@ pip3 install pre-commit pre-commit install ``` +## Clang-tidy + +All code must pass `clang-tidy` checks according to the settings in [`.clang-tidy`](./.clang-tidy). + +There is a Continuous Integration job that runs clang-tidy on pull requests. The CI will check: + +- All changed C++ files (`.cpp`, `.h`, `.ipp`) when only code files are modified +- **All files in the repository** when the `.clang-tidy` configuration file is changed + +This ensures that configuration changes don't introduce new warnings across the codebase. + +### Running clang-tidy locally + +Before running clang-tidy, you must build the project to generate required files (particularly protobuf headers). Refer to [`BUILD.md`](./BUILD.md) for build instructions. + +Then run clang-tidy on your local changes: + +``` +run-clang-tidy -p build src tests +``` + +This will check all source files in the `src` and `tests` directories using the compile commands from your `build` directory. + ## Contracts and instrumentation We are using [Antithesis](https://antithesis.com/) for continuous fuzzing, diff --git a/cspell.config.yaml b/cspell.config.yaml index 87258758c4..e2b20ac098 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -173,6 +173,9 @@ words: - nftokens - nftpage - nikb + - nixfmt + - nixos + - nixpkgs - nonxrp - noripple - nudb diff --git a/docs/build/environment.md b/docs/build/environment.md index c6b735ba48..c67877a082 100644 --- a/docs/build/environment.md +++ b/docs/build/environment.md @@ -3,6 +3,8 @@ environment complete with Git, Python, Conan, CMake, and a C++ compiler. This document exists to help readers set one up on any of the Big Three platforms: Linux, macOS, or Windows. +As an alternative to system packages, the Nix development shell can be used to provide a development environment. See [using nix development shell](./nix.md) for more details. + [BUILD.md]: ../../BUILD.md ## Linux diff --git a/docs/build/nix.md b/docs/build/nix.md new file mode 100644 index 0000000000..33bb3711d0 --- /dev/null +++ b/docs/build/nix.md @@ -0,0 +1,95 @@ +# Using Nix Development Shell for xrpld Development + +This guide explains how to use Nix to set up a reproducible development environment for xrpld. Using Nix eliminates the need to manually install utilities and ensures consistent tooling across different machines. + +## Benefits of Using Nix + +- **Reproducible environment**: Everyone gets the same versions of tools and compilers +- **No system pollution**: Dependencies are isolated and don't affect your system packages +- **Multiple compiler versions**: Easily switch between different GCC and Clang versions +- **Quick setup**: Get started with a single command +- **Works on Linux and macOS**: Consistent experience across platforms + +## Install Nix + +Please follow [the official installation instructions of nix package manager](https://nixos.org/download/) for your system. + +## Entering the Development Shell + +### Basic Usage + +From the root of the xrpld repository, enter the default development shell: + +```bash +nix --experimental-features 'nix-command flakes' develop +``` + +This will: + +- Download and set up all required development tools (CMake, Ninja, Conan, etc.) +- Configure the appropriate compiler for your platform: + - **macOS**: Apple Clang (default system compiler) + - **Linux**: GCC 15 + +The first time you run this command, it will take a few minutes to download and build the environment. Subsequent runs will be much faster. + +> [!TIP] +> To avoid typing `--experimental-features 'nix-command flakes'` every time, you can permanently enable flakes by creating `~/.config/nix/nix.conf`: +> +> ```bash +> mkdir -p ~/.config/nix +> echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf +> ``` +> +> After this, you can simply use `nix develop` instead. + +> [!NOTE] +> The examples below assume you've enabled flakes in your config. If you haven't, add `--experimental-features 'nix-command flakes'` after each `nix` command. + +### Choosing a different compiler + +A compiler can be chosen by providing its name with the `.#` prefix, e.g. `nix develop .#gcc15`. +Use `nix flake show` to see all the available development shells. + +Use `nix develop .#no_compiler` to use the compiler from your system. + +### Example Usage + +```bash +# Use GCC 14 +nix develop .#gcc14 + +# Use Clang 19 +nix develop .#clang19 + +# Use default for your platform +nix develop +``` + +### Using a different shell + +`nix develop` opens bash by default. If you want to use another shell this could be done by adding `-c` flag. For example: + +```bash +nix develop -c zsh +``` + +## Building xrpld with Nix + +Once inside the Nix development shell, follow the standard [build instructions](../../BUILD.md#steps). The Nix shell provides all necessary tools (CMake, Ninja, Conan, etc.). + +## Automatic Activation with direnv + +[direnv](https://direnv.net/) or [nix-direnv](https://github.com/nix-community/nix-direnv) can automatically activate the Nix development shell when you enter the repository directory. + +## Conan and Prebuilt Packages + +Please note that there is no guarantee that binaries from conan cache will work when using nix. If you encounter any errors, please use `--build '*'` to force conan to compile everything from source: + +```bash +conan install .. --output-folder . --build '*' --settings build_type=Release +``` + +## Updating `flake.lock` file + +To update `flake.lock` to the latest revision use `nix flake update` command. diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..fd43f5b683 --- /dev/null +++ b/flake.lock @@ -0,0 +1,26 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1769461804, + "narHash": "sha256-6h5sROT/3CTHvzPy9koKBmoCa2eJKh4fzQK8eYFEgl8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b579d443b37c9c5373044201ea77604e37e748c8", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-unstable", + "type": "indirect" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..4c500f1933 --- /dev/null +++ b/flake.nix @@ -0,0 +1,16 @@ +{ + description = "Nix related things for xrpld"; + inputs = { + nixpkgs.url = "nixpkgs/nixos-unstable"; + }; + + outputs = + { nixpkgs, ... }: + let + forEachSystem = (import ./nix/utils.nix { inherit nixpkgs; }).forEachSystem; + in + { + devShells = forEachSystem (import ./nix/devshell.nix); + formatter = forEachSystem ({ pkgs, ... }: pkgs.nixfmt); + }; +} diff --git a/include/xrpl/basics/MallocTrim.h b/include/xrpl/basics/MallocTrim.h new file mode 100644 index 0000000000..2d0cf989ba --- /dev/null +++ b/include/xrpl/basics/MallocTrim.h @@ -0,0 +1,73 @@ +#pragma once + +#include + +#include +#include +#include + +namespace xrpl { + +// cSpell:ignore ptmalloc + +// ----------------------------------------------------------------------------- +// Allocator interaction note: +// - This facility invokes glibc's malloc_trim(0) on Linux/glibc to request that +// ptmalloc return free heap pages to the OS. +// - If an alternative allocator (e.g. jemalloc or tcmalloc) is linked or +// preloaded (LD_PRELOAD), calling glibc's malloc_trim typically has no effect +// on the *active* heap. The call is harmless but may not reclaim memory +// because those allocators manage their own arenas. +// - Only glibc sbrk/arena space is eligible for trimming; large mmap-backed +// allocations are usually returned to the OS on free regardless of trimming. +// - Call at known reclamation points (e.g., after cache sweeps / online delete) +// and consider rate limiting to avoid churn. +// ----------------------------------------------------------------------------- + +struct MallocTrimReport +{ + bool supported{false}; + int trimResult{-1}; + std::int64_t rssBeforeKB{-1}; + std::int64_t rssAfterKB{-1}; + std::chrono::microseconds durationUs{-1}; + std::int64_t minfltDelta{-1}; + std::int64_t majfltDelta{-1}; + + [[nodiscard]] std::int64_t + deltaKB() const noexcept + { + if (rssBeforeKB < 0 || rssAfterKB < 0) + return 0; + return rssAfterKB - rssBeforeKB; + } +}; + +/** + * @brief Attempt to return freed memory to the operating system. + * + * On Linux with glibc malloc, this issues ::malloc_trim(0), which may release + * free space from ptmalloc arenas back to the kernel. On other platforms, or if + * a different allocator is in use, this function is a no-op and the report will + * indicate that trimming is unsupported or had no effect. + * + * @param tag Identifier for logging/debugging purposes. + * @param journal Journal for diagnostic logging. + * @return Report containing before/after metrics and the trim result. + * + * @note If an alternative allocator (jemalloc/tcmalloc) is linked or preloaded, + * calling glibc's malloc_trim may have no effect on the active heap. The + * call is harmless but typically does not reclaim memory under those + * allocators. + * + * @note Only memory served from glibc's sbrk/arena heaps is eligible for trim. + * Large allocations satisfied via mmap are usually returned on free + * independently of trimming. + * + * @note Intended for use after operations that free significant memory (e.g., + * cache sweeps, ledger cleanup, online delete). Consider rate limiting. + */ +MallocTrimReport +mallocTrim(std::string_view tag, beast::Journal journal); + +} // namespace xrpl diff --git a/include/xrpl/beast/container/aged_multiset.h b/include/xrpl/beast/container/aged_multiset.h index f55e7b34c1..8d906c694f 100644 --- a/include/xrpl/beast/container/aged_multiset.h +++ b/include/xrpl/beast/container/aged_multiset.h @@ -15,5 +15,4 @@ template < class Allocator = std::allocator> using aged_multiset = detail::aged_ordered_container; - } diff --git a/include/xrpl/beast/container/aged_unordered_map.h b/include/xrpl/beast/container/aged_unordered_map.h index 585fc6e5a4..520ffe5848 100644 --- a/include/xrpl/beast/container/aged_unordered_map.h +++ b/include/xrpl/beast/container/aged_unordered_map.h @@ -17,5 +17,4 @@ template < class Allocator = std::allocator>> using aged_unordered_map = detail::aged_unordered_container; - } diff --git a/include/xrpl/beast/container/aged_unordered_multimap.h b/include/xrpl/beast/container/aged_unordered_multimap.h index f2c31b4370..dc6338949b 100644 --- a/include/xrpl/beast/container/aged_unordered_multimap.h +++ b/include/xrpl/beast/container/aged_unordered_multimap.h @@ -17,5 +17,4 @@ template < class Allocator = std::allocator>> using aged_unordered_multimap = detail::aged_unordered_container; - } diff --git a/include/xrpl/beast/container/aged_unordered_set.h b/include/xrpl/beast/container/aged_unordered_set.h index e5879e5919..a1c032e159 100644 --- a/include/xrpl/beast/container/aged_unordered_set.h +++ b/include/xrpl/beast/container/aged_unordered_set.h @@ -16,5 +16,4 @@ template < class Allocator = std::allocator> using aged_unordered_set = detail::aged_unordered_container; - } diff --git a/include/xrpl/nodestore/Backend.h b/include/xrpl/nodestore/Backend.h index 7c3ea57bb8..36fd36ec00 100644 --- a/include/xrpl/nodestore/Backend.h +++ b/include/xrpl/nodestore/Backend.h @@ -77,16 +77,16 @@ public: If the object is not found or an error is encountered, the result will indicate the condition. @note This will be called concurrently. - @param key A pointer to the key data. + @param hash The hash of the object. @param pObject [out] The created object if successful. @return The result of the operation. */ virtual Status - fetch(void const* key, std::shared_ptr* pObject) = 0; + fetch(uint256 const& hash, std::shared_ptr* pObject) = 0; /** Fetch a batch synchronously. */ virtual std::pair>, Status> - fetchBatch(std::vector const& hashes) = 0; + fetchBatch(std::vector const& hashes) = 0; /** Store a single object. Depending on the implementation this may happen immediately diff --git a/include/xrpl/protocol/detail/features.macro b/include/xrpl/protocol/detail/features.macro index 133abdfc59..0579d03e7a 100644 --- a/include/xrpl/protocol/detail/features.macro +++ b/include/xrpl/protocol/detail/features.macro @@ -19,7 +19,7 @@ XRPL_FIX (DisallowIncomingV1_1, Supported::yes, VoteBehavior::DefaultNo) XRPL_FIX (PermissionedDomainInvariant, Supported::yes, VoteBehavior::DefaultNo) XRPL_FIX (ExpiredNFTokenOfferRemoval, Supported::yes, VoteBehavior::DefaultNo) -XRPL_FIX (BatchInnerSigs, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FIX (BatchInnerSigs, Supported::no, VoteBehavior::DefaultNo) XRPL_FEATURE(LendingProtocol, Supported::yes, VoteBehavior::DefaultNo) XRPL_FEATURE(PermissionDelegationV1_1, Supported::no, VoteBehavior::DefaultNo) XRPL_FIX (DirectoryLimit, Supported::yes, VoteBehavior::DefaultNo) @@ -33,7 +33,7 @@ XRPL_FEATURE(TokenEscrow, Supported::yes, VoteBehavior::DefaultNo XRPL_FIX (EnforceNFTokenTrustlineV2, Supported::yes, VoteBehavior::DefaultNo) XRPL_FIX (AMMv1_3, Supported::yes, VoteBehavior::DefaultNo) XRPL_FEATURE(PermissionedDEX, Supported::yes, VoteBehavior::DefaultNo) -XRPL_FEATURE(Batch, Supported::yes, VoteBehavior::DefaultNo) +XRPL_FEATURE(Batch, Supported::no, VoteBehavior::DefaultNo) XRPL_FEATURE(SingleAssetVault, Supported::yes, VoteBehavior::DefaultNo) XRPL_FIX (PayChanCancelAfter, Supported::yes, VoteBehavior::DefaultNo) // Check flags in Credential transactions diff --git a/nix/devshell.nix b/nix/devshell.nix new file mode 100644 index 0000000000..1d907f4d87 --- /dev/null +++ b/nix/devshell.nix @@ -0,0 +1,140 @@ +{ pkgs, ... }: +let + commonPackages = with pkgs; [ + ccache + cmake + conan + gcovr + git + gnumake + llvmPackages_21.clang-tools + ninja + perl # needed for openssl + pkg-config + pre-commit + python314 + ]; + + # Supported compiler versions + gccVersion = pkgs.lib.range 13 15; + clangVersions = pkgs.lib.range 18 21; + + defaultCompiler = if pkgs.stdenv.isDarwin then "apple-clang" else "gcc"; + defaultGccVersion = pkgs.lib.last gccVersion; + defaultClangVersion = pkgs.lib.last clangVersions; + + strToCompilerEnv = + compiler: version: + ( + if compiler == "gcc" then + let + gccPkg = pkgs."gcc${toString version}Stdenv" or null; + in + if gccPkg != null && builtins.elem version gccVersion then + gccPkg + else + throw "Invalid GCC version: ${toString version}. Must be one of: ${toString gccVersion}" + else if compiler == "clang" then + let + clangPkg = pkgs."llvmPackages_${toString version}".stdenv or null; + in + if clangPkg != null && builtins.elem version clangVersions then + clangPkg + else + throw "Invalid Clang version: ${toString version}. Must be one of: ${toString clangVersions}" + else if compiler == "apple-clang" || compiler == "none" then + pkgs.stdenvNoCC + else + throw "Invalid compiler: ${compiler}. Must be one of: gcc, clang, apple-clang, none" + ); + + # Helper function to create a shell with a specific compiler + makeShell = + { + compiler ? defaultCompiler, + version ? ( + if compiler == "gcc" then + defaultGccVersion + else if compiler == "clang" then + defaultClangVersion + else + null + ), + }: + let + compilerStdEnv = strToCompilerEnv compiler version; + + compilerName = + if compiler == "apple-clang" then + "clang" + else if compiler == "none" then + null + else + compiler; + + gccOnMacWarning = + if pkgs.stdenv.isDarwin && compiler == "gcc" then + '' + echo "WARNING: Using GCC on macOS with Conan may not work." + echo " Consider using 'nix develop .#clang' or the default shell instead." + echo "" + '' + else + ""; + + compilerVersion = + if compilerName != null then + '' + echo "Compiler: " + ${compilerName} --version + '' + else + '' + echo "No compiler specified - using system compiler" + ''; + + shellAttrs = { + packages = commonPackages; + + shellHook = '' + echo "Welcome to xrpld development shell"; + ${gccOnMacWarning}${compilerVersion} + ''; + }; + in + pkgs.mkShell.override { stdenv = compilerStdEnv; } shellAttrs; + + # Generate shells for each compiler version + gccShells = builtins.listToAttrs ( + map (version: { + name = "gcc${toString version}"; + value = makeShell { + compiler = "gcc"; + version = version; + }; + }) gccVersion + ); + + clangShells = builtins.listToAttrs ( + map (version: { + name = "clang${toString version}"; + value = makeShell { + compiler = "clang"; + version = version; + }; + }) clangVersions + ); + +in +gccShells +// clangShells +// { + # Default shells + default = makeShell { }; + gcc = makeShell { compiler = "gcc"; }; + clang = makeShell { compiler = "clang"; }; + + # No compiler + no-compiler = makeShell { compiler = "none"; }; + apple-clang = makeShell { compiler = "apple-clang"; }; +} diff --git a/nix/utils.nix b/nix/utils.nix new file mode 100644 index 0000000000..821d60a6f6 --- /dev/null +++ b/nix/utils.nix @@ -0,0 +1,19 @@ +{ nixpkgs }: +{ + forEachSystem = + function: + nixpkgs.lib.genAttrs + [ + "x86_64-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + ] + ( + system: + function { + inherit system; + pkgs = import nixpkgs { inherit system; }; + } + ); +} diff --git a/src/libxrpl/basics/MallocTrim.cpp b/src/libxrpl/basics/MallocTrim.cpp new file mode 100644 index 0000000000..1b0932b39d --- /dev/null +++ b/src/libxrpl/basics/MallocTrim.cpp @@ -0,0 +1,157 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include + +#if defined(__GLIBC__) && BOOST_OS_LINUX +#include + +#include +#include + +// Require RUSAGE_THREAD for thread-scoped page fault tracking +#ifndef RUSAGE_THREAD +#error "MallocTrim rusage instrumentation requires RUSAGE_THREAD on Linux/glibc" +#endif + +namespace { + +bool +getRusageThread(struct rusage& ru) +{ + return ::getrusage(RUSAGE_THREAD, &ru) == 0; // LCOV_EXCL_LINE +} + +} // namespace +#endif + +namespace xrpl { + +namespace detail { + +// cSpell:ignore statm + +#if defined(__GLIBC__) && BOOST_OS_LINUX + +inline int +mallocTrimWithPad(std::size_t padBytes) +{ + return ::malloc_trim(padBytes); +} + +long +parseStatmRSSkB(std::string const& statm) +{ + // /proc/self/statm format: size resident shared text lib data dt + // We want the second field (resident) which is in pages + std::istringstream iss(statm); + long size, resident; + if (!(iss >> size >> resident)) + return -1; + + // Convert pages to KB + long const pageSize = ::sysconf(_SC_PAGESIZE); + if (pageSize <= 0) + return -1; + + return (resident * pageSize) / 1024; +} + +#endif // __GLIBC__ && BOOST_OS_LINUX + +} // namespace detail + +MallocTrimReport +mallocTrim(std::string_view tag, beast::Journal journal) +{ + // LCOV_EXCL_START + + MallocTrimReport report; + +#if !(defined(__GLIBC__) && BOOST_OS_LINUX) + JLOG(journal.debug()) << "malloc_trim not supported on this platform (tag=" << tag << ")"; +#else + // Keep glibc malloc_trim padding at 0 (default): 12h Mainnet tests across 0/256KB/1MB/16MB + // showed no clear, consistent benefit from custom padding—0 provided the best overall balance + // of RSS reduction and trim-latency stability without adding a tuning surface. + constexpr std::size_t TRIM_PAD = 0; + + report.supported = true; + + if (journal.debug()) + { + auto readFile = [](std::string const& path) -> std::string { + std::ifstream ifs(path, std::ios::in | std::ios::binary); + if (!ifs.is_open()) + return {}; + + // /proc files are often not seekable; read as a stream. + std::ostringstream oss; + oss << ifs.rdbuf(); + return oss.str(); + }; + + std::string const tagStr{tag}; + std::string const statmPath = "/proc/self/statm"; + + auto const statmBefore = readFile(statmPath); + long const rssBeforeKB = detail::parseStatmRSSkB(statmBefore); + + struct rusage ru0{}; + bool const have_ru0 = getRusageThread(ru0); + + auto const t0 = std::chrono::steady_clock::now(); + + report.trimResult = detail::mallocTrimWithPad(TRIM_PAD); + + auto const t1 = std::chrono::steady_clock::now(); + + struct rusage ru1{}; + bool const have_ru1 = getRusageThread(ru1); + + auto const statmAfter = readFile(statmPath); + long const rssAfterKB = detail::parseStatmRSSkB(statmAfter); + + // Populate report fields + report.rssBeforeKB = rssBeforeKB; + report.rssAfterKB = rssAfterKB; + report.durationUs = std::chrono::duration_cast(t1 - t0); + + if (have_ru0 && have_ru1) + { + report.minfltDelta = ru1.ru_minflt - ru0.ru_minflt; + report.majfltDelta = ru1.ru_majflt - ru0.ru_majflt; + } + + std::int64_t const deltaKB = (rssBeforeKB < 0 || rssAfterKB < 0) + ? 0 + : (static_cast(rssAfterKB) - static_cast(rssBeforeKB)); + + JLOG(journal.debug()) << "malloc_trim tag=" << tagStr << " result=" << report.trimResult + << " pad=" << TRIM_PAD << " bytes" + << " rss_before=" << rssBeforeKB << "kB" + << " rss_after=" << rssAfterKB << "kB" + << " delta=" << deltaKB << "kB" + << " duration_us=" << report.durationUs.count() + << " minflt_delta=" << report.minfltDelta + << " majflt_delta=" << report.majfltDelta; + } + else + { + report.trimResult = detail::mallocTrimWithPad(TRIM_PAD); + } + +#endif + + return report; + + // LCOV_EXCL_STOP +} + +} // namespace xrpl diff --git a/src/libxrpl/nodestore/DatabaseNodeImp.cpp b/src/libxrpl/nodestore/DatabaseNodeImp.cpp index 5596cb4853..d1452dba86 100644 --- a/src/libxrpl/nodestore/DatabaseNodeImp.cpp +++ b/src/libxrpl/nodestore/DatabaseNodeImp.cpp @@ -33,7 +33,7 @@ DatabaseNodeImp::fetchNodeObject( try { - status = backend_->fetch(hash.data(), &nodeObject); + status = backend_->fetch(hash, &nodeObject); } catch (std::exception const& e) { @@ -68,18 +68,10 @@ DatabaseNodeImp::fetchBatch(std::vector const& hashes) using namespace std::chrono; auto const before = steady_clock::now(); - std::vector batch{}; - batch.reserve(hashes.size()); - for (size_t i = 0; i < hashes.size(); ++i) - { - auto const& hash = hashes[i]; - batch.push_back(&hash); - } - // Get the node objects that match the hashes from the backend. To protect // against the backends returning fewer or more results than expected, the // container is resized to the number of hashes. - auto results = backend_->fetchBatch(batch).first; + auto results = backend_->fetchBatch(hashes).first; XRPL_ASSERT( results.size() == hashes.size() || results.empty(), "number of output objects either matches number of input hashes or is empty"); diff --git a/src/libxrpl/nodestore/DatabaseRotatingImp.cpp b/src/libxrpl/nodestore/DatabaseRotatingImp.cpp index 26d8c30931..e058fa76ac 100644 --- a/src/libxrpl/nodestore/DatabaseRotatingImp.cpp +++ b/src/libxrpl/nodestore/DatabaseRotatingImp.cpp @@ -105,7 +105,7 @@ DatabaseRotatingImp::fetchNodeObject( std::shared_ptr nodeObject; try { - status = backend->fetch(hash.data(), &nodeObject); + status = backend->fetch(hash, &nodeObject); } catch (std::exception const& e) { diff --git a/src/libxrpl/nodestore/backend/MemoryFactory.cpp b/src/libxrpl/nodestore/backend/MemoryFactory.cpp index 8ac23a0bb6..b11d90610a 100644 --- a/src/libxrpl/nodestore/backend/MemoryFactory.cpp +++ b/src/libxrpl/nodestore/backend/MemoryFactory.cpp @@ -116,10 +116,9 @@ public: //-------------------------------------------------------------------------- Status - fetch(void const* key, std::shared_ptr* pObject) override + fetch(uint256 const& hash, std::shared_ptr* pObject) override { XRPL_ASSERT(db_, "xrpl::NodeStore::MemoryBackend::fetch : non-null database"); - uint256 const hash(uint256::fromVoid(key)); std::lock_guard _(db_->mutex); @@ -134,14 +133,14 @@ public: } std::pair>, Status> - fetchBatch(std::vector const& hashes) override + fetchBatch(std::vector const& hashes) override { std::vector> results; results.reserve(hashes.size()); for (auto const& h : hashes) { std::shared_ptr nObj; - Status status = fetch(h->begin(), &nObj); + Status status = fetch(h, &nObj); if (status != ok) results.push_back({}); else diff --git a/src/libxrpl/nodestore/backend/NuDBFactory.cpp b/src/libxrpl/nodestore/backend/NuDBFactory.cpp index e8efa464af..4d7e7be668 100644 --- a/src/libxrpl/nodestore/backend/NuDBFactory.cpp +++ b/src/libxrpl/nodestore/backend/NuDBFactory.cpp @@ -179,17 +179,17 @@ public: } Status - fetch(void const* key, std::shared_ptr* pno) override + fetch(uint256 const& hash, std::shared_ptr* pno) override { Status status; pno->reset(); nudb::error_code ec; db_.fetch( - key, - [key, pno, &status](void const* data, std::size_t size) { + hash.data(), + [&hash, pno, &status](void const* data, std::size_t size) { nudb::detail::buffer bf; auto const result = nodeobject_decompress(data, size, bf); - DecodedBlob decoded(key, result.first, result.second); + DecodedBlob decoded(hash.data(), result.first, result.second); if (!decoded.wasOk()) { status = dataCorrupt; @@ -207,14 +207,14 @@ public: } std::pair>, Status> - fetchBatch(std::vector const& hashes) override + fetchBatch(std::vector const& hashes) override { std::vector> results; results.reserve(hashes.size()); for (auto const& h : hashes) { std::shared_ptr nObj; - Status status = fetch(h->begin(), &nObj); + Status status = fetch(h, &nObj); if (status != ok) results.push_back({}); else diff --git a/src/libxrpl/nodestore/backend/NullFactory.cpp b/src/libxrpl/nodestore/backend/NullFactory.cpp index 4ecca46a9a..ab5b7d0117 100644 --- a/src/libxrpl/nodestore/backend/NullFactory.cpp +++ b/src/libxrpl/nodestore/backend/NullFactory.cpp @@ -36,13 +36,13 @@ public: } Status - fetch(void const*, std::shared_ptr*) override + fetch(uint256 const&, std::shared_ptr*) override { return notFound; } std::pair>, Status> - fetchBatch(std::vector const& hashes) override + fetchBatch(std::vector const& hashes) override { return {}; } diff --git a/src/libxrpl/nodestore/backend/RocksDBFactory.cpp b/src/libxrpl/nodestore/backend/RocksDBFactory.cpp index c84c5f6982..01bc74f5ed 100644 --- a/src/libxrpl/nodestore/backend/RocksDBFactory.cpp +++ b/src/libxrpl/nodestore/backend/RocksDBFactory.cpp @@ -244,7 +244,7 @@ public: //-------------------------------------------------------------------------- Status - fetch(void const* key, std::shared_ptr* pObject) override + fetch(uint256 const& hash, std::shared_ptr* pObject) override { XRPL_ASSERT(m_db, "xrpl::NodeStore::RocksDBBackend::fetch : non-null database"); pObject->reset(); @@ -252,7 +252,7 @@ public: Status status(ok); rocksdb::ReadOptions const options; - rocksdb::Slice const slice(static_cast(key), m_keyBytes); + rocksdb::Slice const slice(std::bit_cast(hash.data()), m_keyBytes); std::string string; @@ -260,7 +260,7 @@ public: if (getStatus.ok()) { - DecodedBlob decoded(key, string.data(), string.size()); + DecodedBlob decoded(hash.data(), string.data(), string.size()); if (decoded.wasOk()) { @@ -295,14 +295,14 @@ public: } std::pair>, Status> - fetchBatch(std::vector const& hashes) override + fetchBatch(std::vector const& hashes) override { std::vector> results; results.reserve(hashes.size()); for (auto const& h : hashes) { std::shared_ptr nObj; - Status status = fetch(h->begin(), &nObj); + Status status = fetch(h, &nObj); if (status != ok) results.push_back({}); else @@ -332,9 +332,8 @@ public: EncodedBlob encoded(e); wb.Put( - rocksdb::Slice(reinterpret_cast(encoded.getKey()), m_keyBytes), - rocksdb::Slice( - reinterpret_cast(encoded.getData()), encoded.getSize())); + rocksdb::Slice(std::bit_cast(encoded.getKey()), m_keyBytes), + rocksdb::Slice(std::bit_cast(encoded.getData()), encoded.getSize())); } rocksdb::WriteOptions const options; diff --git a/src/libxrpl/protocol/Keylet.cpp b/src/libxrpl/protocol/Keylet.cpp index 2c65bb9ed4..6f9656c4ea 100644 --- a/src/libxrpl/protocol/Keylet.cpp +++ b/src/libxrpl/protocol/Keylet.cpp @@ -9,7 +9,7 @@ bool Keylet::check(STLedgerEntry const& sle) const { XRPL_ASSERT( - sle.getType() != ltANY || sle.getType() != ltCHILD, + sle.getType() != ltANY && sle.getType() != ltCHILD, "xrpl::Keylet::check : valid input type"); if (type == ltANY) diff --git a/src/test/nodestore/TestBase.h b/src/test/nodestore/TestBase.h index 4a4d21002e..cb2a8e3bd5 100644 --- a/src/test/nodestore/TestBase.h +++ b/src/test/nodestore/TestBase.h @@ -138,7 +138,7 @@ public: { std::shared_ptr object; - Status const status = backend.fetch(batch[i]->getHash().cbegin(), &object); + Status const status = backend.fetch(batch[i]->getHash(), &object); BEAST_EXPECT(status == ok); @@ -158,7 +158,7 @@ public: { std::shared_ptr object; - Status const status = backend.fetch(batch[i]->getHash().cbegin(), &object); + Status const status = backend.fetch(batch[i]->getHash(), &object); BEAST_EXPECT(status == notFound); } diff --git a/src/test/nodestore/Timing_test.cpp b/src/test/nodestore/Timing_test.cpp index dae131e5e7..b537e3abb7 100644 --- a/src/test/nodestore/Timing_test.cpp +++ b/src/test/nodestore/Timing_test.cpp @@ -314,7 +314,7 @@ public: std::shared_ptr obj; std::shared_ptr result; obj = seq1_.obj(dist_(gen_)); - backend_.fetch(obj->getHash().data(), &result); + backend_.fetch(obj->getHash(), &result); suite_.expect(result && isSame(result, obj)); } catch (std::exception const& e) @@ -377,9 +377,9 @@ public: { try { - auto const key = seq2_.key(i); + auto const hash = seq2_.key(i); std::shared_ptr result; - backend_.fetch(key.data(), &result); + backend_.fetch(hash, &result); suite_.expect(!result); } catch (std::exception const& e) @@ -449,9 +449,9 @@ public: { if (rand_(gen_) < missingNodePercent) { - auto const key = seq2_.key(dist_(gen_)); + auto const hash = seq2_.key(dist_(gen_)); std::shared_ptr result; - backend_.fetch(key.data(), &result); + backend_.fetch(hash, &result); suite_.expect(!result); } else @@ -459,7 +459,7 @@ public: std::shared_ptr obj; std::shared_ptr result; obj = seq1_.obj(dist_(gen_)); - backend_.fetch(obj->getHash().data(), &result); + backend_.fetch(obj->getHash(), &result); suite_.expect(result && isSame(result, obj)); } } @@ -540,8 +540,7 @@ public: std::shared_ptr result; auto const j = older_(gen_); obj = seq1_.obj(j); - std::shared_ptr result1; - backend_.fetch(obj->getHash().data(), &result); + backend_.fetch(obj->getHash(), &result); suite_.expect(result != nullptr); suite_.expect(isSame(result, obj)); } @@ -559,7 +558,7 @@ public: std::shared_ptr result; auto const j = recent_(gen_); obj = seq1_.obj(j); - backend_.fetch(obj->getHash().data(), &result); + backend_.fetch(obj->getHash(), &result); suite_.expect(!result || isSame(result, obj)); break; } diff --git a/src/tests/libxrpl/basics/MallocTrim.cpp b/src/tests/libxrpl/basics/MallocTrim.cpp new file mode 100644 index 0000000000..f01bd91bbf --- /dev/null +++ b/src/tests/libxrpl/basics/MallocTrim.cpp @@ -0,0 +1,209 @@ +#include + +#include + +#include + +using namespace xrpl; + +// cSpell:ignore statm + +#if defined(__GLIBC__) && BOOST_OS_LINUX +namespace xrpl::detail { +long +parseStatmRSSkB(std::string const& statm); +} // namespace xrpl::detail +#endif + +TEST(MallocTrimReport, structure) +{ + // Test default construction + MallocTrimReport report; + EXPECT_EQ(report.supported, false); + EXPECT_EQ(report.trimResult, -1); + EXPECT_EQ(report.rssBeforeKB, -1); + EXPECT_EQ(report.rssAfterKB, -1); + EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1}); + EXPECT_EQ(report.minfltDelta, -1); + EXPECT_EQ(report.majfltDelta, -1); + EXPECT_EQ(report.deltaKB(), 0); + + // Test deltaKB calculation - memory freed + report.rssBeforeKB = 1000; + report.rssAfterKB = 800; + EXPECT_EQ(report.deltaKB(), -200); + + // Test deltaKB calculation - memory increased + report.rssBeforeKB = 500; + report.rssAfterKB = 600; + EXPECT_EQ(report.deltaKB(), 100); + + // Test deltaKB calculation - no change + report.rssBeforeKB = 1234; + report.rssAfterKB = 1234; + EXPECT_EQ(report.deltaKB(), 0); +} + +#if defined(__GLIBC__) && BOOST_OS_LINUX +TEST(parseStatmRSSkB, standard_format) +{ + using xrpl::detail::parseStatmRSSkB; + + // Test standard format: size resident shared text lib data dt + // Assuming 4KB page size: resident=1000 pages = 4000 KB + { + std::string statm = "25365 1000 2377 0 0 5623 0"; + long result = parseStatmRSSkB(statm); + // Note: actual result depends on system page size + // On most systems it's 4KB, so 1000 pages = 4000 KB + EXPECT_GT(result, 0); + } + + // Test with newline + { + std::string statm = "12345 2000 1234 0 0 3456 0\n"; + long result = parseStatmRSSkB(statm); + EXPECT_GT(result, 0); + } + + // Test with tabs + { + std::string statm = "12345\t2000\t1234\t0\t0\t3456\t0"; + long result = parseStatmRSSkB(statm); + EXPECT_GT(result, 0); + } + + // Test zero resident pages + { + std::string statm = "25365 0 2377 0 0 5623 0"; + long result = parseStatmRSSkB(statm); + EXPECT_EQ(result, 0); + } + + // Test with extra whitespace + { + std::string statm = " 25365 1000 2377 "; + long result = parseStatmRSSkB(statm); + EXPECT_GT(result, 0); + } + + // Test empty string + { + std::string statm = ""; + long result = parseStatmRSSkB(statm); + EXPECT_EQ(result, -1); + } + + // Test malformed data (only one field) + { + std::string statm = "25365"; + long result = parseStatmRSSkB(statm); + EXPECT_EQ(result, -1); + } + + // Test malformed data (non-numeric) + { + std::string statm = "abc def ghi"; + long result = parseStatmRSSkB(statm); + EXPECT_EQ(result, -1); + } + + // Test malformed data (second field non-numeric) + { + std::string statm = "25365 abc 2377"; + long result = parseStatmRSSkB(statm); + EXPECT_EQ(result, -1); + } +} +#endif + +TEST(mallocTrim, without_debug_logging) +{ + beast::Journal journal{beast::Journal::getNullSink()}; + + MallocTrimReport report = mallocTrim("without_debug", journal); + +#if defined(__GLIBC__) && BOOST_OS_LINUX + EXPECT_EQ(report.supported, true); + EXPECT_GE(report.trimResult, 0); + EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1}); + EXPECT_EQ(report.minfltDelta, -1); + EXPECT_EQ(report.majfltDelta, -1); +#else + EXPECT_EQ(report.supported, false); + EXPECT_EQ(report.trimResult, -1); + EXPECT_EQ(report.rssBeforeKB, -1); + EXPECT_EQ(report.rssAfterKB, -1); + EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1}); + EXPECT_EQ(report.minfltDelta, -1); + EXPECT_EQ(report.majfltDelta, -1); +#endif +} + +TEST(mallocTrim, empty_tag) +{ + beast::Journal journal{beast::Journal::getNullSink()}; + MallocTrimReport report = mallocTrim("", journal); + +#if defined(__GLIBC__) && BOOST_OS_LINUX + EXPECT_EQ(report.supported, true); + EXPECT_GE(report.trimResult, 0); +#else + EXPECT_EQ(report.supported, false); +#endif +} + +TEST(mallocTrim, with_debug_logging) +{ + struct DebugSink : public beast::Journal::Sink + { + DebugSink() : Sink(beast::severities::kDebug, false) + { + } + void + write(beast::severities::Severity, std::string const&) override + { + } + void + writeAlways(beast::severities::Severity, std::string const&) override + { + } + }; + + DebugSink sink; + beast::Journal journal{sink}; + + MallocTrimReport report = mallocTrim("debug_test", journal); + +#if defined(__GLIBC__) && BOOST_OS_LINUX + EXPECT_EQ(report.supported, true); + EXPECT_GE(report.trimResult, 0); + EXPECT_GE(report.durationUs.count(), 0); + EXPECT_GE(report.minfltDelta, 0); + EXPECT_GE(report.majfltDelta, 0); +#else + EXPECT_EQ(report.supported, false); + EXPECT_EQ(report.trimResult, -1); + EXPECT_EQ(report.durationUs, std::chrono::microseconds{-1}); + EXPECT_EQ(report.minfltDelta, -1); + EXPECT_EQ(report.majfltDelta, -1); +#endif +} + +TEST(mallocTrim, repeated_calls) +{ + beast::Journal journal{beast::Journal::getNullSink()}; + + // Call malloc_trim multiple times to ensure it's safe + for (int i = 0; i < 5; ++i) + { + MallocTrimReport report = mallocTrim("iteration_" + std::to_string(i), journal); + +#if defined(__GLIBC__) && BOOST_OS_LINUX + EXPECT_EQ(report.supported, true); + EXPECT_GE(report.trimResult, 0); +#else + EXPECT_EQ(report.supported, false); +#endif + } +} diff --git a/src/xrpld/app/main/Application.cpp b/src/xrpld/app/main/Application.cpp index 91cc387d54..1162bc497a 100644 --- a/src/xrpld/app/main/Application.cpp +++ b/src/xrpld/app/main/Application.cpp @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -1053,6 +1054,8 @@ public: << "; size after: " << cachedSLEs_.size(); } + mallocTrim("doSweep", m_journal); + // Set timer to do another sweep later. setSweepTimer(); }