diff --git a/.github/actions/build_clio/action.yml b/.github/actions/build-clio/action.yml similarity index 80% rename from .github/actions/build_clio/action.yml rename to .github/actions/build-clio/action.yml index cc2381a4..e3acd56e 100644 --- a/.github/actions/build_clio/action.yml +++ b/.github/actions/build-clio/action.yml @@ -6,7 +6,7 @@ inputs: description: Space-separated build target names default: all subtract_threads: - description: An option for the action get_number_of_threads. See get_number_of_threads + description: An option for the action get-threads-number. required: true default: "0" @@ -14,7 +14,7 @@ runs: using: composite steps: - name: Get number of threads - uses: ./.github/actions/get_number_of_threads + uses: ./.github/actions/get-threads-number id: number_of_threads with: subtract_threads: ${{ inputs.subtract_threads }} diff --git a/.github/actions/build_docker_image/action.yml b/.github/actions/build-docker-image/action.yml similarity index 92% rename from .github/actions/build_docker_image/action.yml rename to .github/actions/build-docker-image/action.yml index 29154f0e..d946fd28 100644 --- a/.github/actions/build_docker_image/action.yml +++ b/.github/actions/build-docker-image/action.yml @@ -34,14 +34,14 @@ runs: steps: - name: Login to DockerHub if: ${{ inputs.push_image == 'true' && inputs.dockerhub_repo != '' }} - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: username: ${{ env.DOCKERHUB_USER }} password: ${{ env.DOCKERHUB_PW }} - name: Login to GitHub Container Registry if: ${{ inputs.push_image == 'true' }} - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} diff --git a/.github/actions/code_coverage/action.yml b/.github/actions/code-coverage/action.yml similarity index 100% rename from .github/actions/code_coverage/action.yml rename to .github/actions/code-coverage/action.yml diff --git a/.github/actions/create_issue/action.yml b/.github/actions/create-issue/action.yml similarity index 100% rename from .github/actions/create_issue/action.yml rename to .github/actions/create-issue/action.yml diff --git a/.github/actions/get_number_of_threads/action.yml b/.github/actions/get-threads-number/action.yml similarity index 100% rename from .github/actions/get_number_of_threads/action.yml rename to .github/actions/get-threads-number/action.yml diff --git a/.github/actions/git_common_ancestor/action.yml b/.github/actions/git-common-ancestor/action.yml similarity index 100% rename from .github/actions/git_common_ancestor/action.yml rename to .github/actions/git-common-ancestor/action.yml diff --git a/.github/actions/restore_cache/action.yml b/.github/actions/restore-cache/action.yml similarity index 95% rename from .github/actions/restore_cache/action.yml rename to .github/actions/restore-cache/action.yml index 1cfb68f3..7f67e022 100644 --- a/.github/actions/restore_cache/action.yml +++ b/.github/actions/restore-cache/action.yml @@ -27,7 +27,7 @@ runs: steps: - name: Find common commit id: git_common_ancestor - uses: ./.github/actions/git_common_ancestor + uses: ./.github/actions/git-common-ancestor - name: Restore ccache cache uses: actions/cache/restore@v4 diff --git a/.github/actions/save_cache/action.yml b/.github/actions/save-cache/action.yml similarity index 95% rename from .github/actions/save_cache/action.yml rename to .github/actions/save-cache/action.yml index 3711f802..dff9e32a 100644 --- a/.github/actions/save_cache/action.yml +++ b/.github/actions/save-cache/action.yml @@ -28,7 +28,7 @@ runs: steps: - name: Find common commit id: git_common_ancestor - uses: ./.github/actions/git_common_ancestor + uses: ./.github/actions/git-common-ancestor - name: Save ccache cache if: ${{ inputs.ccache_cache_hit != 'true' || inputs.ccache_cache_miss_rate == '100.0' }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e55174ce..24d44894 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,7 +14,7 @@ updates: target-branch: develop - package-ecosystem: github-actions - directory: .github/actions/build_clio/ + directory: .github/actions/build-clio/ schedule: interval: weekly day: monday @@ -27,7 +27,7 @@ updates: target-branch: develop - package-ecosystem: github-actions - directory: .github/actions/build_docker_image/ + directory: .github/actions/build-docker-image/ schedule: interval: weekly day: monday @@ -53,7 +53,7 @@ updates: target-branch: develop - package-ecosystem: github-actions - directory: .github/actions/code_coverage/ + directory: .github/actions/code-coverage/ schedule: interval: weekly day: monday @@ -79,7 +79,7 @@ updates: target-branch: develop - package-ecosystem: github-actions - directory: .github/actions/create_issue/ + directory: .github/actions/create-issue/ schedule: interval: weekly day: monday @@ -92,7 +92,7 @@ updates: target-branch: develop - package-ecosystem: github-actions - directory: .github/actions/get_number_of_threads/ + directory: .github/actions/get-threads-number/ schedule: interval: weekly day: monday @@ -105,7 +105,7 @@ updates: target-branch: develop - package-ecosystem: github-actions - directory: .github/actions/git_common_ancestor/ + directory: .github/actions/git-common-ancestor/ schedule: interval: weekly day: monday @@ -118,7 +118,7 @@ updates: target-branch: develop - package-ecosystem: github-actions - directory: .github/actions/restore_cache/ + directory: .github/actions/restore-cache/ schedule: interval: weekly day: monday @@ -131,7 +131,7 @@ updates: target-branch: develop - package-ecosystem: github-actions - directory: .github/actions/save_cache/ + directory: .github/actions/save-cache/ schedule: interval: weekly day: monday diff --git a/.github/scripts/conan/generate_matrix.py b/.github/scripts/conan/generate_matrix.py index 3b3d111c..699b9d29 100755 --- a/.github/scripts/conan/generate_matrix.py +++ b/.github/scripts/conan/generate_matrix.py @@ -3,7 +3,9 @@ import itertools import json LINUX_OS = ["heavy", "heavy-arm64"] -LINUX_CONTAINERS = ['{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }'] +LINUX_CONTAINERS = [ + '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' +] LINUX_COMPILERS = ["gcc", "clang"] MACOS_OS = ["macos15"] diff --git a/.github/workflows/build_clio_docker_image.yml b/.github/workflows/build-clio-docker-image.yml similarity index 98% rename from .github/workflows/build_clio_docker_image.yml rename to .github/workflows/build-clio-docker-image.yml index f399ca0d..4c5922b3 100644 --- a/.github/workflows/build_clio_docker_image.yml +++ b/.github/workflows/build-clio-docker-image.yml @@ -89,7 +89,7 @@ jobs: echo "GHCR_REPO=$(echo ghcr.io/${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> ${GITHUB_OUTPUT} - name: Build Docker image - uses: ./.github/actions/build_docker_image + uses: ./.github/actions/build-docker-image env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 98cc252a..604c965c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,14 +8,14 @@ on: paths: - .github/workflows/build.yml - - .github/workflows/build_and_test.yml - - .github/workflows/build_impl.yml - - .github/workflows/test_impl.yml - - .github/workflows/upload_coverage_report.yml + - .github/workflows/reusable-build-test.yml + - .github/workflows/reusable-build.yml + - .github/workflows/reusable-test.yml + - .github/workflows/reusable-upload-coverage-report.yml - ".github/actions/**" - - "!.github/actions/build_docker_image/**" - - "!.github/actions/create_issue/**" + - "!.github/actions/build-docker-image/**" + - "!.github/actions/create-issue/**" - CMakeLists.txt - conanfile.py @@ -45,7 +45,7 @@ jobs: build_type: [Release, Debug] container: [ - '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }', + '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }', ] static: [true] @@ -56,7 +56,7 @@ jobs: container: "" static: false - uses: ./.github/workflows/build_and_test.yml + uses: ./.github/workflows/reusable-build-test.yml with: runs_on: ${{ matrix.os }} container: ${{ matrix.container }} @@ -72,10 +72,10 @@ jobs: code_coverage: name: Run Code Coverage - uses: ./.github/workflows/build_impl.yml + uses: ./.github/workflows/reusable-build.yml with: runs_on: heavy - container: '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' conan_profile: gcc build_type: Debug download_ccache: true @@ -91,10 +91,10 @@ jobs: package: name: Build packages - uses: ./.github/workflows/build_impl.yml + uses: ./.github/workflows/reusable-build.yml with: runs_on: heavy - container: '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' conan_profile: gcc build_type: Release download_ccache: true @@ -111,7 +111,7 @@ jobs: needs: build-and-test runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/check_libxrpl.yml b/.github/workflows/check-libxrpl.yml similarity index 91% rename from .github/workflows/check_libxrpl.yml rename to .github/workflows/check-libxrpl.yml index ee1b36c2..fe5c5b6b 100644 --- a/.github/workflows/check_libxrpl.yml +++ b/.github/workflows/check-libxrpl.yml @@ -17,7 +17,7 @@ jobs: name: Build Clio / `libXRPL ${{ github.event.client_payload.version }}` runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e steps: - uses: actions/checkout@v4 @@ -51,7 +51,7 @@ jobs: conan_profile: ${{ env.CONAN_PROFILE }} - name: Build Clio - uses: ./.github/actions/build_clio + uses: ./.github/actions/build-clio - name: Strip tests run: strip build/clio_tests @@ -67,7 +67,7 @@ jobs: needs: build runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e steps: - uses: actions/download-artifact@v5 @@ -93,7 +93,7 @@ jobs: - uses: actions/checkout@v4 - name: Create an issue - uses: ./.github/actions/create_issue + uses: ./.github/actions/create-issue env: GH_TOKEN: ${{ github.token }} with: diff --git a/.github/workflows/check_pr_title.yml b/.github/workflows/check-pr-title.yml similarity index 70% rename from .github/workflows/check_pr_title.yml rename to .github/workflows/check-pr-title.yml index 693faade..07e3e945 100644 --- a/.github/workflows/check_pr_title.yml +++ b/.github/workflows/check-pr-title.yml @@ -15,3 +15,10 @@ jobs: task_types: '["build","feat","fix","docs","test","ci","style","refactor","perf","chore"]' add_label: false custom_labels: '{"build":"build", "feat":"enhancement", "fix":"bug", "docs":"documentation", "test":"testability", "ci":"ci", "style":"refactoring", "refactor":"refactoring", "perf":"performance", "chore":"tooling"}' + + - name: Check if message starts with upper-case letter + run: | + if [[ ! "${{ github.event.pull_request.title }}" =~ ^[a-z]+:\ [\[A-Z] ]]; then + echo "Error: PR title must start with an upper-case letter." + exit 1 + fi diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 855b0a6d..fb336703 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -27,7 +27,7 @@ jobs: if: github.event_name != 'push' || contains(github.event.head_commit.message, 'clang-tidy auto fixes') runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e permissions: contents: write @@ -45,7 +45,7 @@ jobs: disable_ccache: true - name: Restore cache - uses: ./.github/actions/restore_cache + uses: ./.github/actions/restore-cache id: restore_cache with: conan_profile: ${{ env.CONAN_PROFILE }} @@ -62,7 +62,7 @@ jobs: conan_profile: ${{ env.CONAN_PROFILE }} - name: Get number of threads - uses: ./.github/actions/get_number_of_threads + uses: ./.github/actions/get-threads-number id: number_of_threads - name: Run clang-tidy @@ -90,7 +90,7 @@ jobs: - name: Create an issue if: ${{ steps.run_clang_tidy.outcome != 'success' && github.event_name != 'pull_request' }} id: create_issue - uses: ./.github/actions/create_issue + uses: ./.github/actions/create-issue env: GH_TOKEN: ${{ github.token }} with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f0f2142e..9d2bf4d1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,7 +14,7 @@ jobs: build: runs-on: ubuntu-latest container: - image: ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e steps: - name: Checkout diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 27f2d4fd..fb2beceb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -8,14 +8,14 @@ on: paths: - .github/workflows/nightly.yml - - .github/workflows/release_impl.yml - - .github/workflows/build_and_test.yml - - .github/workflows/build_impl.yml - - .github/workflows/test_impl.yml - - .github/workflows/build_clio_docker_image.yml + - .github/workflows/reusable-release.yml + - .github/workflows/reusable-build-test.yml + - .github/workflows/reusable-build.yml + - .github/workflows/reusable-test.yml + - .github/workflows/build-clio-docker-image.yml - ".github/actions/**" - - "!.github/actions/code_coverage/**" + - "!.github/actions/code-coverage/**" - .github/scripts/prepare-release-artifacts.sh concurrency: @@ -39,19 +39,19 @@ jobs: conan_profile: gcc build_type: Release static: true - container: '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' - os: heavy conan_profile: gcc build_type: Debug static: true - container: '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' - os: heavy conan_profile: gcc.ubsan build_type: Release static: false - container: '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' - uses: ./.github/workflows/build_and_test.yml + uses: ./.github/workflows/reusable-build-test.yml with: runs_on: ${{ matrix.os }} container: ${{ matrix.container }} @@ -73,13 +73,13 @@ jobs: include: - os: heavy conan_profile: clang - container: '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' static: true - os: macos15 conan_profile: apple-clang container: "" static: false - uses: ./.github/workflows/build_impl.yml + uses: ./.github/workflows/reusable-build.yml with: runs_on: ${{ matrix.os }} container: ${{ matrix.container }} @@ -95,7 +95,7 @@ jobs: nightly_release: needs: build-and-test - uses: ./.github/workflows/release_impl.yml + uses: ./.github/workflows/reusable-release.yml with: overwrite_release: true prerelease: true @@ -109,7 +109,7 @@ jobs: draft: false build_and_publish_docker_image: - uses: ./.github/workflows/build_clio_docker_image.yml + uses: ./.github/workflows/build-clio-docker-image.yml needs: build-and-test secrets: inherit with: @@ -133,7 +133,7 @@ jobs: - uses: actions/checkout@v4 - name: Create an issue - uses: ./.github/actions/create_issue + uses: ./.github/actions/create-issue env: GH_TOKEN: ${{ github.token }} with: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 770d1b6d..69cfeefe 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -11,4 +11,4 @@ jobs: uses: XRPLF/actions/.github/workflows/pre-commit.yml@afbcbdafbe0ce5439492fb87eda6441371086386 with: runs_on: heavy - container: '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }' + container: '{ "image": "ghcr.io/xrplf/clio-pre-commit:213752862ca95ecadeb59a6176c3db91a7864b3e" }' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 479e1fe3..9cb099ca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,9 +29,9 @@ jobs: conan_profile: gcc build_type: Release static: true - container: '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' - uses: ./.github/workflows/build_and_test.yml + uses: ./.github/workflows/reusable-build-test.yml with: runs_on: ${{ matrix.os }} container: ${{ matrix.container }} @@ -47,7 +47,7 @@ jobs: release: needs: build-and-test - uses: ./.github/workflows/release_impl.yml + uses: ./.github/workflows/reusable-release.yml with: overwrite_release: false prerelease: ${{ contains(github.ref_name, '-') }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/reusable-build-test.yml similarity index 96% rename from .github/workflows/build_and_test.yml rename to .github/workflows/reusable-build-test.yml index 59b6d5b7..bf9d8251 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/reusable-build-test.yml @@ -77,7 +77,7 @@ on: jobs: build: - uses: ./.github/workflows/build_impl.yml + uses: ./.github/workflows/reusable-build.yml with: runs_on: ${{ inputs.runs_on }} container: ${{ inputs.container }} @@ -95,7 +95,7 @@ jobs: test: needs: build - uses: ./.github/workflows/test_impl.yml + uses: ./.github/workflows/reusable-test.yml with: runs_on: ${{ inputs.runs_on }} container: ${{ inputs.container }} diff --git a/.github/workflows/build_impl.yml b/.github/workflows/reusable-build.yml similarity index 96% rename from .github/workflows/build_impl.yml rename to .github/workflows/reusable-build.yml index aa9aae8e..04e11177 100644 --- a/.github/workflows/build_impl.yml +++ b/.github/workflows/reusable-build.yml @@ -106,7 +106,7 @@ jobs: - name: Restore cache if: ${{ inputs.download_ccache }} - uses: ./.github/actions/restore_cache + uses: ./.github/actions/restore-cache id: restore_cache with: conan_profile: ${{ inputs.conan_profile }} @@ -131,7 +131,7 @@ jobs: package: ${{ inputs.package }} - name: Build Clio - uses: ./.github/actions/build_clio + uses: ./.github/actions/build-clio with: targets: ${{ inputs.targets }} @@ -198,7 +198,7 @@ jobs: - name: Save cache if: ${{ inputs.upload_ccache && github.ref == 'refs/heads/develop' }} - uses: ./.github/actions/save_cache + uses: ./.github/actions/save-cache with: conan_profile: ${{ inputs.conan_profile }} ccache_dir: ${{ env.CCACHE_DIR }} @@ -216,7 +216,7 @@ jobs: # It's all available in the build job, but not in the test job - name: Run code coverage if: ${{ inputs.code_coverage }} - uses: ./.github/actions/code_coverage + uses: ./.github/actions/code-coverage - name: Verify expected version if: ${{ inputs.expected_version != '' }} @@ -238,6 +238,6 @@ jobs: if: ${{ inputs.code_coverage }} name: Codecov needs: build - uses: ./.github/workflows/upload_coverage_report.yml + uses: ./.github/workflows/reusable-upload-coverage-report.yml secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/release_impl.yml b/.github/workflows/reusable-release.yml similarity index 97% rename from .github/workflows/release_impl.yml rename to .github/workflows/reusable-release.yml index 784be356..e4cd3da8 100644 --- a/.github/workflows/release_impl.yml +++ b/.github/workflows/reusable-release.yml @@ -42,7 +42,7 @@ jobs: release: runs-on: heavy container: - image: ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/test_impl.yml b/.github/workflows/reusable-test.yml similarity index 99% rename from .github/workflows/test_impl.yml rename to .github/workflows/reusable-test.yml index 375d22bf..d7c6e0c3 100644 --- a/.github/workflows/test_impl.yml +++ b/.github/workflows/reusable-test.yml @@ -91,7 +91,7 @@ jobs: - name: Create an issue if: ${{ false && env.SANITIZER_IGNORE_ERRORS == 'true' && steps.check_report.outputs.found_report == 'true' }} - uses: ./.github/actions/create_issue + uses: ./.github/actions/create-issue env: GH_TOKEN: ${{ github.token }} with: diff --git a/.github/workflows/upload_coverage_report.yml b/.github/workflows/reusable-upload-coverage-report.yml similarity index 97% rename from .github/workflows/upload_coverage_report.yml rename to .github/workflows/reusable-upload-coverage-report.yml index 86a4eaa7..b42f7161 100644 --- a/.github/workflows/upload_coverage_report.yml +++ b/.github/workflows/reusable-upload-coverage-report.yml @@ -1,7 +1,6 @@ name: Upload report on: - workflow_dispatch: workflow_call: secrets: CODECOV_TOKEN: diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index e0aad56d..068c971e 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -8,13 +8,13 @@ on: paths: - .github/workflows/sanitizers.yml - - .github/workflows/build_and_test.yml - - .github/workflows/build_impl.yml - - .github/workflows/test_impl.yml + - .github/workflows/reusable-build-test.yml + - .github/workflows/reusable-build.yml + - .github/workflows/reusable-test.yml - ".github/actions/**" - - "!.github/actions/build_docker_image/**" - - "!.github/actions/create_issue/**" + - "!.github/actions/build-docker-image/**" + - "!.github/actions/create-issue/**" - .github/scripts/execute-tests-under-sanitizer - CMakeLists.txt @@ -41,10 +41,10 @@ jobs: sanitizer_ext: [.asan, .tsan, .ubsan] build_type: [Release, Debug] - uses: ./.github/workflows/build_and_test.yml + uses: ./.github/workflows/reusable-build-test.yml with: runs_on: heavy - container: '{ "image": "ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e" }' download_ccache: false upload_ccache: false conan_profile: ${{ matrix.compiler }}${{ matrix.sanitizer_ext }} diff --git a/.github/workflows/update_docker_ci.yml b/.github/workflows/update-docker-ci.yml similarity index 85% rename from .github/workflows/update_docker_ci.yml rename to .github/workflows/update-docker-ci.yml index 584ce2fa..4ef3f93e 100644 --- a/.github/workflows/update_docker_ci.yml +++ b/.github/workflows/update-docker-ci.yml @@ -3,23 +3,23 @@ name: Update CI docker image on: pull_request: paths: - - .github/workflows/update_docker_ci.yml + - .github/workflows/update-docker-ci.yml - - ".github/actions/build_docker_image/**" + - ".github/actions/build-docker-image/**" - - "docker/ci/**" - - "docker/compilers/**" - - "docker/tools/**" + - "docker/**" + - "!docker/clio/**" + - "!docker/develop/**" push: branches: [develop] paths: - - .github/workflows/update_docker_ci.yml + - .github/workflows/update-docker-ci.yml - - ".github/actions/build_docker_image/**" + - ".github/actions/build-docker-image/**" - - "docker/ci/**" - - "docker/compilers/**" - - "docker/tools/**" + - "docker/**" + - "!docker/clio/**" + - "!docker/develop/**" workflow_dispatch: concurrency: @@ -60,7 +60,7 @@ jobs: with: files: "docker/compilers/gcc/**" - - uses: ./.github/actions/build_docker_image + - uses: ./.github/actions/build-docker-image if: ${{ steps.changed-files.outputs.any_changed == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -94,11 +94,11 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5 with: files: "docker/compilers/gcc/**" - - uses: ./.github/actions/build_docker_image + - uses: ./.github/actions/build-docker-image if: ${{ steps.changed-files.outputs.any_changed == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -141,7 +141,7 @@ jobs: - name: Login to GitHub Container Registry if: ${{ github.event_name != 'pull_request' }} - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -149,7 +149,7 @@ jobs: - name: Login to DockerHub if: ${{ github.repository_owner == 'XRPLF' && github.event_name != 'pull_request' }} - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_PW }} @@ -187,7 +187,7 @@ jobs: with: files: "docker/compilers/clang/**" - - uses: ./.github/actions/build_docker_image + - uses: ./.github/actions/build-docker-image if: ${{ steps.changed-files.outputs.any_changed == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -223,7 +223,7 @@ jobs: with: files: "docker/tools/**" - - uses: ./.github/actions/build_docker_image + - uses: ./.github/actions/build-docker-image if: ${{ steps.changed-files.outputs.any_changed == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -250,11 +250,11 @@ jobs: - name: Get changed files id: changed-files - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 + uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5 with: files: "docker/tools/**" - - uses: ./.github/actions/build_docker_image + - uses: ./.github/actions/build-docker-image if: ${{ steps.changed-files.outputs.any_changed == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -290,7 +290,7 @@ jobs: - name: Login to GitHub Container Registry if: ${{ github.event_name != 'pull_request' }} - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 with: registry: ghcr.io username: ${{ github.repository_owner }} @@ -306,6 +306,28 @@ jobs: $image:arm64-latest \ $image:amd64-latest + pre-commit: + name: Build and push pre-commit docker image + runs-on: heavy + needs: [repo, tools-merge] + + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-docker-image + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + images: | + ${{ needs.repo.outputs.GHCR_REPO }}/clio-pre-commit + push_image: ${{ github.event_name != 'pull_request' }} + directory: docker/pre-commit + tags: | + type=raw,value=latest + type=raw,value=${{ github.sha }} + platforms: linux/amd64,linux/arm64 + build_args: | + GHCR_REPO=${{ needs.repo.outputs.GHCR_REPO }} + ci: name: Build and push CI docker image runs-on: heavy @@ -313,7 +335,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: ./.github/actions/build_docker_image + - uses: ./.github/actions/build-docker-image env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} diff --git a/.github/workflows/upload_conan_deps.yml b/.github/workflows/upload-conan-deps.yml similarity index 96% rename from .github/workflows/upload_conan_deps.yml rename to .github/workflows/upload-conan-deps.yml index 601fb604..05ebd0a9 100644 --- a/.github/workflows/upload_conan_deps.yml +++ b/.github/workflows/upload-conan-deps.yml @@ -18,7 +18,7 @@ on: pull_request: branches: [develop] paths: - - .github/workflows/upload_conan_deps.yml + - .github/workflows/upload-conan-deps.yml - .github/actions/conan/action.yml - ".github/scripts/conan/**" @@ -28,7 +28,7 @@ on: push: branches: [develop] paths: - - .github/workflows/upload_conan_deps.yml + - .github/workflows/upload-conan-deps.yml - .github/actions/conan/action.yml - ".github/scripts/conan/**" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a8cf47cf..56561fdc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: # hadolint-docker is a special hook that runs hadolint in a Docker container # Docker is not installed in the environment where pre-commit is run stages: [manual] - entry: hadolint/hadolint:v2.14 hadolint + entry: hadolint/hadolint:v2.14.0 hadolint - repo: https://github.com/codespell-project/codespell rev: 63c8f8312b7559622c0d82815639671ae42132ac # frozen: v2.4.1 diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile index 22fc917c..2f913392 100644 --- a/docker/ci/Dockerfile +++ b/docker/ci/Dockerfile @@ -43,26 +43,20 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Install Python tools -ARG PYTHON_VERSION=3.13 - -RUN add-apt-repository ppa:deadsnakes/ppa \ - && apt-get update \ +RUN apt-get update \ && apt-get install -y --no-install-recommends --no-install-suggests \ - python${PYTHON_VERSION} \ - python${PYTHON_VERSION}-venv \ + python3 \ + python3-pip \ && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} - -# Create a virtual environment for python tools -RUN python${PYTHON_VERSION} -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" + && rm -rf /var/lib/apt/lists/* RUN pip install -q --no-cache-dir \ + # TODO: Remove this once we switch to newer Ubuntu base image + # lxml 6.0.0 is not compatible with our image + 'lxml<6.0.0' \ cmake \ conan==2.20.1 \ - gcovr \ - pre-commit + gcovr # Install LLVM tools ARG LLVM_TOOLS_VERSION=20 diff --git a/docker/ci/README.md b/docker/ci/README.md index 8cb6a92d..f41a9a96 100644 --- a/docker/ci/README.md +++ b/docker/ci/README.md @@ -9,7 +9,7 @@ The image is based on Ubuntu 20.04 and contains: - Clang 19 - ClangBuildAnalyzer 1.6.0 - Conan 2.20.1 -- Doxygen 1.12 +- Doxygen 1.14 - GCC 15.2.0 - GDB 16.3 - gh 2.74 diff --git a/docker/develop/compose.yaml b/docker/develop/compose.yaml index b092a6ee..0c02af1c 100644 --- a/docker/develop/compose.yaml +++ b/docker/develop/compose.yaml @@ -1,6 +1,6 @@ services: clio_develop: - image: ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d + image: ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e volumes: - clio_develop_conan_data:/root/.conan2/p - clio_develop_ccache:/root/.ccache diff --git a/docker/pre-commit/Dockerfile b/docker/pre-commit/Dockerfile new file mode 100644 index 00000000..19ce8cca --- /dev/null +++ b/docker/pre-commit/Dockerfile @@ -0,0 +1,37 @@ +ARG GHCR_REPO=invalid +FROM ${GHCR_REPO}/clio-tools:latest AS clio-tools + +# We're using Ubuntu 24.04 to have a more recent version of Python +FROM ubuntu:24.04 + +ARG DEBIAN_FRONTEND=noninteractive + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3002 +USER root +WORKDIR /root + +# Install common tools and dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends --no-install-suggests \ + curl \ + git \ + software-properties-common \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install Python tools +RUN apt-get update \ + && apt-get install -y --no-install-recommends --no-install-suggests \ + python3 \ + python3-pip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install -q --no-cache-dir --break-system-packages \ + pre-commit + +COPY --from=clio-tools \ + /usr/local/bin/doxygen \ + /usr/local/bin/ diff --git a/docker/tools/Dockerfile b/docker/tools/Dockerfile index 0b90666c..1100a4cc 100644 --- a/docker/tools/Dockerfile +++ b/docker/tools/Dockerfile @@ -51,7 +51,7 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ARG DOXYGEN_VERSION=1.12.0 +ARG DOXYGEN_VERSION=1.14.0 RUN wget --progress=dot:giga "https://github.com/doxygen/doxygen/releases/download/Release_${DOXYGEN_VERSION//./_}/doxygen-${DOXYGEN_VERSION}.src.tar.gz" \ && tar xf "doxygen-${DOXYGEN_VERSION}.src.tar.gz" \ && cd "doxygen-${DOXYGEN_VERSION}" \ diff --git a/docs/Doxyfile b/docs/Doxyfile index 0e987938..65e3b6dc 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -15,6 +15,7 @@ EXTRACT_ANON_NSPACES = NO SORT_MEMBERS_CTORS_1ST = YES INPUT = ${SOURCE}/src +USE_MDFILE_AS_MAINPAGE = ${SOURCE}/src/README.md EXCLUDE_SYMBOLS = ${EXCLUDES} RECURSIVE = YES HAVE_DOT = ${USE_DOT} diff --git a/docs/build-clio.md b/docs/build-clio.md index 5118b650..bb5ebbed 100644 --- a/docs/build-clio.md +++ b/docs/build-clio.md @@ -177,7 +177,7 @@ There are several CMake options you can use to customize the build: ### Generating API docs for Clio -The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen 1.12.0 on your system. +The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen 1.14.0 on your system. To generate the API docs, please use CMake option `-Ddocs=ON` as described above and build the `docs` target. @@ -191,7 +191,7 @@ Open the `index.html` file in your browser to see the documentation pages. It is also possible to build Clio using [Docker](https://www.docker.com/) if you don't want to install all the dependencies on your machine. ```sh -docker run -it ghcr.io/xrplf/clio-ci:384e79cd32f5f6c0ab9be3a1122ead41c5a7e67d +docker run -it ghcr.io/xrplf/clio-ci:213752862ca95ecadeb59a6176c3db91a7864b3e git clone https://github.com/XRPLF/clio cd clio ``` diff --git a/docs/run-clio.md b/docs/run-clio.md index 5b6e9ab4..8576ecf2 100644 --- a/docs/run-clio.md +++ b/docs/run-clio.md @@ -77,7 +77,7 @@ It's possible to configure `minimum`, `maximum` and `default` version like so: All of the above are optional. -Clio will fallback to hardcoded defaults when these values are not specified in the config file, or if the configured values are outside of the minimum and maximum supported versions hardcoded in [src/rpc/common/APIVersion.h](../src/rpc/common/APIVersion.hpp). +Clio will fallback to hardcoded defaults when these values are not specified in the config file, or if the configured values are outside of the minimum and maximum supported versions hardcoded in [src/rpc/common/APIVersion.hpp](../src/rpc/common/APIVersion.hpp). > [!TIP] > See the [example-config.json](../docs/examples/config/example-config.json) for more details. diff --git a/pre-commit-hooks/check-doxygen-docs.sh b/pre-commit-hooks/check-doxygen-docs.sh index dce0b93d..cc548792 100755 --- a/pre-commit-hooks/check-doxygen-docs.sh +++ b/pre-commit-hooks/check-doxygen-docs.sh @@ -36,19 +36,19 @@ EOF exit 0 fi -# Check version of doxygen is at least 1.12 +# Check version of doxygen is at least 1.14 version=$($DOXYGEN --version | grep -o '[0-9\.]*') -if [[ "1.12.0" > "$version" ]]; then +if [[ "1.14.0" > "$version" ]]; then # No hard error if doxygen version is not the one we want - let CI deal with it cat <selectNFTIDsByIssuerTaxon.bind(issuer); + Statement const firstQuery = schema_->selectNFTIDsByIssuerTaxon.bind(issuer); firstQuery.bindAt(1, startTaxon); firstQuery.bindAt(2, startTokenID); firstQuery.bindAt(3, Limit{limit}); @@ -147,7 +147,7 @@ public: if (nftIDs.size() < limit) { auto const remainingLimit = limit - nftIDs.size(); - Statement secondQuery = schema_->selectNFTsAfterTaxonKeyspaces.bind(issuer); + Statement const secondQuery = schema_->selectNFTsAfterTaxonKeyspaces.bind(issuer); secondQuery.bindAt(1, startTaxon); secondQuery.bindAt(2, Limit{remainingLimit}); @@ -197,7 +197,7 @@ private: ) const { std::vector nftIDs; - Statement statement = schema_->selectNFTIDsByIssuerTaxon.bind(issuer); + Statement const statement = schema_->selectNFTIDsByIssuerTaxon.bind(issuer); statement.bindAt(1, taxon); statement.bindAt(2, cursorIn.value_or(ripple::uint256(0))); statement.bindAt(3, Limit{limit}); diff --git a/src/data/README.md b/src/data/README.md index 2744b1f8..2b9c81a6 100644 --- a/src/data/README.md +++ b/src/data/README.md @@ -1,8 +1,10 @@ -# Backend +# Backend + +@page "backend" Backend The backend of Clio is responsible for handling the proper reading and writing of past ledger data from and to a given database. Currently, Cassandra and ScyllaDB are the only supported databases that are production-ready. -To support additional database types, you can create new classes that implement the virtual methods in [BackendInterface.h](https://github.com/XRPLF/clio/blob/develop/src/data/BackendInterface.hpp). Then, leveraging the Factory Object Design Pattern, modify [BackendFactory.h](https://github.com/XRPLF/clio/blob/develop/src/data/BackendFactory.hpp) with logic that returns the new database interface if the relevant `type` is provided in Clio's configuration file. +To support additional database types, you can create new classes that implement the virtual methods in [BackendInterface.hpp](https://github.com/XRPLF/clio/blob/develop/src/data/BackendInterface.hpp). Then, leveraging the Factory Object Design Pattern, modify [BackendFactory.hpp](https://github.com/XRPLF/clio/blob/develop/src/data/BackendFactory.hpp) with logic that returns the new database interface if the relevant `type` is provided in Clio's configuration file. ## Data Model diff --git a/src/data/cassandra/CassandraBackendFamily.hpp b/src/data/cassandra/CassandraBackendFamily.hpp index 7233198d..aec50fdf 100644 --- a/src/data/cassandra/CassandraBackendFamily.hpp +++ b/src/data/cassandra/CassandraBackendFamily.hpp @@ -146,9 +146,6 @@ public: */ CassandraBackendFamily(CassandraBackendFamily&&) = delete; - /** - * @copydoc BackendInterface::fetchAccountTransactions - */ TransactionsAndCursor fetchAccountTransactions( ripple::AccountID const& account, @@ -217,18 +214,12 @@ public: return {txns, {}}; } - /** - * @copydoc BackendInterface::waitForWritesToFinish - */ void waitForWritesToFinish() override { executor_.sync(); } - /** - * @copydoc BackendInterface::writeLedger - */ void writeLedger(ripple::LedgerHeader const& ledgerHeader, std::string&& blob) override { @@ -239,9 +230,6 @@ public: ledgerSequence_ = ledgerHeader.seq; } - /** - * @copydoc BackendInterface::fetchLatestLedgerSequence - */ std::optional fetchLatestLedgerSequence(boost::asio::yield_context yield) const override { @@ -262,9 +250,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchLedgerBySequence - */ std::optional fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const override { @@ -292,9 +277,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchLedgerByHash - */ std::optional fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context yield) const override { @@ -315,9 +297,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::hardFetchLedgerRange(boost::asio::yield_context) const - */ std::optional hardFetchLedgerRange(boost::asio::yield_context yield) const override { @@ -356,9 +335,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchAllTransactionsInLedger - */ std::vector fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override { @@ -366,9 +342,6 @@ public: return fetchTransactions(hashes, yield); } - /** - * @copydoc BackendInterface::fetchAllTransactionHashesInLedger - */ std::vector fetchAllTransactionHashesInLedger( std::uint32_t const ledgerSequence, @@ -402,9 +375,6 @@ public: return hashes; } - /** - * @copydoc BackendInterface::fetchNFT - */ std::optional fetchNFT( ripple::uint256 const& tokenID, @@ -444,9 +414,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchNFTTransactions - */ TransactionsAndCursor fetchNFTTransactions( ripple::uint256 const& tokenID, @@ -518,9 +485,6 @@ public: return {txns, {}}; } - /** - * @copydoc BackendInterface::fetchMPTHolders - */ MPTHoldersAndCursor fetchMPTHolders( ripple::uint192 const& mptID, @@ -560,9 +524,6 @@ public: return {mptObjects, {}}; } - /** - * @copydoc BackendInterface::doFetchLedgerObject - */ std::optional doFetchLedgerObject( ripple::uint256 const& key, @@ -585,9 +546,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::doFetchLedgerObjectSeq - */ std::optional doFetchLedgerObjectSeq( ripple::uint256 const& key, @@ -609,9 +567,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchTransaction - */ std::optional fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const override { @@ -629,9 +584,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::doFetchSuccessorKey - */ std::optional doFetchSuccessorKey( ripple::uint256 key, @@ -654,9 +606,6 @@ public: return std::nullopt; } - /** - * @copydoc BackendInterface::fetchTransactions - */ std::vector fetchTransactions(std::vector const& hashes, boost::asio::yield_context yield) const override { @@ -698,9 +647,6 @@ public: return results; } - /** - * @copydoc BackendInterface::doFetchLedgerObjects - */ std::vector doFetchLedgerObjects( std::vector const& keys, @@ -741,9 +687,6 @@ public: return results; } - /** - * @copydoc BackendInterface::fetchLedgerDiff - */ std::vector fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override { @@ -789,9 +732,6 @@ public: return results; } - /** - * @copydoc BackendInterface::fetchMigratorStatus - */ std::optional fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const override { @@ -812,9 +752,6 @@ public: return {}; } - /** - * @copydoc BackendInterface::fetchClioNodesData - */ std::expected>, std::string> fetchClioNodesData(boost::asio::yield_context yield) const override { @@ -831,9 +768,6 @@ public: return result; } - /** - * @copydoc BackendInterface::doWriteLedgerObject - */ void doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override { @@ -845,9 +779,6 @@ public: executor_.write(schema_->insertObject, std::move(key), seq, std::move(blob)); } - /** - * @copydoc BackendInterface::writeSuccessor - */ void writeSuccessor(std::string&& key, std::uint32_t const seq, std::string&& successor) override { @@ -859,9 +790,6 @@ public: executor_.write(schema_->insertSuccessor, std::move(key), seq, std::move(successor)); } - /** - * @copydoc BackendInterface::writeAccountTransactions - */ void writeAccountTransactions(std::vector data) override { @@ -881,9 +809,6 @@ public: executor_.write(std::move(statements)); } - /** - * @copydoc BackendInterface::writeAccountTransaction - */ void writeAccountTransaction(AccountTransactionsData record) override { @@ -901,9 +826,6 @@ public: executor_.write(std::move(statements)); } - /** - * @copydoc BackendInterface::writeNFTTransactions - */ void writeNFTTransactions(std::vector const& data) override { @@ -919,9 +841,6 @@ public: executor_.write(std::move(statements)); } - /** - * @copydoc BackendInterface::writeTransaction - */ void writeTransaction( std::string&& hash, @@ -939,9 +858,6 @@ public: ); } - /** - * @copydoc BackendInterface::writeNFTs - */ void writeNFTs(std::vector const& data) override { @@ -980,9 +896,6 @@ public: executor_.writeEach(std::move(statements)); } - /** - * @copydoc BackendInterface::writeNFTs - */ void writeMPTHolders(std::vector const& data) override { @@ -994,9 +907,6 @@ public: executor_.write(std::move(statements)); } - /** - * @copydoc BackendInterface::startWrites - */ void startWrites() const override { @@ -1004,9 +914,6 @@ public: // probably was used in PG to start a transaction or smth. } - /** - * @copydoc BackendInterface::writeMigratorStatus - */ void writeMigratorStatus(std::string const& migratorName, std::string const& status) override { @@ -1015,27 +922,18 @@ public: ); } - /** - * @copydoc BackendInterface::writeNodeMessage - */ void writeNodeMessage(boost::uuids::uuid const& uuid, std::string message) override { executor_.writeSync(schema_->updateClioNodeMessage, data::cassandra::Text{std::move(message)}, uuid); } - /** - * @copydoc BackendInterface::isTooBusy - */ bool isTooBusy() const override { return executor_.isTooBusy(); } - /** - * @copydoc BackendInterface::stats - */ boost::json::object stats() const override { diff --git a/src/etl/README.md b/src/etl/README.md index 39e5575d..4b8cbf11 100644 --- a/src/etl/README.md +++ b/src/etl/README.md @@ -1,5 +1,7 @@ # ETL subsystem +@page "etl" ETL subsystem + A single Clio node has one or more ETL sources specified in the config file. Clio subscribes to the `ledgers` stream of each of the ETL sources. The stream sends a message whenever a new ledger is validated. Upon receiving a message on the stream, Clio fetches the data associated with the newly validated ledger from one of the ETL sources. The fetch is performed via a gRPC request called `GetLedger`. This request returns the ledger header, transactions and metadata blobs, and every ledger object added/modified/deleted as part of this ledger. The ETL subsystem then writes all of this data to the databases, and moves on to the next ledger. diff --git a/src/etl/impl/GrpcSource.cpp b/src/etl/impl/GrpcSource.cpp index 45c9922f..d1b8b638 100644 --- a/src/etl/impl/GrpcSource.cpp +++ b/src/etl/impl/GrpcSource.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include #include +#include #include #include #include @@ -52,17 +54,25 @@ GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort, std:: try { boost::asio::io_context ctx; boost::asio::ip::tcp::resolver resolver{ctx}; + auto const resolverResult = resolver.resolve(ip, grpcPort); - if (resolverResult.empty()) { + if (resolverResult.empty()) throw std::runtime_error("Failed to resolve " + ip + ":" + grpcPort); - } + std::stringstream ss; ss << resolverResult.begin()->endpoint(); + grpc::ChannelArguments chArgs; chArgs.SetMaxReceiveMessageSize(-1); + chArgs.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, kKEEPALIVE_PING_INTERVAL_MS); + chArgs.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, kKEEPALIVE_TIMEOUT_MS); + chArgs.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, static_cast(kKEEPALIVE_PERMIT_WITHOUT_CALLS)); + chArgs.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, kMAX_PINGS_WITHOUT_DATA); + stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub( grpc::CreateCustomChannel(ss.str(), grpc::InsecureChannelCredentials(), chArgs) ); + LOG(log_.debug()) << "Made stub for remote."; } catch (std::exception const& e) { LOG(log_.warn()) << "Exception while creating stub: " << e.what() << "."; @@ -76,10 +86,11 @@ GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighb if (!stub_) return {{grpc::StatusCode::INTERNAL, "No Stub"}, response}; - // Ledger header with txns and metadata org::xrpl::rpc::v1::GetLedgerRequest request; grpc::ClientContext context; + context.set_deadline(std::chrono::system_clock::now() + kDEADLINE); // Prevent indefinite blocking + request.mutable_ledger()->set_sequence(sequence); request.set_transactions(true); request.set_expand(true); diff --git a/src/etl/impl/GrpcSource.hpp b/src/etl/impl/GrpcSource.hpp index 248f4191..e1e37547 100644 --- a/src/etl/impl/GrpcSource.hpp +++ b/src/etl/impl/GrpcSource.hpp @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -38,6 +39,12 @@ class GrpcSource { std::unique_ptr stub_; std::shared_ptr backend_; + static constexpr auto kKEEPALIVE_PING_INTERVAL_MS = 10000; + static constexpr auto kKEEPALIVE_TIMEOUT_MS = 5000; + static constexpr auto kKEEPALIVE_PERMIT_WITHOUT_CALLS = true; // Allow keepalive pings when no calls + static constexpr auto kMAX_PINGS_WITHOUT_DATA = 0; // No limit + static constexpr auto kDEADLINE = std::chrono::seconds(30); + public: GrpcSource(std::string const& ip, std::string const& grpcPort, std::shared_ptr backend); diff --git a/src/etlng/AmendmentBlockHandlerInterface.hpp b/src/etlng/AmendmentBlockHandlerInterface.hpp index 985d0e9c..2d7618c6 100644 --- a/src/etlng/AmendmentBlockHandlerInterface.hpp +++ b/src/etlng/AmendmentBlockHandlerInterface.hpp @@ -32,6 +32,12 @@ struct AmendmentBlockHandlerInterface { */ virtual void notifyAmendmentBlocked() = 0; + + /** + * @brief Stop the block handler from repeatedly executing + */ + virtual void + stop() = 0; }; } // namespace etlng diff --git a/src/etlng/impl/AmendmentBlockHandler.cpp b/src/etlng/impl/AmendmentBlockHandler.cpp index efb9fbc4..6e3e4776 100644 --- a/src/etlng/impl/AmendmentBlockHandler.cpp +++ b/src/etlng/impl/AmendmentBlockHandler.cpp @@ -25,6 +25,7 @@ #include #include +#include #include namespace etlng::impl { @@ -45,6 +46,11 @@ AmendmentBlockHandler::AmendmentBlockHandler( { } +AmendmentBlockHandler::~AmendmentBlockHandler() +{ + stop(); +} + void AmendmentBlockHandler::notifyAmendmentBlocked() { @@ -53,4 +59,13 @@ AmendmentBlockHandler::notifyAmendmentBlocked() operation_.emplace(ctx_.executeRepeatedly(interval_, action_)); } +void +AmendmentBlockHandler::stop() +{ + if (operation_.has_value()) { + operation_->abort(); + operation_.reset(); + } +} + } // namespace etlng::impl diff --git a/src/etlng/impl/AmendmentBlockHandler.hpp b/src/etlng/impl/AmendmentBlockHandler.hpp index 6275d4ea..f5c327d8 100644 --- a/src/etlng/impl/AmendmentBlockHandler.hpp +++ b/src/etlng/impl/AmendmentBlockHandler.hpp @@ -56,11 +56,10 @@ public: ActionType action = kDEFAULT_AMENDMENT_BLOCK_ACTION ); - ~AmendmentBlockHandler() override - { - if (operation_.has_value()) - operation_.value().abort(); - } + ~AmendmentBlockHandler() override; + + void + stop() override; void notifyAmendmentBlocked() override; diff --git a/src/etlng/impl/GrpcSource.cpp b/src/etlng/impl/GrpcSource.cpp index 80012589..e653db25 100644 --- a/src/etlng/impl/GrpcSource.cpp +++ b/src/etlng/impl/GrpcSource.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -36,6 +37,7 @@ #include #include +#include #include #include #include @@ -63,13 +65,18 @@ resolve(std::string const& ip, std::string const& port) namespace etlng::impl { -GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort) +GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort, std::chrono::system_clock::duration deadline) : log_(fmt::format("ETL_Grpc[{}:{}]", ip, grpcPort)) , initialLoadShouldStop_(std::make_unique(false)) + , deadline_{deadline} { try { grpc::ChannelArguments chArgs; chArgs.SetMaxReceiveMessageSize(-1); + chArgs.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, kKEEPALIVE_PING_INTERVAL_MS); + chArgs.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, kKEEPALIVE_TIMEOUT_MS); + chArgs.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, static_cast(kKEEPALIVE_PERMIT_WITHOUT_CALLS)); + chArgs.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, kMAX_PINGS_WITHOUT_DATA); stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub( grpc::CreateCustomChannel(resolve(ip, grpcPort), grpc::InsecureChannelCredentials(), chArgs) @@ -88,10 +95,11 @@ GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighb if (!stub_) return {{grpc::StatusCode::INTERNAL, "No Stub"}, response}; - // Ledger header with txns and metadata org::xrpl::rpc::v1::GetLedgerRequest request; grpc::ClientContext context; + context.set_deadline(std::chrono::system_clock::now() + deadline_); // Prevent indefinite blocking + request.mutable_ledger()->set_sequence(sequence); request.set_transactions(true); request.set_expand(true); diff --git a/src/etlng/impl/GrpcSource.hpp b/src/etlng/impl/GrpcSource.hpp index 3f177e65..3e167533 100644 --- a/src/etlng/impl/GrpcSource.hpp +++ b/src/etlng/impl/GrpcSource.hpp @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -40,9 +41,20 @@ class GrpcSource { util::Logger log_; std::unique_ptr stub_; std::unique_ptr initialLoadShouldStop_; + std::chrono::system_clock::duration deadline_; + + static constexpr auto kKEEPALIVE_PING_INTERVAL_MS = 10000; + static constexpr auto kKEEPALIVE_TIMEOUT_MS = 5000; + static constexpr auto kKEEPALIVE_PERMIT_WITHOUT_CALLS = true; // Allow keepalive pings when no calls + static constexpr auto kMAX_PINGS_WITHOUT_DATA = 0; // No limit + static constexpr auto kDEADLINE = std::chrono::seconds(30); public: - GrpcSource(std::string const& ip, std::string const& grpcPort); + GrpcSource( + std::string const& ip, + std::string const& grpcPort, + std::chrono::system_clock::duration deadline = kDEADLINE + ); /** * @brief Fetch data for a specific ledger. diff --git a/src/main/Mainpage.hpp b/src/main/Mainpage.hpp deleted file mode 100644 index 079d82b3..00000000 --- a/src/main/Mainpage.hpp +++ /dev/null @@ -1,42 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of clio: https://github.com/XRPLF/clio - Copyright (c) 2023, the clio developers. - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -/** - * @mainpage Clio API server - * - * @section intro Introduction - * - * Clio is an XRP Ledger API server optimized for RPC calls over WebSocket or JSON-RPC. - * - * It stores validated historical ledger and transaction data in a more space efficient format, and uses up to 4 times - * less space than rippled. - * - * Clio can be configured to store data in Apache Cassandra or - * ScyllaDB, enabling scalable read throughput. Multiple Clio nodes can share - * access to the same dataset, which allows for a highly available cluster of Clio nodes without the need for redundant - * data storage or computation. - * - * @section Develop - * - * As you prepare to develop code for Clio, please be sure you are aware of our current - * Contribution guidelines. - * - * Read [rpc/README.md](../rpc/README.md) carefully to know more about writing your own handlers for - * Clio. - */ diff --git a/src/migration/README.md b/src/migration/README.md index 9f0d5455..93ace419 100644 --- a/src/migration/README.md +++ b/src/migration/README.md @@ -1,5 +1,7 @@ # Clio Migration +@page "migration" Clio Migration + Clio maintains the off-chain data of XRPL and multiple indexes tables to powering complex queries. To simplify the creation of index tables, this migration framework handles the process of database change and facilitates the migration of historical data seamlessly. ## Command Line Usage diff --git a/src/rpc/README.md b/src/rpc/README.md index bc1a3eb2..717a173e 100644 --- a/src/rpc/README.md +++ b/src/rpc/README.md @@ -1,4 +1,6 @@ -# RPC subsystem +# RPC subsystem + +@page "rpc" RPC subsystem The RPC subsystem is where the common framework for handling incoming JSON requests is implemented. diff --git a/src/util/async/README.md b/src/util/async/README.md index 13c86fdd..c01a2869 100644 --- a/src/util/async/README.md +++ b/src/util/async/README.md @@ -1,5 +1,7 @@ # Async framework +@page "async" Async framework + ## Introduction Clio uses threads intensively. Multiple parts of Clio were/are implemented by running a `std::thread` with some sort of loop inside. Every time this pattern is reimplemented in a slightly different way. State is managed using asynchronous queues, atomic flags, mutexes and other low level primitives. diff --git a/src/web/README.md b/src/web/README.md index 76b0ce41..094c1a5e 100644 --- a/src/web/README.md +++ b/src/web/README.md @@ -1,5 +1,7 @@ # Web server subsystem +@page "web" Web server subsystem + This folder contains all of the classes for running the web server. The web server subsystem: diff --git a/tests/common/util/MockAmendmentBlockHandler.hpp b/tests/common/util/MockAmendmentBlockHandler.hpp index 6f1b1444..859d1393 100644 --- a/tests/common/util/MockAmendmentBlockHandler.hpp +++ b/tests/common/util/MockAmendmentBlockHandler.hpp @@ -25,4 +25,5 @@ struct MockAmendmentBlockHandler : etlng::AmendmentBlockHandlerInterface { MOCK_METHOD(void, notifyAmendmentBlocked, (), (override)); + MOCK_METHOD(void, stop, (), (override)); }; diff --git a/tests/common/util/MockXrpLedgerAPIService.hpp b/tests/common/util/MockXrpLedgerAPIService.hpp index b7ed6768..a881c9c7 100644 --- a/tests/common/util/MockXrpLedgerAPIService.hpp +++ b/tests/common/util/MockXrpLedgerAPIService.hpp @@ -32,7 +32,9 @@ #include #include +#include #include +#include #include #include @@ -90,8 +92,7 @@ struct WithMockXrpLedgerAPIService : virtual ::testing::Test { ~WithMockXrpLedgerAPIService() override { - server_->Shutdown(); - serverThread_.join(); + shutdown(); } int @@ -99,6 +100,19 @@ struct WithMockXrpLedgerAPIService : virtual ::testing::Test { { return port_; } + + void + shutdown(std::optional deadline = std::nullopt) + { + if (deadline.has_value()) { + server_->Shutdown(std::chrono::system_clock::now() + *deadline); + } else { + server_->Shutdown(); + } + if (serverThread_.joinable()) + serverThread_.join(); + } + MockXrpLedgerAPIService mockXrpLedgerAPIService; private: diff --git a/tests/unit/etl/AmendmentBlockHandlerTests.cpp b/tests/unit/etl/AmendmentBlockHandlerTests.cpp index dee8d95d..85a2b8f8 100644 --- a/tests/unit/etl/AmendmentBlockHandlerTests.cpp +++ b/tests/unit/etl/AmendmentBlockHandlerTests.cpp @@ -36,7 +36,7 @@ struct AmendmentBlockHandlerTest : util::prometheus::WithPrometheus, SyncAsioCon etl::SystemState state; }; -TEST_F(AmendmentBlockHandlerTest, CallTonotifyAmendmentBlockedSetsStateAndRepeatedlyCallsAction) +TEST_F(AmendmentBlockHandlerTest, CallToNotifyAmendmentBlockedSetsStateAndRepeatedlyCallsAction) { AmendmentBlockHandler handler{ctx_, state, std::chrono::nanoseconds{1}, actionMock.AsStdFunction()}; diff --git a/tests/unit/etlng/AmendmentBlockHandlerTests.cpp b/tests/unit/etlng/AmendmentBlockHandlerTests.cpp index c77c4f7f..74cfd894 100644 --- a/tests/unit/etlng/AmendmentBlockHandlerTests.cpp +++ b/tests/unit/etlng/AmendmentBlockHandlerTests.cpp @@ -40,7 +40,7 @@ protected: util::async::CoroExecutionContext ctx_; }; -TEST_F(AmendmentBlockHandlerNgTests, CallTonotifyAmendmentBlockedSetsStateAndRepeatedlyCallsAction) +TEST_F(AmendmentBlockHandlerNgTests, CallToNotifyAmendmentBlockedSetsStateAndRepeatedlyCallsAction) { static constexpr auto kMAX_ITERATIONS = 10uz; etlng::impl::AmendmentBlockHandler handler{ctx_, state_, std::chrono::nanoseconds{1}, actionMock_.AsStdFunction()}; @@ -55,6 +55,7 @@ TEST_F(AmendmentBlockHandlerNgTests, CallTonotifyAmendmentBlockedSetsStateAndRep handler.notifyAmendmentBlocked(); stop.acquire(); // wait for the counter to reach over kMAX_ITERATIONS + handler.stop(); EXPECT_TRUE(state_.isAmendmentBlocked); } diff --git a/tests/unit/etlng/GrpcSourceTests.cpp b/tests/unit/etlng/GrpcSourceTests.cpp index af0067fd..945d6233 100644 --- a/tests/unit/etlng/GrpcSourceTests.cpp +++ b/tests/unit/etlng/GrpcSourceTests.cpp @@ -41,15 +41,18 @@ #include #include +#include #include #include #include #include #include #include +#include #include #include #include +#include #include #include @@ -357,3 +360,34 @@ TEST_F(GrpcSourceStopTests, LoadInitialLedgerStopsWhenRequested) ASSERT_FALSE(res.has_value()); EXPECT_EQ(res.error(), etlng::InitialLedgerLoadError::Cancelled); } + +TEST_F(GrpcSourceNgTests, DeadlineIsHandledCorrectly) +{ + static constexpr auto kDEADLINE = std::chrono::milliseconds{5}; + + uint32_t const sequence = 123u; + bool const getObjects = true; + bool const getObjectNeighbors = false; + + std::binary_semaphore sem(0); + + auto grpcSource = + std::make_unique("localhost", std::to_string(getXRPLMockPort()), kDEADLINE); + + EXPECT_CALL(mockXrpLedgerAPIService, GetLedger) + .WillOnce([&](grpc::ServerContext*, + org::xrpl::rpc::v1::GetLedgerRequest const*, + org::xrpl::rpc::v1::GetLedgerResponse*) { + // wait for main thread to discard us and fail the test if unsuccessful within expected timeframe + [&] { ASSERT_TRUE(sem.try_acquire_for(std::chrono::milliseconds{50})); }(); + return grpc::Status{}; + }); + + auto const [status, response] = grpcSource->fetchLedger(sequence, getObjects, getObjectNeighbors); + ASSERT_FALSE(status.ok()); // timed out after kDEADLINE + + sem.release(); // we don't need to hold GetLedger thread any longer + grpcSource.reset(); + + shutdown(std::chrono::milliseconds{10}); +}