chore: Commits for 2.5.0 (#2352)

Testing in: https://github.com/XRPLF/clio/pull/2345

.github/actions/build_docker_image/action.yml

@@ -17,6 +17,9 @@ inputs:
   platforms:
     description: Platforms to build the image for (e.g. linux/amd64,linux/arm64)
     required: true
+  build_args:
+    description: List of build-time variables
+    required: false
 
   dockerhub_repo:
     description: DockerHub repository name
@@ -61,13 +64,4 @@ runs:
         platforms: ${{ inputs.platforms }}
         push: ${{ inputs.push_image == 'true' }}
         tags: ${{ steps.meta.outputs.tags }}
-
-    - name: Update DockerHub description
-      if: ${{ inputs.push_image == 'true' && inputs.dockerhub_repo != '' }}
-      uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2
-      with:
-        username: ${{ env.DOCKERHUB_USER }}
-        password: ${{ env.DOCKERHUB_PW }}
-        repository: ${{ inputs.dockerhub_repo }}
-        short-description: ${{ inputs.dockerhub_description }}
-        readme-filepath: ${{ inputs.directory }}/README.md
+        build-args: ${{ inputs.build_args }}

.github/actions/setup_conan_macos/action.yml

@@ -1,39 +0,0 @@
-name: Setup conan
-description: Setup conan profiles and artifactory on macOS runner
-
-inputs:
-  conan_files_dir:
-    description: Directory with conan files
-    required: true
-
-runs:
-  using: composite
-  steps:
-    - name: Fail on non-macOS
-      if: runner.os != 'macOS'
-      shell: bash
-      run: exit 1
-
-    - name: Copy global.conf
-      shell: bash
-      run: |
-        cp "${{ inputs.conan_files_dir }}/global.conf" "${{ env.CONAN_HOME }}/global.conf"
-
-    - name: Create apple-clang conan profile
-      shell: bash
-      run: |
-        mkdir -p "${{ env.CONAN_HOME }}/profiles"
-        cp .github/actions/setup_conan_macos/apple-clang.profile "${{ env.CONAN_HOME }}/profiles/apple-clang"
-
-    - name: Create conan profiles for sanitizers
-      shell: bash
-      working-directory: ${{ inputs.conan_files_dir }}
-      run: |
-        cp ./sanitizer_template.profile "${{ env.CONAN_HOME }}/profiles/apple-clang.asan"
-        cp ./sanitizer_template.profile "${{ env.CONAN_HOME }}/profiles/apple-clang.tsan"
-        cp ./sanitizer_template.profile "${{ env.CONAN_HOME }}/profiles/apple-clang.ubsan"
-
-    - name: Add artifactory remote
-      shell: bash
-      run: |
-        conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev

.github/dependabot.yml

@@ -142,16 +142,3 @@ updates:
     commit-message:
       prefix: "ci: [DEPENDABOT] "
     target-branch: develop
-
-  - package-ecosystem: github-actions
-    directory: .github/actions/setup_conan_macos/
-    schedule:
-      interval: weekly
-      day: monday
-      time: "04:00"
-      timezone: Etc/GMT
-    reviewers:
-      - XRPLF/clio-dev-team
-    commit-message:
-      prefix: "ci: [DEPENDABOT] "
-    target-branch: develop

.github/scripts/conan/apple-clang-local.profile

@@ -0,0 +1,11 @@
+[settings]
+arch={{detect_api.detect_arch()}}
+build_type=Release
+compiler=apple-clang
+compiler.cppstd=20
+compiler.libcxx=libc++
+compiler.version=17
+os=Macos
+
+[conf]
+grpc/1.50.1:tools.build:cxxflags+=["-Wno-missing-template-arg-list-after-template-kw"]

.github/scripts/conan/generate_matrix.py

@@ -22,9 +22,6 @@ def generate_matrix():
         itertools.product(MACOS_OS, MACOS_CONTAINERS, MACOS_COMPILERS),
     ):
         for sanitizer_ext, build_type in itertools.product(SANITIZER_EXT, BUILD_TYPES):
-            # libbacktrace doesn't build on arm64 with gcc.tsan
-            if os == "heavy-arm64" and compiler == "gcc" and sanitizer_ext == ".tsan":
-                continue
             configurations.append(
                 {
                     "os": os,

.github/scripts/conan/init.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -ex
+
+CURRENT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_DIR="$(cd "$CURRENT_DIR/../../../" && pwd)"
+
+CONAN_DIR="${CONAN_HOME:-$HOME/.conan2}"
+PROFILES_DIR="$CONAN_DIR/profiles"
+
+if [[ -z "$CI" ]]; then
+    APPLE_CLANG_PROFILE="$CURRENT_DIR/apple-clang-local.profile"
+else
+    APPLE_CLANG_PROFILE="$CURRENT_DIR/apple-clang-ci.profile"
+fi
+
+GCC_PROFILE="$REPO_DIR/docker/ci/conan/gcc.profile"
+CLANG_PROFILE="$REPO_DIR/docker/ci/conan/clang.profile"
+
+SANITIZER_TEMPLATE_FILE="$REPO_DIR/docker/ci/conan/sanitizer_template.profile"
+
+rm -rf "$CONAN_DIR"
+
+conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev
+
+cp "$REPO_DIR/docker/ci/conan/global.conf" "$CONAN_DIR/global.conf"
+
+create_profile_with_sanitizers() {
+    profile_name="$1"
+    profile_source="$2"
+
+    cp "$profile_source" "$PROFILES_DIR/$profile_name"
+    cp "$SANITIZER_TEMPLATE_FILE" "$PROFILES_DIR/$profile_name.asan"
+    cp "$SANITIZER_TEMPLATE_FILE" "$PROFILES_DIR/$profile_name.tsan"
+    cp "$SANITIZER_TEMPLATE_FILE" "$PROFILES_DIR/$profile_name.ubsan"
+}
+
+mkdir -p "$PROFILES_DIR"
+
+if [[ "$(uname)" == "Darwin" ]]; then
+    create_profile_with_sanitizers "apple-clang" "$APPLE_CLANG_PROFILE"
+    echo "include(apple-clang)" > "$PROFILES_DIR/default"
+else
+    create_profile_with_sanitizers "clang" "$CLANG_PROFILE"
+    create_profile_with_sanitizers "gcc" "$GCC_PROFILE"
+    echo "include(gcc)" > "$PROFILES_DIR/default"
+fi

.github/workflows/build.yml

@@ -2,9 +2,9 @@ name: Build
 
 on:
   push:
-    branches: [master, release/*, develop]
+    branches: [release/*, develop]
   pull_request:
-    branches: [master, release/*, develop]
+    branches: [release/*, develop]
     paths:
       - .github/workflows/build.yml
 

.github/workflows/build_impl.yml

@@ -77,11 +77,10 @@ jobs:
         with:
           disable_ccache: ${{ inputs.disable_cache }}
 
-      - name: Setup conan
+      - name: Setup conan on macOS
         if: runner.os == 'macOS'
-        uses: ./.github/actions/setup_conan_macos
-        with:
-          conan_files_dir: docker/ci/conan/
+        shell: bash
+        run: ./.github/scripts/conan/init.sh
 
       - name: Restore cache
         if: ${{ !inputs.disable_cache }}

.github/workflows/clang-tidy.yml

@@ -18,6 +18,7 @@ concurrency:
 
 env:
   CONAN_PROFILE: clang
+  LLVM_TOOLS_VERSION: 20
 
 jobs:
   clang_tidy:
@@ -61,7 +62,7 @@ jobs:
         shell: bash
         id: run_clang_tidy
         run: |
-          run-clang-tidy-19 -p build -j "${{ steps.number_of_threads.outputs.threads_number }}" -fix -quiet 1>output.txt
+          run-clang-tidy-${{ env.LLVM_TOOLS_VERSION }} -p build -j "${{ steps.number_of_threads.outputs.threads_number }}" -fix -quiet 1>output.txt
 
       - name: Fix local includes and clang-format style
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}

.github/workflows/docs.yml

@@ -22,6 +22,11 @@ jobs:
         with:
           lfs: true
 
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+          disable_ccache: true
+
       - name: Create build directory
         run: mkdir build_docs
 

.github/workflows/nightly.yml

@@ -96,6 +96,7 @@ jobs:
     uses: ./.github/workflows/release_impl.yml
     with:
       overwrite_release: true
+      prerelease: true
       title: "Clio development (nightly) build"
       version: nightly
       header: >

.github/workflows/pre-commit-autoupdate.yml

@@ -26,14 +26,25 @@ jobs:
       - run: pre-commit autoupdate --freeze
       - run: pre-commit run --all-files || true
 
+      - uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec # v6.3.0
+        if: github.event_name != 'pull_request'
+        with:
+          gpg_private_key: ${{ secrets.ACTIONS_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.ACTIONS_GPG_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+
       - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
         if: always()
         env:
           GH_REPO: ${{ github.repository }}
           GH_TOKEN: ${{ github.token }}
         with:
-          branch: update/pre-commit-hooks
-          title: "style: Update pre-commit hooks"
           commit-message: "style: Update pre-commit hooks"
+          committer: Clio CI <skuznetsov@ripple.com>
+          branch: update/pre-commit-hooks
+          branch-suffix: timestamp
+          delete-branch: true
+          title: "style: Update pre-commit hooks"
           body: Update versions of pre-commit hooks to latest version.
           reviewers: "godexsoft,kuznetsss,PeterChen13579,mathbunnyru"

.github/workflows/pre-commit.yml

@@ -3,8 +3,7 @@ name: Run pre-commit hooks
 on:
   pull_request:
   push:
-    branches:
-      - develop
+    branches: [develop]
   workflow_dispatch:
 
 jobs:

.github/workflows/release.yml

@@ -48,6 +48,7 @@ jobs:
     uses: ./.github/workflows/release_impl.yml
     with:
       overwrite_release: false
+      prerelease: ${{ contains(github.ref_name, '-') }}
       title: "${{ github.ref_name}}"
       version: "${{ github.ref_name }}"
       header: >

.github/workflows/release_impl.yml

@@ -8,6 +8,11 @@ on:
         required: true
         type: boolean
 
+      prerelease:
+        description: "Create a prerelease"
+        required: true
+        type: boolean
+
       title:
         description: "Release title"
         required: true
@@ -25,12 +30,12 @@ on:
 
       generate_changelog:
         description: "Generate changelog"
-        required: false
+        required: true
         type: boolean
 
       draft:
         description: "Create a draft release"
-        required: false
+        required: true
         type: boolean
 
 jobs:
@@ -69,9 +74,9 @@ jobs:
         shell: bash
         if: ${{ inputs.generate_changelog }}
         run: |
-          LAST_TAG=$(gh release view --json tagName -q .tagName)
-          LAST_TAG_COMMIT=$(git rev-parse $LAST_TAG)
-          BASE_COMMIT=$(git merge-base HEAD $LAST_TAG_COMMIT)
+          LAST_TAG="$(gh release view --json tagName -q .tagName)"
+          LAST_TAG_COMMIT="$(git rev-parse $LAST_TAG)"
+          BASE_COMMIT="$(git merge-base HEAD $LAST_TAG_COMMIT)"
           git-cliff "${BASE_COMMIT}..HEAD" --ignore-tags "nightly|-b"
           cat CHANGELOG.md >> "${RUNNER_TEMP}/release_notes.md"
 
@@ -108,10 +113,10 @@ jobs:
         if: ${{ github.event_name != 'pull_request' }}
         shell: bash
         run: |
-          gh release create ${{ inputs.version }} \
-            ${{ inputs.overwrite_release && '--prerelease' || '' }} \
+          gh release create "${{ inputs.version }}" \
+            ${{ inputs.prerelease && '--prerelease' || '' }} \
             --title "${{ inputs.title }}" \
-            --target $GITHUB_SHA \
+            --target "${GITHUB_SHA}" \
             ${{ inputs.draft && '--draft' || '' }} \
             --notes-file "${RUNNER_TEMP}/release_notes.md" \
             ./release_artifacts/clio_server*

.github/workflows/sanitizers.yml

@@ -37,12 +37,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        compiler: ["gcc", "clang"]
-        sanitizer_ext: [".asan", ".tsan", ".ubsan"]
-        exclude:
-          # Currently, clang.tsan unit tests hang
-          - compiler: clang
-            sanitizer_ext: .tsan
+        compiler: [gcc, clang]
+        sanitizer_ext: [.asan, .tsan, .ubsan]
+        build_type: [Release, Debug]
 
     uses: ./.github/workflows/build_and_test.yml
     with:
@@ -50,9 +47,10 @@ jobs:
       container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
       disable_cache: true
       conan_profile: ${{ matrix.compiler }}${{ matrix.sanitizer_ext }}
-      build_type: Release
+      build_type: ${{ matrix.build_type }}
       static: false
-      run_unit_tests: true
+      # Currently, both gcc.tsan and clang.tsan unit tests hang
+      run_unit_tests: ${{ matrix.sanitizer_ext != '.tsan' }}
       run_integration_tests: false
       upload_clio_server: false
       targets: clio_tests clio_integration_tests

.github/workflows/test_impl.yml

@@ -85,7 +85,7 @@ jobs:
         if: env.SANITIZER_IGNORE_ERRORS == 'true' && steps.check_report.outputs.found_report == 'true'
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ inputs.conan_profile }}_report
+          name: sanitizer_report_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
           path: .sanitizer-report/*
           include-hidden-files: true
 

.github/workflows/update_docker_ci.yml

@@ -23,18 +23,33 @@ on:
   workflow_dispatch:
 
 concurrency:
-  # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
+  # Only matches runs for the current workflow - matches against branch & tags
   group: ${{ github.workflow }}-${{ github.ref }}
   # We want to execute all builds sequentially in develop
   cancel-in-progress: false
 
 env:
-  GHCR_REPO: ghcr.io/${{ github.repository_owner }}
+  CLANG_MAJOR_VERSION: 19
+  GCC_MAJOR_VERSION: 14
+  GCC_VERSION: 14.3.0
 
 jobs:
+  repo:
+    name: Calculate repo name
+    runs-on: ubuntu-latest
+    outputs:
+      GHCR_REPO: ${{ steps.set-ghcr-repo.outputs.GHCR_REPO }}
+
+    steps:
+      - name: Set GHCR_REPO
+        id: set-ghcr-repo
+        run: |
+          echo "GHCR_REPO=$(echo ghcr.io/${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> ${GITHUB_OUTPUT}
+
   gcc-amd64:
     name: Build and push GCC docker image (amd64)
     runs-on: heavy
+    needs: repo
 
     steps:
       - uses: actions/checkout@v4
@@ -53,22 +68,26 @@ jobs:
           DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
         with:
           images: |
-            ${{ env.GHCR_REPO }}/clio-gcc
+            ${{ needs.repo.outputs.GHCR_REPO }}/clio-gcc
             rippleci/clio_gcc
           push_image: ${{ github.event_name != 'pull_request' }}
           directory: docker/compilers/gcc
           tags: |
             type=raw,value=amd64-latest
-            type=raw,value=amd64-12
-            type=raw,value=amd64-12.3.0
+            type=raw,value=amd64-${{ env.GCC_MAJOR_VERSION }}
+            type=raw,value=amd64-${{ env.GCC_VERSION }}
             type=raw,value=amd64-${{ github.sha }}
           platforms: linux/amd64
+          build_args: |
+            GCC_MAJOR_VERSION=${{ env.GCC_MAJOR_VERSION }}
+            GCC_VERSION=${{ env.GCC_VERSION }}
           dockerhub_repo: rippleci/clio_gcc
           dockerhub_description: GCC compiler for XRPLF/clio.
 
   gcc-arm64:
     name: Build and push GCC docker image (arm64)
     runs-on: heavy-arm64
+    needs: repo
 
     steps:
       - uses: actions/checkout@v4
@@ -87,23 +106,26 @@ jobs:
           DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
         with:
           images: |
-            ${{ env.GHCR_REPO }}/clio-gcc
+            ${{ needs.repo.outputs.GHCR_REPO }}/clio-gcc
             rippleci/clio_gcc
           push_image: ${{ github.event_name != 'pull_request' }}
           directory: docker/compilers/gcc
           tags: |
             type=raw,value=arm64-latest
-            type=raw,value=arm64-12
-            type=raw,value=arm64-12.3.0
+            type=raw,value=arm64-${{ env.GCC_MAJOR_VERSION }}
+            type=raw,value=arm64-${{ env.GCC_VERSION }}
             type=raw,value=arm64-${{ github.sha }}
           platforms: linux/arm64
+          build_args: |
+            GCC_MAJOR_VERSION=${{ env.GCC_MAJOR_VERSION }}
+            GCC_VERSION=${{ env.GCC_VERSION }}
           dockerhub_repo: rippleci/clio_gcc
           dockerhub_description: GCC compiler for XRPLF/clio.
 
   gcc-merge:
     name: Merge and push multi-arch GCC docker image
     runs-on: heavy
-    needs: [gcc-amd64, gcc-arm64]
+    needs: [repo, gcc-amd64, gcc-arm64]
 
     steps:
       - uses: actions/checkout@v4
@@ -132,18 +154,14 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_PW }}
 
-      - name: Make GHCR_REPO lowercase
-        run: |
-          echo "GHCR_REPO_LC=$(echo ${{env.GHCR_REPO}} | tr '[:upper:]' '[:lower:]')" >> ${GITHUB_ENV}
-
       - name: Create and push multi-arch manifest
         if: github.event_name != 'pull_request' && steps.changed-files.outputs.any_changed == 'true'
         run: |
-          for image in ${{ env.GHCR_REPO_LC }}/clio-gcc rippleci/clio_gcc; do
+          for image in ${{ needs.repo.outputs.GHCR_REPO }}/clio-gcc rippleci/clio_gcc; do
             docker buildx imagetools create \
               -t $image:latest \
-              -t $image:12 \
-              -t $image:12.3.0 \
+              -t $image:${{ env.GCC_MAJOR_VERSION }} \
+              -t $image:${{ env.GCC_VERSION }} \
               -t $image:${{ github.sha }} \
               $image:arm64-latest \
               $image:amd64-latest
@@ -152,6 +170,7 @@ jobs:
   clang:
     name: Build and push Clang docker image
     runs-on: heavy
+    needs: repo
 
     steps:
       - uses: actions/checkout@v4
@@ -170,21 +189,24 @@ jobs:
           DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
         with:
           images: |
-            ${{ env.GHCR_REPO }}/clio-clang
+            ${{ needs.repo.outputs.GHCR_REPO }}/clio-clang
             rippleci/clio_clang
           push_image: ${{ github.event_name != 'pull_request' }}
           directory: docker/compilers/clang
           tags: |
             type=raw,value=latest
-            type=raw,value=16
+            type=raw,value=${{ env.CLANG_MAJOR_VERSION }}
             type=raw,value=${{ github.sha }}
           platforms: linux/amd64,linux/arm64
+          build_args: |
+            CLANG_MAJOR_VERSION=${{ env.CLANG_MAJOR_VERSION }}
           dockerhub_repo: rippleci/clio_clang
           dockerhub_description: Clang compiler for XRPLF/clio.
 
-  tools:
-    name: Build and push tools docker image
+  tools-amd64:
+    name: Build and push tools docker image (amd64)
     runs-on: heavy
+    needs: [repo, gcc-merge]
 
     steps:
       - uses: actions/checkout@v4
@@ -201,18 +223,87 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           images: |
-            ${{ env.GHCR_REPO }}/clio-tools
+            ${{ needs.repo.outputs.GHCR_REPO }}/clio-tools
           push_image: ${{ github.event_name != 'pull_request' }}
           directory: docker/tools
           tags: |
-            type=raw,value=latest
-            type=raw,value=${{ github.sha }}
-          platforms: linux/amd64,linux/arm64
+            type=raw,value=amd64-latest
+            type=raw,value=amd64-${{ github.sha }}
+          platforms: linux/amd64
+          build_args: |
+            GHCR_REPO=${{ needs.repo.outputs.GHCR_REPO }}
+            GCC_VERSION=${{ env.GCC_VERSION }}
+
+  tools-arm64:
+    name: Build and push tools docker image (arm64)
+    runs-on: heavy-arm64
+    needs: [repo, gcc-merge]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
+        with:
+          files: "docker/tools/**"
+
+      - uses: ./.github/actions/build_docker_image
+        if: steps.changed-files.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          images: |
+            ${{ needs.repo.outputs.GHCR_REPO }}/clio-tools
+          push_image: ${{ github.event_name != 'pull_request' }}
+          directory: docker/tools
+          tags: |
+            type=raw,value=arm64-latest
+            type=raw,value=arm64-${{ github.sha }}
+          platforms: linux/arm64
+          build_args: |
+            GHCR_REPO=${{ needs.repo.outputs.GHCR_REPO }}
+            GCC_VERSION=${{ env.GCC_VERSION }}
+
+  tools-merge:
+    name: Merge and push multi-arch tools docker image
+    runs-on: heavy
+    needs: [repo, tools-amd64, tools-arm64]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
+        with:
+          files: "docker/tools/**"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push multi-arch manifest
+        if: github.event_name != 'pull_request' && steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          image=${{ needs.repo.outputs.GHCR_REPO }}/clio-tools
+          docker buildx imagetools create \
+            -t $image:latest \
+            -t $image:${{ github.sha }} \
+            $image:arm64-latest \
+            $image:amd64-latest
 
   ci:
     name: Build and push CI docker image
     runs-on: heavy
-    needs: [gcc-merge, clang, tools]
+    needs: [repo, gcc-merge, clang, tools-merge]
 
     steps:
       - uses: actions/checkout@v4
@@ -223,14 +314,19 @@ jobs:
           DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
         with:
           images: |
-            ${{ env.GHCR_REPO }}/clio-ci
+            ${{ needs.repo.outputs.GHCR_REPO }}/clio-ci
             rippleci/clio_ci
           push_image: ${{ github.event_name != 'pull_request' }}
           directory: docker/ci
           tags: |
             type=raw,value=latest
-            type=raw,value=gcc_12_clang_16
+            type=raw,value=gcc_${{ env.GCC_MAJOR_VERSION }}_clang_${{ env.CLANG_MAJOR_VERSION }}
             type=raw,value=${{ github.sha }}
           platforms: linux/amd64,linux/arm64
+          build_args: |
+            GHCR_REPO=${{ needs.repo.outputs.GHCR_REPO }}
+            CLANG_MAJOR_VERSION=${{ env.CLANG_MAJOR_VERSION }}
+            GCC_MAJOR_VERSION=${{ env.GCC_MAJOR_VERSION }}
+            GCC_VERSION=${{ env.GCC_VERSION }}
           dockerhub_repo: rippleci/clio_ci
           dockerhub_description: CI image for XRPLF/clio.

.github/workflows/upload_conan_deps.yml

@@ -11,19 +11,27 @@ on:
         default: false
         type: boolean
   pull_request:
-    branches:
-      - develop
+    branches: [develop]
     paths:
       - .github/workflows/upload_conan_deps.yml
-      - .github/scripts/generate_conan_matrix.py
+
+      - .github/actions/generate/action.yml
+      - .github/actions/prepare_runner/action.yml
+      - ".github/scripts/conan/**"
+      - "!.github/scripts/conan/apple-clang-local.profile"
+
       - conanfile.py
       - conan.lock
   push:
-    branches:
-      - develop
+    branches: [develop]
     paths:
       - .github/workflows/upload_conan_deps.yml
-      - .github/scripts/generate_conan_matrix.py
+
+      - .github/actions/generate/action.yml
+      - .github/actions/prepare_runner/action.yml
+      - ".github/scripts/conan/**"
+      - "!.github/scripts/conan/apple-clang-local.profile"
+
       - conanfile.py
       - conan.lock
 
@@ -41,7 +49,7 @@ jobs:
 
       - name: Calculate conan matrix
         id: set-matrix
-        run: .github/scripts/generate_conan_matrix.py >> "${GITHUB_OUTPUT}"
+        run: .github/scripts/conan/generate_matrix.py >> "${GITHUB_OUTPUT}"
 
   upload-conan-deps:
     name: Build ${{ matrix.compiler }}${{ matrix.sanitizer_ext }} ${{ matrix.build_type }}
@@ -66,11 +74,10 @@ jobs:
         with:
           disable_ccache: true
 
-      - name: Setup conan
+      - name: Setup conan on macOS
         if: runner.os == 'macOS'
-        uses: ./.github/actions/setup_conan_macos
-        with:
-          conan_files_dir: docker/ci/conan/
+        shell: bash
+        run: ./.github/scripts/conan/init.sh
 
       - name: Show conan profile
         run: conan profile show --profile:all ${{ env.CONAN_PROFILE }}

.pre-commit-config.yaml

@@ -26,12 +26,12 @@ repos:
 
   # Autoformat: YAML, JSON, Markdown, etc.
   - repo: https://github.com/rbubley/mirrors-prettier
-    rev: 787fb9f542b140ba0b2aced38e6a3e68021647a3 # frozen: v3.5.3
+    rev: 5ba47274f9b181bce26a5150a725577f3c336011 # frozen: v3.6.2
     hooks:
       - id: prettier
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: 586c3ea3f51230da42bab657c6a32e9e66c364f0 # frozen: v0.44.0
+    rev: 192ad822316c3a22fb3d3cc8aa6eafa0b8488360 # frozen: v0.45.0
     hooks:
       - id: markdownlint-fix
         exclude: LICENSE.md
@@ -55,17 +55,38 @@ repos:
             --ignore-words=pre-commit-hooks/codespell_ignore.txt,
           ]
 
-  # Running fix-local-includes before clang-format
-  # to ensure that the include order is correct.
+  - repo: https://github.com/trufflesecurity/trufflehog
+    rev: 6641d4ba5b684fffe195b9820345de1bf19f3181 # frozen: v3.89.2
+    hooks:
+      - id: trufflehog
+        entry: trufflehog git file://. --since-commit HEAD --no-verification --fail
+
+  # Running some C++ hooks before clang-format
+  # to ensure that the style is consistent.
   - repo: local
     hooks:
+      - id: json-in-cpp
+        name: Fix JSON style in C++
+        entry: pre-commit-hooks/json_in_cpp.py
+        types: [c++]
+        language: python
+        exclude: |
+          (?x)^(
+            tests/unit/etl/SubscriptionSourceTests.cpp|
+            tests/unit/web/ServerTests.cpp|
+            tests/unit/web/impl/ErrorHandlingTests.cpp|
+            tests/unit/web/ng/ServerTests.cpp|
+            tests/unit/web/ng/impl/ErrorHandlingTests.cpp
+          )$
+
       - id: fix-local-includes
         name: Fix Local Includes
         entry: pre-commit-hooks/fix-local-includes.sh
         types: [c++]
         language: script
+
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: f9a52e87b6cdcb01b0a62b8611d9ba9f2dad0067 # frozen: v19.1.7
+    rev: 6b9072cd80691b1b48d80046d884409fb1d962d1 # frozen: v20.1.7
     hooks:
       - id: clang-format
         args: [--style=file]

CMakeLists.txt

@@ -69,15 +69,17 @@ endif ()
 # Enable selected sanitizer if enabled via `san`
 if (san)
   set(SUPPORTED_SANITIZERS "address" "thread" "memory" "undefined")
-  list(FIND SUPPORTED_SANITIZERS "${san}" INDEX)
-  if (INDEX EQUAL -1)
+  if (NOT san IN_LIST SUPPORTED_SANITIZERS)
     message(FATAL_ERROR "Error: Unsupported sanitizer '${san}'. Supported values are: ${SUPPORTED_SANITIZERS}.")
   endif ()
 
-  target_compile_options(
-    clio_options INTERFACE # Sanitizers recommend minimum of -O1 for reasonable performance
-                           $<$<CONFIG:Debug>:-O1> ${SAN_FLAG} -fno-omit-frame-pointer
-  )
+  # Sanitizers recommend minimum of -O1 for reasonable performance so we enable it for debug builds
+  set(SAN_OPTIMIZATION_FLAG "")
+  if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(SAN_OPTIMIZATION_FLAG -O1)
+  endif ()
+  target_compile_options(clio_options INTERFACE ${SAN_OPTIMIZATION_FLAG} ${SAN_FLAG} -fno-omit-frame-pointer)
+
   target_compile_definitions(
     clio_options INTERFACE $<$<STREQUAL:${san},address>:SANITIZER=ASAN> $<$<STREQUAL:${san},thread>:SANITIZER=TSAN>
                            $<$<STREQUAL:${san},memory>:SANITIZER=MSAN> $<$<STREQUAL:${san},undefined>:SANITIZER=UBSAN>

cmake/ClangTidy.cmake

@@ -8,7 +8,7 @@ if (lint)
     endif ()
     message(STATUS "Using clang-tidy from CLIO_CLANG_TIDY_BIN")
   else ()
-    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-19" "clang-tidy" REQUIRED)
+    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-20" "clang-tidy" REQUIRED)
   endif ()
 
   if (NOT _CLANG_TIDY_BIN)

cmake/ClioVersion.cmake

@@ -4,39 +4,42 @@
 
 find_package(Git REQUIRED)
 
-set(GIT_COMMAND rev-parse --short HEAD)
+set(GIT_COMMAND describe --tags --exact-match)
 execute_process(
-  COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE REV
+  COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND}
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  OUTPUT_VARIABLE TAG
+  RESULT_VARIABLE RC
   OUTPUT_STRIP_TRAILING_WHITESPACE
 )
 
-set(GIT_COMMAND branch --show-current)
-execute_process(
-  COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE BRANCH
-  OUTPUT_STRIP_TRAILING_WHITESPACE
-)
+if (RC EQUAL 0)
+  # if we are on a tag, use the tag name
+  set(CLIO_VERSION "${TAG}")
+  set(DOC_CLIO_VERSION "${TAG}")
+else ()
+  # if not, use YYYYMMDDHMS-<branch>-<git-rev>
 
-if (BRANCH STREQUAL "")
-  set(BRANCH "dev")
-endif ()
-
-if (NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name
-                                                             # YYYYMMDDHMS-<branch>-<git-rev>
   set(GIT_COMMAND show -s --date=format:%Y%m%d%H%M%S --format=%cd)
   execute_process(
-    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE DATE
-    OUTPUT_STRIP_TRAILING_WHITESPACE
+    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE DATE
+    OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY
   )
+
+  set(GIT_COMMAND branch --show-current)
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE BRANCH
+    OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY
+  )
+
+  set(GIT_COMMAND rev-parse --short HEAD)
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE REV
+    OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY
+  )
+
   set(CLIO_VERSION "${DATE}-${BRANCH}-${REV}")
   set(DOC_CLIO_VERSION "develop")
-else ()
-  set(GIT_COMMAND describe --tags)
-  execute_process(
-    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE CLIO_TAG_VERSION
-    OUTPUT_STRIP_TRAILING_WHITESPACE
-  )
-  set(CLIO_VERSION "${CLIO_TAG_VERSION}")
-  set(DOC_CLIO_VERSION "${CLIO_TAG_VERSION}")
 endif ()
 
 if (CMAKE_BUILD_TYPE MATCHES Debug)

conan.lock

@@ -1,46 +1,46 @@
 {
     "version": "0.5",
     "requires": [
-        "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1750263732.782",
-        "xxhash/0.8.2#7856c968c985b2981b707ee8f2413b2b%1750263730.908",
-        "xrpl/2.5.0-rc1#e5897e048ea5712d2c71561c507d949d%1750263725.455",
-        "sqlite3/3.47.0#7a0904fd061f5f8a2366c294f9387830%1750263721.79",
-        "soci/4.0.3#a9f8d773cd33e356b5879a4b0564f287%1750263717.455",
-        "re2/20230301#dfd6e2bf050eb90ddd8729cfb4c844a4%1750263715.145",
-        "rapidjson/cci.20220822#1b9d8c2256876a154172dc5cfbe447c6%1750263713.526",
-        "protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1750263698.841",
-        "openssl/1.1.1v#216374e4fb5b2e0f5ab1fb6f27b5b434%1750263685.885",
-        "nudb/2.0.8#63990d3e517038e04bf529eb8167f69f%1750263683.814",
-        "minizip/1.2.13#9e87d57804bd372d6d1e32b1871517a3%1750263681.745",
-        "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1750263679.891",
-        "libuv/1.46.0#78565d142ac7102776256328a26cdf60%1750263677.819",
-        "libiconv/1.17#1ae2f60ab5d08de1643a22a81b360c59%1750257497.552",
-        "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1750263675.748",
-        "libarchive/3.7.6#e0453864b2a4d225f06b3304903cb2b7%1750263671.05",
-        "http_parser/2.9.4#98d91690d6fd021e9e624218a85d9d97%1750263668.751",
-        "gtest/1.14.0#f8f0757a574a8dd747d16af62d6eb1b7%1750263666.833",
-        "grpc/1.50.1#02291451d1e17200293a409410d1c4e1%1750263646.614",
-        "fmt/10.1.1#021e170cf81db57da82b5f737b6906c1%1750263644.741",
-        "date/3.0.3#cf28fe9c0aab99fe12da08aa42df65e1%1750263643.099",
-        "cassandra-cpp-driver/2.17.0#e50919efac8418c26be6671fd702540a%1750263632.157",
-        "c-ares/1.34.5#b78b91e7cfb1f11ce777a285bbf169c6%1750263630.06",
-        "bzip2/1.0.8#00b4a4658791c1f06914e087f0e792f5%1750263627.95",
-        "boost/1.83.0#8eb22f36ddfb61f54bbc412c4555bd66%1750263616.444",
-        "benchmark/1.8.3#1a2ce62c99e2b3feaa57b1f0c15a8c46%1724323740.181",
-        "abseil/20230802.1#f0f91485b111dc9837a68972cb19ca7b%1750263609.776"
+        "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1752006674.465",
+        "xxhash/0.8.2#7856c968c985b2981b707ee8f2413b2b%1752006674.334",
+        "xrpl/2.5.0#7880d1696f11fceb1d498570f1a184c8%1752006708.218",
+        "sqlite3/3.47.0#7a0904fd061f5f8a2366c294f9387830%1752006674.338",
+        "soci/4.0.3#a9f8d773cd33e356b5879a4b0564f287%1752006674.465",
+        "re2/20230301#dfd6e2bf050eb90ddd8729cfb4c844a4%1752006674.077",
+        "rapidjson/cci.20220822#1b9d8c2256876a154172dc5cfbe447c6%1752006673.227",
+        "protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1752006673.172",
+        "openssl/1.1.1v#216374e4fb5b2e0f5ab1fb6f27b5b434%1752006673.069",
+        "nudb/2.0.8#63990d3e517038e04bf529eb8167f69f%1752006673.862",
+        "minizip/1.2.13#9e87d57804bd372d6d1e32b1871517a3%1752006672.983",
+        "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1752006672.825",
+        "libuv/1.46.0#78565d142ac7102776256328a26cdf60%1752006672.827",
+        "libiconv/1.17#1e65319e945f2d31941a9d28cc13c058%1752006672.826",
+        "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1752006672.822",
+        "libarchive/3.7.6#e0453864b2a4d225f06b3304903cb2b7%1752006672.917",
+        "http_parser/2.9.4#98d91690d6fd021e9e624218a85d9d97%1752006672.658",
+        "gtest/1.14.0#f8f0757a574a8dd747d16af62d6eb1b7%1752006671.555",
+        "grpc/1.50.1#02291451d1e17200293a409410d1c4e1%1752006671.777",
+        "fmt/11.2.0#579bb2cdf4a7607621beea4eb4651e0f%1752006671.557",
+        "date/3.0.3#cf28fe9c0aab99fe12da08aa42df65e1%1752006671.553",
+        "cassandra-cpp-driver/2.17.0#e50919efac8418c26be6671fd702540a%1752006671.654",
+        "c-ares/1.34.5#b78b91e7cfb1f11ce777a285bbf169c6%1752006671.554",
+        "bzip2/1.0.8#00b4a4658791c1f06914e087f0e792f5%1752006671.549",
+        "boost/1.83.0#5bcb2a14a35875e328bf312e080d3562%1752006671.557",
+        "benchmark/1.8.3#1a2ce62c99e2b3feaa57b1f0c15a8c46%1752006671.408",
+        "abseil/20230802.1#f0f91485b111dc9837a68972cb19ca7b%1752006671.555"
     ],
     "build_requires": [
-        "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1750263732.782",
-        "protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1750263698.841",
-        "protobuf/3.21.9#64ce20e1d9ea24f3d6c504015d5f6fa8%1750263690.822",
-        "cmake/3.31.6#ed0e6c1d49bd564ce6fed1a19653b86d%1750263636.055",
-        "b2/5.3.2#7b5fabfe7088ae933fb3e78302343ea0%1750263614.565"
+        "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1752006674.465",
+        "protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1752006673.172",
+        "protobuf/3.21.9#64ce20e1d9ea24f3d6c504015d5f6fa8%1752006673.173",
+        "cmake/3.31.7#57c3e118bcf267552c0ea3f8bee1e7d5%1752006671.64",
+        "b2/5.3.2#7b5fabfe7088ae933fb3e78302343ea0%1752006671.407"
     ],
     "python_requires": [],
     "overrides": {
         "boost/1.83.0": [
             null,
-            "boost/1.83.0#8eb22f36ddfb61f54bbc412c4555bd66"
+            "boost/1.83.0#5bcb2a14a35875e328bf312e080d3562"
         ],
         "protobuf/3.21.9": [
             null,

conanfile.py

@@ -26,11 +26,11 @@ class ClioConan(ConanFile):
     requires = [
         'boost/1.83.0',
         'cassandra-cpp-driver/2.17.0',
-        'fmt/10.1.1',
+        'fmt/11.2.0',
         'protobuf/3.21.12',
         'grpc/1.50.1',
         'openssl/1.1.1v',
-        'xrpl/2.5.0-rc1',
+        'xrpl/2.5.0',
         'zlib/1.3.1',
         'libbacktrace/cci.20210118'
     ]

docker/ci/Dockerfile

@@ -1,7 +1,11 @@
-FROM ghcr.io/xrplf/clio-gcc:12.3.0 AS clio-gcc
-FROM ghcr.io/xrplf/clio-tools:latest AS clio-tools
+ARG GHCR_REPO=invalid
+ARG CLANG_MAJOR_VERSION=invalid
+ARG GCC_VERSION=invalid
 
-FROM ghcr.io/xrplf/clio-clang:16
+FROM ${GHCR_REPO}/clio-gcc:${GCC_VERSION} AS clio-gcc
+FROM ${GHCR_REPO}/clio-tools:latest AS clio-tools
+
+FROM ${GHCR_REPO}/clio-clang:${CLANG_MAJOR_VERSION}
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -16,7 +20,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 USER root
 WORKDIR /root
 
-ARG LLVM_TOOLS_VERSION=19
+ARG LLVM_TOOLS_VERSION=20
 
 # Add repositories
 RUN apt-get update \
@@ -45,6 +49,10 @@ RUN apt-get update \
         zip \
     && pip3 install -q --upgrade --no-cache-dir pip \
     && pip3 install -q --no-cache-dir \
+        # TODO: Remove this once we switch to newer Ubuntu base image
+        # lxml 6.0.0 is not compatible with our image
+        'lxml<6.0.0' \
+        \
         cmake==3.31.6 \
         conan==2.17.0 \
         gcovr \
@@ -52,29 +60,33 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Install gcc-12 and make ldconfig aware of the new libstdc++ location (for gcc)
+ARG GCC_MAJOR_VERSION=invalid
+
+# Install custom-built gcc and make ldconfig aware of the new libstdc++ location (for gcc)
 # Note: Clang is using libc++ instead
-COPY --from=clio-gcc /gcc12.deb /
+COPY --from=clio-gcc /gcc${GCC_MAJOR_VERSION}.deb /
 RUN apt-get update \
     && apt-get install -y --no-install-recommends --no-install-suggests \
         binutils \
         libc6-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
-    && dpkg -i /gcc12.deb \
-    && rm -rf /gcc12.deb \
+    && dpkg -i /gcc${GCC_MAJOR_VERSION}.deb \
+    && rm -rf /gcc${GCC_MAJOR_VERSION}.deb \
     && ldconfig
 
-# Rewire to use gcc-12 as default compiler
-RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
-    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 \
-    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 \
-    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 \
-    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 100 \
-    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
-    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100
+# Rewire to use our custom-built gcc as default compiler
+RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_MAJOR_VERSION} 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-${GCC_MAJOR_VERSION} 100 \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_MAJOR_VERSION} 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-${GCC_MAJOR_VERSION} 100 \
+    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-${GCC_MAJOR_VERSION} 100 \
+    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-${GCC_MAJOR_VERSION} 100 \
+    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-${GCC_MAJOR_VERSION} 100
 
 COPY --from=clio-tools \
+    /usr/local/bin/mold \
+    /usr/local/bin/ld.mold \
     /usr/local/bin/ccache \
     /usr/local/bin/doxygen \
     /usr/local/bin/ClangBuildAnalyzer \

docker/ci/README.md

@@ -6,13 +6,14 @@ It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but c
 The image is based on Ubuntu 20.04 and contains:
 
 - ccache 4.11.3
-- clang 16.0.6
+- Clang 19
 - ClangBuildAnalyzer 1.6.0
-- conan 2.17.0
-- doxygen 1.12
-- gcc 12.3.0
+- Conan 2.17.0
+- Doxygen 1.12
+- GCC 14.3.0
 - gh 2.74
 - git-cliff 2.9.1
+- mold 2.40.1
 - and some other useful tools
 
 Conan is set up to build Clio without any additional steps.

docker/ci/conan/clang.profile

@@ -4,8 +4,9 @@ build_type=Release
 compiler=clang
 compiler.cppstd=20
 compiler.libcxx=libc++
-compiler.version=16
+compiler.version=19
 os=Linux
 
 [conf]
-tools.build:compiler_executables={'c': '/usr/bin/clang-16', 'cpp': '/usr/bin/clang++-16'}
+tools.build:compiler_executables={"c": "/usr/bin/clang-19", "cpp": "/usr/bin/clang++-19"}
+grpc/1.50.1:tools.build:cxxflags+=["-Wno-missing-template-arg-list-after-template-kw"]

docker/ci/conan/gcc.profile

@@ -4,8 +4,8 @@ build_type=Release
 compiler=gcc
 compiler.cppstd=20
 compiler.libcxx=libstdc++11
-compiler.version=12
+compiler.version=14
 os=Linux
 
 [conf]
-tools.build:compiler_executables={'c': '/usr/bin/gcc-12', 'cpp': '/usr/bin/g++-12'}
+tools.build:compiler_executables={"c": "/usr/bin/gcc-14", "cpp": "/usr/bin/g++-14"}

docker/ci/conan/global.conf

@@ -1,3 +1,2 @@
 core.download:parallel={{os.cpu_count()}}
 core.upload:parallel={{os.cpu_count()}}
-tools.info.package_id:confs = ["tools.build:cflags", "tools.build:cxxflags", "tools.build:exelinkflags", "tools.build:sharedlinkflags"]

docker/ci/conan/sanitizer_template.profile

@@ -1,7 +1,8 @@
 {% set compiler, sani = profile_name.split('.') %}
 
-{% set sanitizer_opt_map = {'asan': 'address', 'tsan': 'thread', 'ubsan': 'undefined'} %}
+{% set sanitizer_opt_map = {"asan": "address", "tsan": "thread", "ubsan": "undefined"} %}
 {% set sanitizer = sanitizer_opt_map[sani] %}
+
 {% set sanitizer_build_flags_str = "-fsanitize=" ~ sanitizer ~ " -g -O1 -fno-omit-frame-pointer" %}
 {% set sanitizer_build_flags = sanitizer_build_flags_str.split(' ') %}
 {% set sanitizer_link_flags_str = "-fsanitize=" ~ sanitizer %}
@@ -10,11 +11,13 @@
 include({{ compiler }})
 
 [options]
-boost/*:extra_b2_flags = "cxxflags=\"{{ sanitizer_build_flags_str }}\" linkflags=\"{{ sanitizer_link_flags_str }}\""
-boost/*:without_stacktrace = True
+boost/*:extra_b2_flags="cxxflags=\"{{ sanitizer_build_flags_str }}\" linkflags=\"{{ sanitizer_link_flags_str }}\""
+boost/*:without_stacktrace=True
 
 [conf]
-tools.build:cflags += {{ sanitizer_build_flags }}
-tools.build:cxxflags += {{ sanitizer_build_flags }}
-tools.build:exelinkflags += {{ sanitizer_link_flags }}
-tools.build:sharedlinkflags += {{ sanitizer_link_flags }}
+tools.build:cflags+={{ sanitizer_build_flags }}
+tools.build:cxxflags+={{ sanitizer_build_flags }}
+tools.build:exelinkflags+={{ sanitizer_link_flags }}
+tools.build:sharedlinkflags+={{ sanitizer_link_flags }}
+
+tools.info.package_id:confs+=["tools.build:cflags", "tools.build:cxxflags", "tools.build:exelinkflags", "tools.build:sharedlinkflags"]

docker/compilers/clang/Dockerfile

@@ -8,8 +8,6 @@ SHELL ["/bin/bash", "-c"]
 USER root
 WORKDIR /root
 
-ARG CLANG_VERSION=16
-
 RUN apt-get update \
     && apt-get install -y --no-install-recommends --no-install-suggests \
         wget \
@@ -18,13 +16,17 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+ARG CLANG_MAJOR_VERSION=invalid
+# Bump this version to force rebuild of the image
+ARG BUILD_VERSION=0
+
 RUN wget --progress=dot:giga https://apt.llvm.org/llvm.sh \
     && chmod +x llvm.sh \
-    && ./llvm.sh ${CLANG_VERSION} \
+    && ./llvm.sh ${CLANG_MAJOR_VERSION} \
     && rm -rf llvm.sh \
     && apt-get update \
     && apt-get install -y --no-install-recommends --no-install-suggests \
-        libc++-${CLANG_VERSION}-dev \
-        libc++abi-${CLANG_VERSION}-dev \
+        libc++-${CLANG_MAJOR_VERSION}-dev \
+        libc++abi-${CLANG_MAJOR_VERSION}-dev \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*

docker/compilers/gcc/Dockerfile

@@ -1,16 +1,14 @@
 ARG UBUNTU_VERSION=20.04
 
-ARG GCC_MAJOR_VERSION=12
+ARG GCC_MAJOR_VERSION=invalid
 
 FROM ubuntu:$UBUNTU_VERSION AS build
 
 ARG UBUNTU_VERSION
 
 ARG GCC_MAJOR_VERSION
-ARG GCC_MINOR_VERSION=3
-ARG GCC_PATCH_VERSION=0
-ARG GCC_VERSION=${GCC_MAJOR_VERSION}.${GCC_MINOR_VERSION}.${GCC_PATCH_VERSION}
-ARG BUILD_VERSION=6
+
+ARG BUILD_VERSION=0
 
 ARG DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
@@ -27,6 +25,8 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+ARG GCC_VERSION
+
 WORKDIR /
 RUN wget --progress=dot:giga https://gcc.gnu.org/pub/gcc/releases/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz \
     && tar xf gcc-$GCC_VERSION.tar.gz
@@ -68,12 +68,13 @@ RUN /gcc-$GCC_VERSION/configure \
         --enable-checking=release
 
 RUN make -j "$(nproc)"
+RUN make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION
 
-RUN make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION \
-    && mkdir -p /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64 \
+RUN export GDB_AUTOLOAD_DIR="/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64" \
+    && mkdir -p "$GDB_AUTOLOAD_DIR" \
     && mv \
-        /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/lib64/libstdc++.so.6.0.30-gdb.py \
-        /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64/libstdc++.so.6.0.30-gdb.py
+        /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/lib64/libstdc++.so.*-gdb.py \
+        $GDB_AUTOLOAD_DIR/
 
 # Generate deb
 WORKDIR /

docker/compilers/gcc/control.m4

@@ -1,4 +1,4 @@
-Package: gcc-12-ubuntu-UBUNTUVERSION
+Package: gcc-14-ubuntu-UBUNTUVERSION
 Version: VERSION
 Architecture: TARGETARCH
 Maintainer: Alex Kremer <akremer@ripple.com>

docker/tools/Dockerfile

@@ -1,24 +1,41 @@
-FROM ubuntu:20.04
+ARG GHCR_REPO=invalid
+ARG GCC_VERSION=invalid
+
+FROM ${GHCR_REPO}/clio-gcc:${GCC_VERSION}
 
 ARG DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+ARG BUILD_VERSION=1
+
 RUN apt-get update \
     && apt-get install -y --no-install-recommends --no-install-suggests \
         bison \
-        build-essential \
-        cmake \
         flex \
         ninja-build \
+        python3 \
+        python3-pip \
         software-properties-common \
         wget \
+    && pip3 install -q --no-cache-dir \
+        cmake==3.31.6 \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /tmp
 
+ARG MOLD_VERSION=2.40.1
+RUN wget --progress=dot:giga "https://github.com/rui314/mold/archive/refs/tags/v${MOLD_VERSION}.tar.gz" \
+    && tar xf "v${MOLD_VERSION}.tar.gz" \
+    && cd "mold-${MOLD_VERSION}" \
+    && mkdir build \
+    && cd build \
+    && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
+    && ninja install \
+    && rm -rf /tmp/* /var/tmp/*
+
 ARG CCACHE_VERSION=4.11.3
 RUN wget --progress=dot:giga "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz" \
     && tar xf "ccache-${CCACHE_VERSION}.tar.gz" \
@@ -26,7 +43,7 @@ RUN wget --progress=dot:giga "https://github.com/ccache/ccache/releases/download
     && mkdir build \
     && cd build \
     && cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=False .. \
-    && cmake --build . --target install \
+    && ninja install \
     && rm -rf /tmp/* /var/tmp/*
 
 ARG DOXYGEN_VERSION=1.12.0
@@ -36,7 +53,7 @@ RUN wget --progress=dot:giga "https://github.com/doxygen/doxygen/releases/downlo
     && mkdir build \
     && cd build \
     && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install \
+    && ninja install \
     && rm -rf /tmp/* /var/tmp/*
 
 ARG CLANG_BUILD_ANALYZER_VERSION=1.6.0
@@ -46,7 +63,7 @@ RUN wget --progress=dot:giga "https://github.com/aras-p/ClangBuildAnalyzer/archi
     && mkdir build \
     && cd build \
     && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install \
+    && ninja install \
     && rm -rf /tmp/* /var/tmp/*
 
 ARG GIT_CLIFF_VERSION=2.9.1

docs/build-clio.md

@@ -6,7 +6,7 @@
 ## Minimum Requirements
 
 - [Python 3.7](https://www.python.org/downloads/)
-- [Conan 1.55, <2.0](https://conan.io/downloads.html)
+- [Conan 2.17.0](https://conan.io/downloads.html)
 - [CMake 3.20, <4.0](https://cmake.org/download/)
 - [**Optional**] [GCovr](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html): needed for code coverage generation
 - [**Optional**] [CCache](https://ccache.dev/): speeds up compilation if you are going to compile Clio often
@@ -19,12 +19,23 @@
 
 ### Conan Configuration
 
-Clio requires `compiler.cppstd=20` in your Conan profile (`~/.conan2/profiles/default`).
+By default, Conan uses `~/.conan2` as it's home folder.
+You can change it by using `$CONAN_HOME` env variable.
+[More info about Conan home](https://docs.conan.io/2/reference/environment.html#conan-home).
 
-> [!NOTE]
-> Although Clio is built using C++23, it's required to set `compiler.cppstd=20` for the time being as some of Clio's dependencies are not yet capable of building under C++23.
+> [!TIP]
+> To setup Conan automatically, you can run `.github/scripts/conan/init.sh`.
+> This will delete Conan home directory (if it exists), set up profiles and add Artifactory remote.
 
-**Mac apple-clang 16 example**:
+The instruction below assumes that `$CONAN_HOME` is not set.
+
+#### Profiles
+
+The default profile is the file in `~/.conan2/profiles/default`.
+
+Here are some examples of possible profiles:
+
+**Mac apple-clang 17 example**:
 
 ```text
 [settings]
@@ -33,7 +44,7 @@ build_type=Release
 compiler=apple-clang
 compiler.cppstd=20
 compiler.libcxx=libc++
-compiler.version=16
+compiler.version=17
 os=Macos
 
 [conf]
@@ -53,13 +64,19 @@ compiler.version=12
 os=Linux
 
 [conf]
-tools.build:compiler_executables={'c': '/usr/bin/gcc-12', 'cpp': '/usr/bin/g++-12'}
+tools.build:compiler_executables={"c": "/usr/bin/gcc-12", "cpp": "/usr/bin/g++-12"}
 ```
 
-Add the following to the `~/.conan2/global.conf` file:
+> [!NOTE]
+> Although Clio is built using C++23, it's required to set `compiler.cppstd=20` in your profile for the time being as some of Clio's dependencies are not yet capable of building under C++23.
+
+#### global.conf file
+
+To increase the speed of downloading and uploading packages, add the following to the `~/.conan2/global.conf` file:
 
 ```text
-tools.info.package_id:confs = ["tools.build:cflags", "tools.build:cxxflags", "tools.build:exelinkflags", "tools.build:sharedlinkflags"]
+core.download:parallel={{os.cpu_count()}}
+core.upload:parallel={{os.cpu_count()}}
 ```
 
 #### Artifactory
@@ -70,16 +87,7 @@ Make sure artifactory is setup with Conan.
 conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev
 ```
 
-Now you should be able to download the prebuilt `xrpl` package on some platforms.
-
-> [!NOTE]
-> You may need to edit the `~/.conan2/remotes.json` file to ensure that this newly added artifactory is listed last. Otherwise, you could see compilation errors when building the project with gcc version 13 (or newer).
-
-Remove old packages you may have cached interactively.
-
-```sh
-conan remove xrpl
-```
+Now you should be able to download the prebuilt dependencies (including `xrpl` package) on supported platforms.
 
 #### Conan lockfile
 
@@ -102,6 +110,7 @@ Navigate to Clio's root directory and run:
 
 ```sh
 mkdir build && cd build
+# You can also specify profile explicitly by adding `--profile:all <PROFILE_NAME>`
 conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True'
 # You can also add -GNinja to use Ninja build system instead of Make
 cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
@@ -213,18 +222,16 @@ Sometimes, during development, you need to build against a custom version of `li
 
 ## Using `clang-tidy` for static analysis
 
-The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 19.0.
-
 Clang-tidy can be run by CMake when building the project.
 To achieve this, you just need to provide the option `-o '&:lint=True'` for the `conan install` command:
 
 ```sh
-conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True' -o '&:lint=True'
+conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True' -o '&:lint=True' --profile:all clang
 ```
 
 By default CMake will try to find `clang-tidy` automatically in your system.
 To force CMake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example:
 
 ```sh
-export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@19/bin/clang-tidy
+export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm/bin/clang-tidy
 ```

docs/configure-clio.md

@@ -5,7 +5,6 @@
 Clio needs access to a `rippled` server in order to work. The following configurations are required for Clio and `rippled` to communicate:
 
 1. In the Clio config file, provide the following:
-
    - The IP of the `rippled` server
 
    - The port on which `rippled` is accepting unencrypted WebSocket connections
@@ -13,7 +12,6 @@ Clio needs access to a `rippled` server in order to work. The following configur
    - The port on which `rippled` is handling gRPC requests
 
 2. In the `rippled` config file, you need to open:
-
    - A port to accept unencrypted WebSocket connections
 
    - A port to handle gRPC requests, with the IP(s) of Clio specified in the `secure_gateway` entry

pre-commit-hooks/json_in_cpp.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+import argparse
+import re
+from pathlib import Path
+
+
+def fix_json_style(cpp_content: str) -> str:
+    cpp_content = cpp_content.replace('R"json(', 'R"JSON(').replace(')json"', ')JSON"')
+
+    pattern = r'R"JSON\((.*?)\)JSON"'
+
+    def replace_json(match):
+        raw_json = match.group(1)
+
+        raw_json = (
+            raw_json.replace(" :", ":")
+            .replace(" ,", ",")
+            .replace(" null", "null")
+            .replace(':"', ': "')
+            .replace(',"', ', "')
+            .replace('":{', '": {')
+            .replace('":[', '": [')
+            .replace('":true', '": true')
+            .replace('":false', '": false')
+            .replace('":null', '": null')
+        )
+        for digit in range(10):
+            raw_json = raw_json.replace(f'":{digit}', f'": {digit}')
+        return f'R"JSON({raw_json})JSON"'
+
+    return re.sub(pattern, replace_json, cpp_content, flags=re.DOTALL)
+
+
+def process_file(file_path: Path, dry_run: bool) -> bool:
+    content = file_path.read_text(encoding="utf-8")
+    new_content = fix_json_style(content)
+
+    if new_content != content:
+        print(f"Processing file: {file_path}")
+        if dry_run:
+            print("Dry run: changes won't be written to the file.")
+        else:
+            print("Writing changes to file.")
+            file_path.write_text(new_content, encoding="utf-8")
+    return new_content == content
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Fix JSON style in C++ files",
+    )
+    parser.add_argument(
+        "--dry-run",
+        default=False,
+        action="store_true",
+        help="Don't modify files, just print what would be changed",
+    )
+    parser.add_argument(
+        "files",
+        nargs="*",
+        help="Specific files to process",
+    )
+
+    args = parser.parse_args()
+
+    success = True
+    for file in args.files:
+        success = success and process_file(Path(file), dry_run=args.dry_run)
+    if not success:
+        print("Errors occurred while processing files.")
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

src/app/CMakeLists.txt

@@ -10,4 +10,5 @@ target_link_libraries(
          clio_web
          clio_rpc
          clio_migration
+  PRIVATE Boost::program_options
 )

src/app/CliArgs.cpp

@@ -60,17 +60,17 @@ CliArgs::parse(int argc, char const* argv[])
     po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
     po::notify(parsed);
 
-    if (parsed.count("help") != 0u) {
+    if (parsed.contains("help")) {
         std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n" << description;
         return Action{Action::Exit{EXIT_SUCCESS}};
     }
 
-    if (parsed.count("version") != 0u) {
+    if (parsed.contains("version")) {
         std::cout << util::build::getClioFullVersionString() << '\n';
         return Action{Action::Exit{EXIT_SUCCESS}};
     }
 
-    if (parsed.count("config-description") != 0u) {
+    if (parsed.contains("config-description")) {
         std::filesystem::path const filePath = parsed["config-description"].as<std::string>();
 
         auto const res = util::config::ClioConfigDescription::generateConfigDescriptionToFile(filePath);
@@ -83,18 +83,17 @@ CliArgs::parse(int argc, char const* argv[])
 
     auto configPath = parsed["conf"].as<std::string>();
 
-    if (parsed.count("migrate") != 0u) {
+    if (parsed.contains("migrate")) {
         auto const opt = parsed["migrate"].as<std::string>();
         if (opt == "status")
             return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::status()}};
         return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::migration(opt)}};
     }
 
-    if (parsed.count("verify") != 0u)
+    if (parsed.contains("verify"))
         return Action{Action::VerifyConfig{.configPath = std::move(configPath)}};
 
-    return Action{Action::Run{.configPath = std::move(configPath), .useNgWebServer = parsed.count("ng-web-server") != 0}
-    };
+    return Action{Action::Run{.configPath = std::move(configPath), .useNgWebServer = parsed.contains("ng-web-server")}};
 }
 
 }  // namespace app

src/app/ClioApplication.cpp

@@ -41,7 +41,6 @@
 #include "util/build/Build.hpp"
 #include "util/config/ConfigDefinition.hpp"
 #include "util/log/Logger.hpp"
-#include "util/prometheus/Prometheus.hpp"
 #include "web/AdminVerificationStrategy.hpp"
 #include "web/RPCServerHandler.hpp"
 #include "web/Server.hpp"
@@ -91,7 +90,6 @@ ClioApplication::ClioApplication(util::config::ClioConfigDefinition const& confi
     : config_(config), signalsHandler_{config_}
 {
     LOG(util::LogService::info()) << "Clio version: " << util::build::getClioFullVersionString();
-    PrometheusService::init(config);
     signalsHandler_.subscribeToStop([this]() { appStopper_.stop(); });
 }
 

src/app/WebHandlers.cpp

@@ -97,7 +97,7 @@ HealthCheckHandler::operator()(
     boost::asio::yield_context
 )
 {
-    static auto constexpr kHEALTH_CHECK_HTML = R"html(
+    static constexpr auto kHEALTH_CHECK_HTML = R"html(
     <!DOCTYPE html>
     <html>
         <head><title>Test page for Clio</title></head>

src/data/AmendmentCenter.cpp

@@ -78,17 +78,20 @@ WritingAmendmentKey::WritingAmendmentKey(std::string amendmentName) : AmendmentK
 
 }  // namespace impl
 
-AmendmentKey::operator std::string const&() const
+AmendmentKey::
+operator std::string const&() const
 {
     return name;
 }
 
-AmendmentKey::operator std::string_view() const
+AmendmentKey::
+operator std::string_view() const
 {
     return name;
 }
 
-AmendmentKey::operator ripple::uint256() const
+AmendmentKey::
+operator ripple::uint256() const
 {
     return Amendment::getAmendmentId(name);
 }

src/data/BackendCounters.cpp

@@ -49,35 +49,45 @@ durationInMillisecondsSince(std::chrono::steady_clock::time_point const startTim
 using namespace util::prometheus;
 
 BackendCounters::BackendCounters()
-    : tooBusyCounter_(PrometheusService::counterInt(
-          "backend_too_busy_total_number",
-          Labels(),
-          "The total number of times the backend was too busy to process a request"
-      ))
-    , writeSyncCounter_(PrometheusService::counterInt(
-          "backend_operations_total_number",
-          Labels({Label{"operation", "write_sync"}}),
-          "The total number of times the backend had to write synchronously"
-      ))
-    , writeSyncRetryCounter_(PrometheusService::counterInt(
-          "backend_operations_total_number",
-          Labels({Label{"operation", "write_sync_retry"}}),
-          "The total number of times the backend had to retry a synchronous write"
-      ))
+    : tooBusyCounter_(
+          PrometheusService::counterInt(
+              "backend_too_busy_total_number",
+              Labels(),
+              "The total number of times the backend was too busy to process a request"
+          )
+      )
+    , writeSyncCounter_(
+          PrometheusService::counterInt(
+              "backend_operations_total_number",
+              Labels({Label{"operation", "write_sync"}}),
+              "The total number of times the backend had to write synchronously"
+          )
+      )
+    , writeSyncRetryCounter_(
+          PrometheusService::counterInt(
+              "backend_operations_total_number",
+              Labels({Label{"operation", "write_sync_retry"}}),
+              "The total number of times the backend had to retry a synchronous write"
+          )
+      )
     , asyncWriteCounters_{"write_async"}
     , asyncReadCounters_{"read_async"}
-    , readDurationHistogram_(PrometheusService::histogramInt(
-          "backend_duration_milliseconds_histogram",
-          Labels({Label{"operation", "read"}}),
-          kHISTOGRAM_BUCKETS,
-          "The duration of backend read operations including retries"
-      ))
-    , writeDurationHistogram_(PrometheusService::histogramInt(
-          "backend_duration_milliseconds_histogram",
-          Labels({Label{"operation", "write"}}),
-          kHISTOGRAM_BUCKETS,
-          "The duration of backend write operations including retries"
-      ))
+    , readDurationHistogram_(
+          PrometheusService::histogramInt(
+              "backend_duration_milliseconds_histogram",
+              Labels({Label{"operation", "read"}}),
+              kHISTOGRAM_BUCKETS,
+              "The duration of backend read operations including retries"
+          )
+      )
+    , writeDurationHistogram_(
+          PrometheusService::histogramInt(
+              "backend_duration_milliseconds_histogram",
+              Labels({Label{"operation", "write"}}),
+              kHISTOGRAM_BUCKETS,
+              "The duration of backend write operations including retries"
+          )
+      )
 {
 }
 
@@ -170,26 +180,34 @@ BackendCounters::report() const
 
 BackendCounters::AsyncOperationCounters::AsyncOperationCounters(std::string name)
     : name_(std::move(name))
-    , pendingCounter_(PrometheusService::gaugeInt(
-          "backend_operations_current_number",
-          Labels({{"operation", name_}, {"status", "pending"}}),
-          "The current number of pending " + name_ + " operations"
-      ))
-    , completedCounter_(PrometheusService::counterInt(
-          "backend_operations_total_number",
-          Labels({{"operation", name_}, {"status", "completed"}}),
-          "The total number of completed " + name_ + " operations"
-      ))
-    , retryCounter_(PrometheusService::counterInt(
-          "backend_operations_total_number",
-          Labels({{"operation", name_}, {"status", "retry"}}),
-          "The total number of retried " + name_ + " operations"
-      ))
-    , errorCounter_(PrometheusService::counterInt(
-          "backend_operations_total_number",
-          Labels({{"operation", name_}, {"status", "error"}}),
-          "The total number of errored " + name_ + " operations"
-      ))
+    , pendingCounter_(
+          PrometheusService::gaugeInt(
+              "backend_operations_current_number",
+              Labels({{"operation", name_}, {"status", "pending"}}),
+              "The current number of pending " + name_ + " operations"
+          )
+      )
+    , completedCounter_(
+          PrometheusService::counterInt(
+              "backend_operations_total_number",
+              Labels({{"operation", name_}, {"status", "completed"}}),
+              "The total number of completed " + name_ + " operations"
+          )
+      )
+    , retryCounter_(
+          PrometheusService::counterInt(
+              "backend_operations_total_number",
+              Labels({{"operation", name_}, {"status", "retry"}}),
+              "The total number of retried " + name_ + " operations"
+          )
+      )
+    , errorCounter_(
+          PrometheusService::counterInt(
+              "backend_operations_total_number",
+              Labels({{"operation", name_}, {"status", "error"}}),
+              "The total number of errored " + name_ + " operations"
+          )
+      )
 {
 }
 

src/data/BackendInterface.hpp

@@ -234,8 +234,12 @@ public:
      * @return A vector of ripple::uint256 representing the account roots
      */
     virtual std::vector<ripple::uint256>
-    fetchAccountRoots(std::uint32_t number, std::uint32_t pageSize, std::uint32_t seq, boost::asio::yield_context yield)
-        const = 0;
+    fetchAccountRoots(
+        std::uint32_t number,
+        std::uint32_t pageSize,
+        std::uint32_t seq,
+        boost::asio::yield_context yield
+    ) const = 0;
 
     /**
      * @brief Updates the range of sequences that are stored in the DB.
@@ -459,8 +463,11 @@ public:
      * @return The sequence in unit32_t on success; nullopt otherwise
      */
     virtual std::optional<std::uint32_t>
-    doFetchLedgerObjectSeq(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield)
-        const = 0;
+    doFetchLedgerObjectSeq(
+        ripple::uint256 const& key,
+        std::uint32_t sequence,
+        boost::asio::yield_context yield
+    ) const = 0;
 
     /**
      * @brief The database-specific implementation for fetching ledger objects.

src/data/CassandraBackend.hpp

@@ -40,7 +40,7 @@
 #include <boost/uuid/string_generator.hpp>
 #include <boost/uuid/uuid.hpp>
 #include <cassandra.h>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/basics/Blob.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/basics/strHex.h>
@@ -361,8 +361,10 @@ public:
     }
 
     std::vector<ripple::uint256>
-    fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
-        const override
+    fetchAllTransactionHashesInLedger(
+        std::uint32_t const ledgerSequence,
+        boost::asio::yield_context yield
+    ) const override
     {
         auto start = std::chrono::system_clock::now();
         auto const res = executor_.read(yield, schema_->selectAllTransactionHashesInLedger, ledgerSequence);
@@ -392,8 +394,11 @@ public:
     }
 
     std::optional<NFT>
-    fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
-        const override
+    fetchNFT(
+        ripple::uint256 const& tokenID,
+        std::uint32_t const ledgerSequence,
+        boost::asio::yield_context yield
+    ) const override
     {
         auto const res = executor_.read(yield, schema_->selectNFT, tokenID, ledgerSequence);
         if (not res)
@@ -554,10 +559,9 @@ public:
         selectNFTStatements.reserve(nftIDs.size());
 
         std::transform(
-            std::cbegin(nftIDs),
-            std::cend(nftIDs),
-            std::back_inserter(selectNFTStatements),
-            [&](auto const& nftID) { return schema_->selectNFT.bind(nftID, ledgerSequence); }
+            std::cbegin(nftIDs), std::cend(nftIDs), std::back_inserter(selectNFTStatements), [&](auto const& nftID) {
+                return schema_->selectNFT.bind(nftID, ledgerSequence);
+            }
         );
 
         auto const nftInfos = executor_.readEach(yield, selectNFTStatements);
@@ -566,10 +570,9 @@ public:
         selectNFTURIStatements.reserve(nftIDs.size());
 
         std::transform(
-            std::cbegin(nftIDs),
-            std::cend(nftIDs),
-            std::back_inserter(selectNFTURIStatements),
-            [&](auto const& nftID) { return schema_->selectNFTURI.bind(nftID, ledgerSequence); }
+            std::cbegin(nftIDs), std::cend(nftIDs), std::back_inserter(selectNFTURIStatements), [&](auto const& nftID) {
+                return schema_->selectNFTURI.bind(nftID, ledgerSequence);
+            }
         );
 
         auto const nftUris = executor_.readEach(yield, selectNFTURIStatements);
@@ -626,8 +629,11 @@ public:
     }
 
     std::optional<Blob>
-    doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield)
-        const override
+    doFetchLedgerObject(
+        ripple::uint256 const& key,
+        std::uint32_t const sequence,
+        boost::asio::yield_context yield
+    ) const override
     {
         LOG(log_.debug()) << "Fetching ledger object for seq " << sequence << ", key = " << ripple::to_string(key);
         if (auto const res = executor_.read(yield, schema_->selectObject, key, sequence); res) {
@@ -645,8 +651,11 @@ public:
     }
 
     std::optional<std::uint32_t>
-    doFetchLedgerObjectSeq(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield)
-        const override
+    doFetchLedgerObjectSeq(
+        ripple::uint256 const& key,
+        std::uint32_t const sequence,
+        boost::asio::yield_context yield
+    ) const override
     {
         LOG(log_.debug()) << "Fetching ledger object for seq " << sequence << ", key = " << ripple::to_string(key);
         if (auto const res = executor_.read(yield, schema_->selectObject, key, sequence); res) {
@@ -680,8 +689,11 @@ public:
     }
 
     std::optional<ripple::uint256>
-    doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
-        const override
+    doFetchSuccessorKey(
+        ripple::uint256 key,
+        std::uint32_t const ledgerSequence,
+        boost::asio::yield_context yield
+    ) const override
     {
         if (auto const res = executor_.read(yield, schema_->selectSuccessor, key, ledgerSequence); res) {
             if (auto const result = res->template get<ripple::uint256>(); result) {
@@ -714,10 +726,9 @@ public:
         auto const timeDiff = util::timed([this, yield, &results, &hashes, &statements]() {
             // TODO: seems like a job for "hash IN (list of hashes)" instead?
             std::transform(
-                std::cbegin(hashes),
-                std::cend(hashes),
-                std::back_inserter(statements),
-                [this](auto const& hash) { return schema_->selectTransaction.bind(hash); }
+                std::cbegin(hashes), std::cend(hashes), std::back_inserter(statements), [this](auto const& hash) {
+                    return schema_->selectTransaction.bind(hash);
+                }
             );
 
             auto const entries = executor_.readEach(yield, statements);
@@ -761,18 +772,14 @@ public:
 
         // TODO: seems like a job for "key IN (list of keys)" instead?
         std::transform(
-            std::cbegin(keys),
-            std::cend(keys),
-            std::back_inserter(statements),
-            [this, &sequence](auto const& key) { return schema_->selectObject.bind(key, sequence); }
+            std::cbegin(keys), std::cend(keys), std::back_inserter(statements), [this, &sequence](auto const& key) {
+                return schema_->selectObject.bind(key, sequence);
+            }
         );
 
         auto const entries = executor_.readEach(yield, statements);
         std::transform(
-            std::cbegin(entries),
-            std::cend(entries),
-            std::back_inserter(results),
-            [](auto const& res) -> Blob {
+            std::cbegin(entries), std::cend(entries), std::back_inserter(results), [](auto const& res) -> Blob {
                 if (auto const maybeValue = res.template get<Blob>(); maybeValue)
                     return *maybeValue;
 
@@ -785,8 +792,12 @@ public:
     }
 
     std::vector<ripple::uint256>
-    fetchAccountRoots(std::uint32_t number, std::uint32_t pageSize, std::uint32_t seq, boost::asio::yield_context yield)
-        const override
+    fetchAccountRoots(
+        std::uint32_t number,
+        std::uint32_t pageSize,
+        std::uint32_t seq,
+        boost::asio::yield_context yield
+    ) const override
     {
         std::vector<ripple::uint256> liveAccounts;
         std::optional<ripple::AccountID> lastItem;

src/data/DBHelpers.hpp

@@ -198,39 +198,6 @@ struct MPTHolderData {
     ripple::AccountID holder;
 };
 
-/**
- * @brief Check whether the supplied object is an offer.
- *
- * @param object The object to check
- * @return true if the object is an offer; false otherwise
- */
-template <typename T>
-inline bool
-isOffer(T const& object)
-{
-    static constexpr short kOFFER_OFFSET = 0x006f;
-    static constexpr short kSHIFT = 8;
-
-    short offerBytes = (object[1] << kSHIFT) | object[2];
-    return offerBytes == kOFFER_OFFSET;
-}
-
-/**
- * @brief Check whether the supplied hex represents an offer object.
- *
- * @param object The object to check
- * @return true if the object is an offer; false otherwise
- */
-template <typename T>
-inline bool
-isOfferHex(T const& object)
-{
-    auto blob = ripple::strUnHex(4, object.begin(), object.begin() + 4);
-    if (blob)
-        return isOffer(*blob);
-    return false;
-}
-
 /**
  * @brief Check whether the supplied object is a dir node.
  *
@@ -241,6 +208,10 @@ template <typename T>
 inline bool
 isDirNode(T const& object)
 {
+    static constexpr auto kMIN_SIZE_REQUIRED = 3;
+    if (std::size(object) < kMIN_SIZE_REQUIRED)
+        return false;
+
     static constexpr short kDIR_NODE_SPACE_KEY = 0x0064;
     short const spaceKey = (object.data()[1] << 8) | object.data()[2];
     return spaceKey == kDIR_NODE_SPACE_KEY;
@@ -264,23 +235,6 @@ isBookDir(T const& key, R const& object)
     return !sle[~ripple::sfOwner].has_value();
 }
 
-/**
- * @brief Get the book out of an offer object.
- *
- * @param offer The offer to get the book for
- * @return Book as ripple::uint256
- */
-template <typename T>
-inline ripple::uint256
-getBook(T const& offer)
-{
-    ripple::SerialIter it{offer.data(), offer.size()};
-    ripple::SLE const sle{it, {}};
-    ripple::uint256 book = sle.getFieldH256(ripple::sfBookDirectory);
-
-    return book;
-}
-
 /**
  * @brief Get the book base.
  *

src/data/README.md

@@ -6,7 +6,7 @@ To support additional database types, you can create new classes that implement
 
 ## Data Model
 
-The data model used by Clio to read and write ledger data is different from what `rippled` uses. `rippled` uses a novel data structure named [_SHAMap_](https://github.com/ripple/rippled/blob/master/src/ripple/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record.
+The data model used by Clio to read and write ledger data is different from what `rippled` uses. `rippled` uses a novel data structure named [_SHAMap_](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record.
 
 `rippled` nodes can also generate a proof-tree by forming a subtree with all the path nodes and their neighbors, which can then be used to prove the existence of the leaf node data to other `rippled` nodes. In short, the main purpose of the SHAMap data structure is to facilitate the fast validation of data integrity between different decentralized `rippled` nodes.
 

src/data/cassandra/Handle.hpp

@@ -99,7 +99,7 @@ public:
     connect() const;
 
     /**
-     * @brief Connect to the the specified keyspace asynchronously.
+     * @brief Connect to the specified keyspace asynchronously.
      *
      * @param keyspace The keyspace to use
      * @return A future
@@ -137,7 +137,7 @@ public:
     disconnect() const;
 
     /**
-     * @brief Reconnect to the the specified keyspace asynchronously.
+     * @brief Reconnect to the specified keyspace asynchronously.
      *
      * @param keyspace The keyspace to use
      * @return A future

src/data/cassandra/impl/Cluster.cpp

@@ -25,7 +25,7 @@
 #include "util/log/Logger.hpp"
 
 #include <cassandra.h>
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <stdexcept>
 #include <string>
@@ -45,7 +45,8 @@ Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), k
 
     cass_cluster_set_token_aware_routing(*this, cass_true);
     if (auto const rc = cass_cluster_set_protocol_version(*this, CASS_PROTOCOL_VERSION_V4); rc != CASS_OK) {
-        throw std::runtime_error(fmt::format("Error setting cassandra protocol version to v4: {}", cass_error_desc(rc))
+        throw std::runtime_error(
+            fmt::format("Error setting cassandra protocol version to v4: {}", cass_error_desc(rc))
         );
     }
 

src/data/cassandra/impl/RetryPolicy.hpp

@@ -45,11 +45,13 @@ public:
      * @brief Create a new retry policy instance with the io_context provided
      */
     ExponentialBackoffRetryPolicy(boost::asio::io_context& ioc)
-        : retry_(util::makeRetryExponentialBackoff(
-              std::chrono::milliseconds(1),
-              std::chrono::seconds(1),
-              boost::asio::make_strand(ioc)
-          ))
+        : retry_(
+              util::makeRetryExponentialBackoff(
+                  std::chrono::milliseconds(1),
+                  std::chrono::seconds(1),
+                  boost::asio::make_strand(ioc)
+              )
+          )
     {
     }
 

src/data/cassandra/impl/Statement.hpp

@@ -28,7 +28,7 @@
 #include <boost/uuid/uuid.hpp>
 #include <boost/uuid/uuid_io.hpp>
 #include <cassandra.h>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/protocol/AccountID.h>
 #include <xrpl/protocol/STAccount.h>

src/etl/ETLService.cpp

@@ -171,9 +171,11 @@ ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
     auto pipe = DataPipeType{numExtractors, startSequence};
 
     for (auto i = 0u; i < numExtractors; ++i) {
-        extractors.push_back(std::make_unique<ExtractorType>(
-            pipe, networkValidatedLedgers_, ledgerFetcher_, startSequence + i, finishSequence_, state_
-        ));
+        extractors.push_back(
+            std::make_unique<ExtractorType>(
+                pipe, networkValidatedLedgers_, ledgerFetcher_, startSequence + i, finishSequence_, state_
+            )
+        );
     }
 
     auto transformer =

src/etl/LoadBalancer.cpp

@@ -44,7 +44,7 @@
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <algorithm>
 #include <chrono>
@@ -57,7 +57,6 @@
 #include <string>
 #include <thread>
 #include <utility>
-#include <variant>
 #include <vector>
 
 using namespace util::config;
@@ -184,11 +183,14 @@ LoadBalancer::LoadBalancer(
             LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString()
                              << " Please check the configuration and network";
         } else if (etlState_ && etlState_->networkID != stateOpt->networkID) {
-            checkOnETLFailure(fmt::format(
-                "ETL sources must be on the same network. Source network id = {} does not match others network id = {}",
-                stateOpt->networkID,
-                etlState_->networkID
-            ));
+            checkOnETLFailure(
+                fmt::format(
+                    "ETL sources must be on the same network. Source network id = {} does not match others network id "
+                    "= {}",
+                    stateOpt->networkID,
+                    etlState_->networkID
+                )
+            );
         } else {
             etlState_ = stateOpt;
         }
@@ -275,42 +277,46 @@ LoadBalancer::forwardToRippled(
         return std::unexpected{rpc::ClioError::RpcCommandIsMissing};
 
     auto const cmd = boost::json::value_to<std::string>(request.at("command"));
-
-    if (forwardingCache_ and forwardingCache_->shouldCache(cmd)) {
-        bool servedFromCache = true;
-        auto updater =
-            [this, &request, &clientIp, &servedFromCache, isAdmin](boost::asio::yield_context yield
-            ) -> std::expected<util::ResponseExpirationCache::EntryData, util::ResponseExpirationCache::Error> {
-            servedFromCache = false;
-            auto result = forwardToRippledImpl(request, clientIp, isAdmin, yield);
-            if (result.has_value()) {
-                return util::ResponseExpirationCache::EntryData{
-                    .lastUpdated = std::chrono::steady_clock::now(), .response = std::move(result).value()
-                };
-            }
-            return std::unexpected{
-                util::ResponseExpirationCache::Error{.status = rpc::Status{result.error()}, .warnings = {}}
-            };
-        };
-
-        auto result = forwardingCache_->getOrUpdate(
-            yield,
-            cmd,
-            std::move(updater),
-            [](util::ResponseExpirationCache::EntryData const& entry) { return not entry.response.contains("error"); }
-        );
-        if (servedFromCache) {
-            ++forwardingCounters_.cacheHit.get();
+    if (forwardingCache_) {
+        if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
+            forwardingCounters_.cacheHit.get() += 1;
+            return std::move(cachedResponse).value();
         }
-        if (result.has_value()) {
-            return std::move(result).value();
+    }
+    forwardingCounters_.cacheMiss.get() += 1;
+
+    ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
+    std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
+
+    auto numAttempts = 0u;
+
+    auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
+
+    std::optional<boost::json::object> response;
+    rpc::ClioError error = rpc::ClioError::EtlConnectionError;
+    while (numAttempts < sources_.size()) {
+        auto [res, duration] =
+            util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
+        if (res) {
+            forwardingCounters_.successDuration.get() += duration;
+            response = std::move(res).value();
+            break;
         }
-        auto const combinedError = result.error().status.code;
-        ASSERT(std::holds_alternative<rpc::ClioError>(combinedError), "There could be only ClioError here");
-        return std::unexpected{std::get<rpc::ClioError>(combinedError)};
+        forwardingCounters_.failDuration.get() += duration;
+        ++forwardingCounters_.retries.get();
+        error = std::max(error, res.error());  // Choose the best result between all sources
+
+        sourceIdx = (sourceIdx + 1) % sources_.size();
+        ++numAttempts;
     }
 
-    return forwardToRippledImpl(request, clientIp, isAdmin, yield);
+    if (response) {
+        if (forwardingCache_ and not response->contains("error"))
+            forwardingCache_->put(cmd, *response);
+        return std::move(response).value();
+    }
+
+    return std::unexpected{error};
 }
 
 boost::json::value
@@ -401,47 +407,4 @@ LoadBalancer::chooseForwardingSource()
     }
 }
 
-std::expected<boost::json::object, rpc::CombinedError>
-LoadBalancer::forwardToRippledImpl(
-    boost::json::object const& request,
-    std::optional<std::string> const& clientIp,
-    bool const isAdmin,
-    boost::asio::yield_context yield
-)
-{
-    ++forwardingCounters_.cacheMiss.get();
-
-    ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
-    std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
-
-    auto numAttempts = 0u;
-
-    auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
-
-    std::optional<boost::json::object> response;
-    rpc::ClioError error = rpc::ClioError::EtlConnectionError;
-    while (numAttempts < sources_.size()) {
-        auto [res, duration] =
-            util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
-
-        if (res) {
-            forwardingCounters_.successDuration.get() += duration;
-            response = std::move(res).value();
-            break;
-        }
-        forwardingCounters_.failDuration.get() += duration;
-        ++forwardingCounters_.retries.get();
-        error = std::max(error, res.error());  // Choose the best result between all sources
-
-        sourceIdx = (sourceIdx + 1) % sources_.size();
-        ++numAttempts;
-    }
-
-    if (response.has_value()) {
-        return std::move(response).value();
-    }
-
-    return std::unexpected{error};
-}
-
 }  // namespace etl

src/etl/LoadBalancer.hpp

@@ -49,6 +49,7 @@
 #include <concepts>
 #include <cstdint>
 #include <expected>
+#include <functional>
 #include <memory>
 #include <optional>
 #include <string>
@@ -172,8 +173,10 @@ public:
      * @return A std::vector<std::string> The ledger data
      */
     std::vector<std::string>
-    loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2})
-        override;
+    loadInitialLedger(
+        uint32_t sequence,
+        std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
+    ) override;
 
     /**
      * @brief Load the initial ledger, writing data to the queue.
@@ -279,14 +282,6 @@ private:
      */
     void
     chooseForwardingSource();
-
-    std::expected<boost::json::object, rpc::CombinedError>
-    forwardToRippledImpl(
-        boost::json::object const& request,
-        std::optional<std::string> const& clientIp,
-        bool isAdmin,
-        boost::asio::yield_context yield
-    );
 };
 
 }  // namespace etl

src/etl/NFTHelpers.cpp

@@ -18,8 +18,9 @@
 //==============================================================================
 
 #include "data/DBHelpers.hpp"
+#include "util/Assert.hpp"
 
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/basics/strHex.h>
 #include <xrpl/protocol/AccountID.h>
@@ -138,7 +139,8 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
     // There should always be a difference so the returned finalIDs
     // iterator should never be end().  But better safe than sorry.
     if (finalIDs.size() != prevIDs.size() + 1 || diff.first == finalIDs.end() || !owner) {
-        throw std::runtime_error(fmt::format(" - unexpected NFTokenMint data in tx {}", strHex(sttx.getTransactionID()))
+        throw std::runtime_error(
+            fmt::format(" - unexpected NFTokenMint data in tx {}", strHex(sttx.getTransactionID()))
         );
     }
 
@@ -358,14 +360,18 @@ getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
 std::vector<NFTsData>
 getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string const& blob)
 {
-    std::vector<NFTsData> nfts;
-    ripple::STLedgerEntry const sle =
+    // https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0020-non-fungible-tokens#tokenpage-id-format
+    ASSERT(key.size() == ripple::uint256::size(), "The size of the key (token) is expected to fit uint256 exactly");
+
+    auto const sle =
         ripple::STLedgerEntry(ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data()));
 
     if (sle.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE)
-        return nfts;
+        return {};
 
     auto const owner = ripple::AccountID::fromVoid(key.data());
+    std::vector<NFTsData> nfts;
+
     for (ripple::STObject const& node : sle.getFieldArray(ripple::sfNFTokens))
         nfts.emplace_back(node.getFieldH256(ripple::sfNFTokenID), seq, owner, node.getFieldVL(ripple::sfURI));
 

src/etl/impl/Extractor.hpp

@@ -94,8 +94,8 @@ private:
         double totalTime = 0.0;
         auto currentSequence = startSequence_;
 
-        while (!shouldFinish(currentSequence) && networkValidatedLedgers_->waitUntilValidatedByNetwork(currentSequence)
-        ) {
+        while (!shouldFinish(currentSequence) &&
+               networkValidatedLedgers_->waitUntilValidatedByNetwork(currentSequence)) {
             auto [fetchResponse, time] = ::util::timed<std::chrono::duration<double>>([this, currentSequence]() {
                 return ledgerFetcher_.get().fetchDataAndDiff(currentSequence);
             });

src/etl/impl/ForwardingSource.cpp

@@ -28,7 +28,7 @@
 #include <boost/json/object.hpp>
 #include <boost/json/parse.hpp>
 #include <boost/json/serialize.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <chrono>
 #include <exception>

src/etl/impl/GrpcSource.cpp

@@ -26,7 +26,7 @@
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/ip/tcp.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <grpcpp/client_context.h>
 #include <grpcpp/security/credentials.h>
 #include <grpcpp/support/channel_arguments.h>

src/etl/impl/LedgerLoader.hpp

@@ -209,47 +209,49 @@ public:
             size_t numWrites = 0;
             backend_->cache().setFull();
 
-            auto seconds = ::util::timed<std::chrono::seconds>([this, keys = std::move(edgeKeys), sequence, &numWrites](
-                                                               ) mutable {
-                for (auto& key : keys) {
-                    LOG(log_.debug()) << "Writing edge key = " << ripple::strHex(key);
-                    auto succ = backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
-                    if (succ)
-                        backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
-                }
-
-                ripple::uint256 prev = data::kFIRST_KEY;
-                while (auto cur = backend_->cache().getSuccessor(prev, sequence)) {
-                    ASSERT(cur.has_value(), "Successor for key {} must exist", ripple::strHex(prev));
-                    if (prev == data::kFIRST_KEY)
-                        backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
-
-                    if (isBookDir(cur->key, cur->blob)) {
-                        auto base = getBookBase(cur->key);
-                        // make sure the base is not an actual object
-                        if (!backend_->cache().get(base, sequence)) {
-                            auto succ = backend_->cache().getSuccessor(base, sequence);
-                            ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
-                            if (succ->key == cur->key) {
-                                LOG(log_.debug()) << "Writing book successor = " << ripple::strHex(base) << " - "
-                                                  << ripple::strHex(cur->key);
-
-                                backend_->writeSuccessor(uint256ToString(base), sequence, uint256ToString(cur->key));
-                            }
-                        }
-
-                        ++numWrites;
+            auto seconds =
+                ::util::timed<std::chrono::seconds>([this, keys = std::move(edgeKeys), sequence, &numWrites]() mutable {
+                    for (auto& key : keys) {
+                        LOG(log_.debug()) << "Writing edge key = " << ripple::strHex(key);
+                        auto succ = backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
+                        if (succ)
+                            backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
                     }
 
-                    prev = cur->key;
-                    static constexpr std::size_t kLOG_STRIDE = 100000;
-                    if (numWrites % kLOG_STRIDE == 0 && numWrites != 0)
-                        LOG(log_.info()) << "Wrote " << numWrites << " book successors";
-                }
+                    ripple::uint256 prev = data::kFIRST_KEY;
+                    while (auto cur = backend_->cache().getSuccessor(prev, sequence)) {
+                        ASSERT(cur.has_value(), "Successor for key {} must exist", ripple::strHex(prev));
+                        if (prev == data::kFIRST_KEY)
+                            backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
 
-                backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(data::kLAST_KEY));
-                ++numWrites;
-            });
+                        if (isBookDir(cur->key, cur->blob)) {
+                            auto base = getBookBase(cur->key);
+                            // make sure the base is not an actual object
+                            if (!backend_->cache().get(base, sequence)) {
+                                auto succ = backend_->cache().getSuccessor(base, sequence);
+                                ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
+                                if (succ->key == cur->key) {
+                                    LOG(log_.debug()) << "Writing book successor = " << ripple::strHex(base) << " - "
+                                                      << ripple::strHex(cur->key);
+
+                                    backend_->writeSuccessor(
+                                        uint256ToString(base), sequence, uint256ToString(cur->key)
+                                    );
+                                }
+                            }
+
+                            ++numWrites;
+                        }
+
+                        prev = cur->key;
+                        static constexpr std::size_t kLOG_STRIDE = 100000;
+                        if (numWrites % kLOG_STRIDE == 0 && numWrites != 0)
+                            LOG(log_.info()) << "Wrote " << numWrites << " book successors";
+                    }
+
+                    backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(data::kLAST_KEY));
+                    ++numWrites;
+                });
 
             LOG(log_.info()) << "Looping through cache and submitting all writes took " << seconds
                              << " seconds. numWrites = " << std::to_string(numWrites);

src/etl/impl/LedgerPublisher.hpp

@@ -249,8 +249,9 @@ public:
     std::chrono::time_point<std::chrono::system_clock>
     getLastPublish() const override
     {
-        return std::chrono::time_point<std::chrono::system_clock>{std::chrono::seconds{lastPublishSeconds_.get().value()
-        }};
+        return std::chrono::time_point<std::chrono::system_clock>{
+            std::chrono::seconds{lastPublishSeconds_.get().value()}
+        };
     }
 
     /**

src/etl/impl/SubscriptionSource.cpp

@@ -40,7 +40,7 @@
 #include <boost/json/parse.hpp>
 #include <boost/json/serialize.hpp>
 #include <boost/json/value_to.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/protocol/jss.h>
 
 #include <algorithm>
@@ -79,11 +79,13 @@ SubscriptionSource::SubscriptionSource(
     , onConnect_(std::move(onConnect))
     , onDisconnect_(std::move(onDisconnect))
     , onLedgerClosed_(std::move(onLedgerClosed))
-    , lastMessageTimeSecondsSinceEpoch_(PrometheusService::gaugeInt(
-          "subscription_source_last_message_time",
-          util::prometheus::Labels({{"source", fmt::format("{}:{}", ip, wsPort)}}),
-          "Seconds since epoch of the last message received from rippled subscription streams"
-      ))
+    , lastMessageTimeSecondsSinceEpoch_(
+          PrometheusService::gaugeInt(
+              "subscription_source_last_message_time",
+              util::prometheus::Labels({{"source", fmt::format("{}:{}", ip, wsPort)}}),
+              "Seconds since epoch of the last message received from rippled subscription streams"
+          )
+      )
 {
     wsConnectionBuilder_.addHeader({boost::beast::http::field::user_agent, "clio-client"})
         .addHeader({"X-User", "clio-client"})
@@ -329,9 +331,13 @@ SubscriptionSource::setValidatedRange(std::string range)
             pairs.emplace_back(sequence, sequence);
         } else {
             if (minAndMax.size() != 2) {
-                throw std::runtime_error(fmt::format(
-                    "Error parsing range: {}.Min and max should be of size 2. Got size = {}", range, minAndMax.size()
-                ));
+                throw std::runtime_error(
+                    fmt::format(
+                        "Error parsing range: {}.Min and max should be of size 2. Got size = {}",
+                        range,
+                        minAndMax.size()
+                    )
+                );
             }
             uint32_t const min = std::stoll(minAndMax[0]);
             uint32_t const max = std::stoll(minAndMax[1]);

src/etl/impl/SubscriptionSource.hpp

@@ -34,7 +34,7 @@
 #include <boost/asio/spawn.hpp>
 #include <boost/asio/strand.hpp>
 #include <boost/beast/http/field.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <atomic>
 #include <chrono>

src/etlng/ETLService.cpp

@@ -56,7 +56,6 @@
 
 #include <boost/json/object.hpp>
 #include <boost/signals2/connection.hpp>
-#include <fmt/core.h>
 #include <xrpl/protocol/LedgerHeader.h>
 
 #include <chrono>

src/etlng/ETLService.hpp

@@ -58,7 +58,7 @@
 #include <boost/asio/io_context.hpp>
 #include <boost/json/object.hpp>
 #include <boost/signals2/connection.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/basics/Blob.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/basics/strHex.h>

src/etlng/LoadBalancer.cpp

@@ -45,7 +45,7 @@
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <algorithm>
 #include <chrono>
@@ -58,7 +58,6 @@
 #include <string>
 #include <thread>
 #include <utility>
-#include <variant>
 #include <vector>
 
 using namespace util::config;
@@ -184,11 +183,14 @@ LoadBalancer::LoadBalancer(
             LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString()
                              << " Please check the configuration and network";
         } else if (etlState_ && etlState_->networkID != stateOpt->networkID) {
-            checkOnETLFailure(fmt::format(
-                "ETL sources must be on the same network. Source network id = {} does not match others network id = {}",
-                stateOpt->networkID,
-                etlState_->networkID
-            ));
+            checkOnETLFailure(
+                fmt::format(
+                    "ETL sources must be on the same network. Source network id = {} does not match others network id "
+                    "= {}",
+                    stateOpt->networkID,
+                    etlState_->networkID
+                )
+            );
         } else {
             etlState_ = stateOpt;
         }
@@ -281,42 +283,46 @@ LoadBalancer::forwardToRippled(
         return std::unexpected{rpc::ClioError::RpcCommandIsMissing};
 
     auto const cmd = boost::json::value_to<std::string>(request.at("command"));
-
-    if (forwardingCache_ and forwardingCache_->shouldCache(cmd)) {
-        bool servedFromCache = true;
-        auto updater =
-            [this, &request, &clientIp, &servedFromCache, isAdmin](boost::asio::yield_context yield
-            ) -> std::expected<util::ResponseExpirationCache::EntryData, util::ResponseExpirationCache::Error> {
-            servedFromCache = false;
-            auto result = forwardToRippledImpl(request, clientIp, isAdmin, yield);
-            if (result.has_value()) {
-                return util::ResponseExpirationCache::EntryData{
-                    .lastUpdated = std::chrono::steady_clock::now(), .response = std::move(result).value()
-                };
-            }
-            return std::unexpected{
-                util::ResponseExpirationCache::Error{.status = rpc::Status{result.error()}, .warnings = {}}
-            };
-        };
-
-        auto result = forwardingCache_->getOrUpdate(
-            yield,
-            cmd,
-            std::move(updater),
-            [](util::ResponseExpirationCache::EntryData const& entry) { return not entry.response.contains("error"); }
-        );
-        if (servedFromCache) {
-            ++forwardingCounters_.cacheHit.get();
+    if (forwardingCache_) {
+        if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
+            forwardingCounters_.cacheHit.get() += 1;
+            return std::move(cachedResponse).value();
         }
-        if (result.has_value()) {
-            return std::move(result).value();
+    }
+    forwardingCounters_.cacheMiss.get() += 1;
+
+    ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
+    std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
+
+    auto numAttempts = 0u;
+
+    auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
+
+    std::optional<boost::json::object> response;
+    rpc::ClioError error = rpc::ClioError::EtlConnectionError;
+    while (numAttempts < sources_.size()) {
+        auto [res, duration] =
+            util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
+        if (res) {
+            forwardingCounters_.successDuration.get() += duration;
+            response = std::move(res).value();
+            break;
         }
-        auto const combinedError = result.error().status.code;
-        ASSERT(std::holds_alternative<rpc::ClioError>(combinedError), "There could be only ClioError here");
-        return std::unexpected{std::get<rpc::ClioError>(combinedError)};
+        forwardingCounters_.failDuration.get() += duration;
+        ++forwardingCounters_.retries.get();
+        error = std::max(error, res.error());  // Choose the best result between all sources
+
+        sourceIdx = (sourceIdx + 1) % sources_.size();
+        ++numAttempts;
     }
 
-    return forwardToRippledImpl(request, clientIp, isAdmin, yield);
+    if (response) {
+        if (forwardingCache_ and not response->contains("error"))
+            forwardingCache_->put(cmd, *response);
+        return std::move(response).value();
+    }
+
+    return std::unexpected{error};
 }
 
 boost::json::value
@@ -407,47 +413,4 @@ LoadBalancer::chooseForwardingSource()
     }
 }
 
-std::expected<boost::json::object, rpc::CombinedError>
-LoadBalancer::forwardToRippledImpl(
-    boost::json::object const& request,
-    std::optional<std::string> const& clientIp,
-    bool isAdmin,
-    boost::asio::yield_context yield
-)
-{
-    ++forwardingCounters_.cacheMiss.get();
-
-    ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
-    std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
-
-    auto numAttempts = 0u;
-
-    auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
-
-    std::optional<boost::json::object> response;
-    rpc::ClioError error = rpc::ClioError::EtlConnectionError;
-    while (numAttempts < sources_.size()) {
-        auto [res, duration] =
-            util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
-
-        if (res) {
-            forwardingCounters_.successDuration.get() += duration;
-            response = std::move(res).value();
-            break;
-        }
-        forwardingCounters_.failDuration.get() += duration;
-        ++forwardingCounters_.retries.get();
-        error = std::max(error, res.error());  // Choose the best result between all sources
-
-        sourceIdx = (sourceIdx + 1) % sources_.size();
-        ++numAttempts;
-    }
-
-    if (response.has_value()) {
-        return std::move(response).value();
-    }
-
-    return std::unexpected{error};
-}
-
 }  // namespace etlng

src/etlng/LoadBalancer.hpp

@@ -282,14 +282,6 @@ private:
      */
     void
     chooseForwardingSource();
-
-    std::expected<boost::json::object, rpc::CombinedError>
-    forwardToRippledImpl(
-        boost::json::object const& request,
-        std::optional<std::string> const& clientIp,
-        bool isAdmin,
-        boost::asio::yield_context yield
-    );
 };
 
 }  // namespace etlng

src/etlng/Models.hpp

@@ -22,7 +22,7 @@
 #include "util/Concepts.hpp"
 
 #include <boost/json/object.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/basics/Blob.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>

src/etlng/impl/ForwardingSource.cpp

@@ -28,7 +28,7 @@
 #include <boost/json/object.hpp>
 #include <boost/json/parse.hpp>
 #include <boost/json/serialize.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <chrono>
 #include <exception>

src/etlng/impl/GrpcSource.cpp

@@ -27,7 +27,7 @@
 #include "web/Resolver.hpp"
 
 #include <boost/asio/spawn.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <grpcpp/client_context.h>
 #include <grpcpp/security/credentials.h>
 #include <grpcpp/support/channel_arguments.h>

src/etlng/impl/LedgerPublisher.hpp

@@ -34,7 +34,7 @@
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/post.hpp>
 #include <boost/asio/strand.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/basics/chrono.h>
 #include <xrpl/protocol/Fees.h>
 #include <xrpl/protocol/LedgerHeader.h>
@@ -227,8 +227,9 @@ public:
     std::chrono::time_point<std::chrono::system_clock>
     getLastPublish() const override
     {
-        return std::chrono::time_point<std::chrono::system_clock>{std::chrono::seconds{lastPublishSeconds_.get().value()
-        }};
+        return std::chrono::time_point<std::chrono::system_clock>{
+            std::chrono::seconds{lastPublishSeconds_.get().value()}
+        };
     }
 
     /**

src/feed/SubscriptionManager.hpp

@@ -159,8 +159,10 @@ public:
      * @param transactions The transactions in the current ledger.
      */
     void
-    pubBookChanges(ripple::LedgerHeader const& lgrInfo, std::vector<data::TransactionAndMetadata> const& transactions)
-        final;
+    pubBookChanges(
+        ripple::LedgerHeader const& lgrInfo,
+        std::vector<data::TransactionAndMetadata> const& transactions
+    ) final;
 
     /**
      * @brief Subscribe to the proposed transactions feed.

src/feed/impl/ProposedTransactionFeed.cpp

@@ -57,9 +57,7 @@ ProposedTransactionFeed::sub(ripple::AccountID const& account, SubscriberSharedP
 {
     auto const weakPtr = std::weak_ptr(subscriber);
     auto const added = accountSignal_.connectTrackableSlot(
-        subscriber,
-        account,
-        [this, weakPtr](std::shared_ptr<std::string> const& msg) {
+        subscriber, account, [this, weakPtr](std::shared_ptr<std::string> const& msg) {
             if (auto connectionPtr = weakPtr.lock()) {
                 // Check if this connection already sent
                 if (notified_.contains(connectionPtr.get()))

src/feed/impl/ProposedTransactionFeed.hpp

@@ -31,7 +31,7 @@
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/strand.hpp>
 #include <boost/json/object.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/protocol/AccountID.h>
 #include <xrpl/protocol/Book.h>
 #include <xrpl/protocol/LedgerHeader.h>

src/feed/impl/TransactionFeed.cpp

@@ -280,7 +280,7 @@ TransactionFeed::pub(
                         data->getFieldAmount(ripple::sfTakerPays).issue(),
                         (*data)[~ripple::sfDomainID]
                     };
-                    if (affectedBooks.find(book) == affectedBooks.end()) {
+                    if (!affectedBooks.contains(book)) {
                         affectedBooks.insert(book);
                     }
                 }

src/feed/impl/TransactionFeed.hpp

@@ -33,7 +33,7 @@
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/strand.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/protocol/AccountID.h>
 #include <xrpl/protocol/Book.h>
 #include <xrpl/protocol/LedgerHeader.h>

src/feed/impl/Util.hpp

@@ -23,7 +23,7 @@
 #include "util/prometheus/Label.hpp"
 #include "util/prometheus/Prometheus.hpp"
 
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <string>
 

src/main/Main.cpp

@@ -25,6 +25,7 @@
 #include "util/TerminationHandler.hpp"
 #include "util/config/ConfigDefinition.hpp"
 #include "util/log/Logger.hpp"
+#include "util/prometheus/Prometheus.hpp"
 
 #include <cstdlib>
 #include <exception>
@@ -52,6 +53,7 @@ try {
             if (not app::parseConfig(run.configPath))
                 return EXIT_FAILURE;
 
+            PrometheusService::init(gClioConfig);
             if (auto const initSuccess = util::LogService::init(gClioConfig); not initSuccess) {
                 std::cerr << initSuccess.error() << std::endl;
                 return EXIT_FAILURE;

src/migration/MigrationManagerInterface.hpp

@@ -32,7 +32,7 @@ namespace migration {
  */
 struct MigrationManagerInterface : virtual public MigrationInspectorInterface {
     /**
-     * @brief Run the the migration according to the given migrator's name
+     * @brief Run the migration according to the given migrator's name
      */
     virtual void
     runMigration(std::string const&) = 0;

src/migration/README.md

@@ -60,7 +60,6 @@ Most indexes are based on either ledger states or transactions. We provide the `
 If you need to do full scan against other table, you can follow below steps:
 
 - Describe the table which needs full scan in a struct. It has to satisfy the `TableSpec`(cassandra/Spec.hpp) concept, containing static member:
-
   - Tuple type `Row`, it's the type of each field in a row. The order of types should match what database will return in a row. Key types should come first, followed by other field types sorted in alphabetical order.
   - `kPARTITION_KEY`, it's the name of the partition key of the table.
   - `kTABLE_NAME`

src/migration/cassandra/impl/CassandraMigrationSchema.hpp

@@ -24,7 +24,7 @@
 #include "data/cassandra/SettingsProvider.hpp"
 #include "data/cassandra/Types.hpp"
 
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <functional>
 #include <string>
@@ -63,16 +63,18 @@ public:
         std::string const& key
     )
     {
-        return handler.prepare(fmt::format(
-            R"(
+        return handler.prepare(
+            fmt::format(
+                R"(
             SELECT *
               FROM {}
              WHERE TOKEN({}) >= ? AND TOKEN({}) <= ?
             )",
-            data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), tableName),
-            key,
-            key
-        ));
+                data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), tableName),
+                key,
+                key
+            )
+        );
     }
 
     /**
@@ -84,14 +86,16 @@ public:
     data::cassandra::PreparedStatement const&
     getPreparedInsertMigratedMigrator(data::cassandra::Handle const& handler)
     {
-        static auto kPREPARED = handler.prepare(fmt::format(
-            R"(
+        static auto kPREPARED = handler.prepare(
+            fmt::format(
+                R"(
             INSERT INTO {}
                    (migrator_name, status)
             VALUES (?, ?)
             )",
-            data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), "migrator_status")
-        ));
+                data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), "migrator_status")
+            )
+        );
         return kPREPARED;
     }
 };

src/migration/impl/MigrationManagerBase.hpp

@@ -56,7 +56,7 @@ public:
     }
 
     /**
-     * @brief Run the the migration according to the given migrator's name
+     * @brief Run the migration according to the given migrator's name
      *
      * @param name The name of the migrator
      */

src/rpc/CMakeLists.txt

@@ -2,6 +2,7 @@
 add_library(clio_rpc_center)
 target_sources(clio_rpc_center PRIVATE RPCCenter.cpp)
 target_include_directories(clio_rpc_center PUBLIC "${CMAKE_SOURCE_DIR}/src")
+target_link_libraries(clio_rpc_center PUBLIC clio_options)
 
 add_library(clio_rpc)
 
@@ -51,6 +52,7 @@ target_sources(
           handlers/Subscribe.cpp
           handlers/TransactionEntry.cpp
           handlers/Unsubscribe.cpp
+          handlers/VaultInfo.cpp
 )
 
 target_link_libraries(clio_rpc PRIVATE clio_util)

src/rpc/Counters.cpp

@@ -25,7 +25,7 @@
 #include "util/prometheus/Prometheus.hpp"
 
 #include <boost/json/object.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/protocol/jss.h>
 
 #include <chrono>
@@ -40,41 +40,55 @@ using util::prometheus::Label;
 using util::prometheus::Labels;
 
 Counters::MethodInfo::MethodInfo(std::string const& method)
-    : started(PrometheusService::counterInt(
-          "rpc_method_total_number",
-          Labels{{{"status", "started"}, {"method", method}}},
-          fmt::format("Total number of started calls to the method {}", method)
-      ))
-    , finished(PrometheusService::counterInt(
-          "rpc_method_total_number",
-          Labels{{{"status", "finished"}, {"method", method}}},
-          fmt::format("Total number of finished calls to the method {}", method)
-      ))
-    , failed(PrometheusService::counterInt(
-          "rpc_method_total_number",
-          Labels{{{"status", "failed"}, {"method", method}}},
-          fmt::format("Total number of failed calls to the method {}", method)
-      ))
-    , errored(PrometheusService::counterInt(
-          "rpc_method_total_number",
-          Labels{{{"status", "errored"}, {"method", method}}},
-          fmt::format("Total number of errored calls to the method {}", method)
-      ))
-    , forwarded(PrometheusService::counterInt(
-          "rpc_method_total_number",
-          Labels{{{"status", "forwarded"}, {"method", method}}},
-          fmt::format("Total number of forwarded calls to the method {}", method)
-      ))
-    , failedForward(PrometheusService::counterInt(
-          "rpc_method_total_number",
-          Labels{{{"status", "failed_forward"}, {"method", method}}},
-          fmt::format("Total number of failed forwarded calls to the method {}", method)
-      ))
-    , duration(PrometheusService::counterInt(
-          "rpc_method_duration_us",
-          Labels({util::prometheus::Label{"method", method}}),
-          fmt::format("Total duration of calls to the method {}", method)
-      ))
+    : started(
+          PrometheusService::counterInt(
+              "rpc_method_total_number",
+              Labels{{{"status", "started"}, {"method", method}}},
+              fmt::format("Total number of started calls to the method {}", method)
+          )
+      )
+    , finished(
+          PrometheusService::counterInt(
+              "rpc_method_total_number",
+              Labels{{{"status", "finished"}, {"method", method}}},
+              fmt::format("Total number of finished calls to the method {}", method)
+          )
+      )
+    , failed(
+          PrometheusService::counterInt(
+              "rpc_method_total_number",
+              Labels{{{"status", "failed"}, {"method", method}}},
+              fmt::format("Total number of failed calls to the method {}", method)
+          )
+      )
+    , errored(
+          PrometheusService::counterInt(
+              "rpc_method_total_number",
+              Labels{{{"status", "errored"}, {"method", method}}},
+              fmt::format("Total number of errored calls to the method {}", method)
+          )
+      )
+    , forwarded(
+          PrometheusService::counterInt(
+              "rpc_method_total_number",
+              Labels{{{"status", "forwarded"}, {"method", method}}},
+              fmt::format("Total number of forwarded calls to the method {}", method)
+          )
+      )
+    , failedForward(
+          PrometheusService::counterInt(
+              "rpc_method_total_number",
+              Labels{{{"status", "failed_forward"}, {"method", method}}},
+              fmt::format("Total number of failed forwarded calls to the method {}", method)
+          )
+      )
+    , duration(
+          PrometheusService::counterInt(
+              "rpc_method_duration_us",
+              Labels({util::prometheus::Label{"method", method}}),
+              fmt::format("Total duration of calls to the method {}", method)
+          )
+      )
 {
 }
 
@@ -89,31 +103,41 @@ Counters::getMethodInfo(std::string const& method)
 }
 
 Counters::Counters(WorkQueue const& wq)
-    : tooBusyCounter_(PrometheusService::counterInt(
-          "rpc_error_total_number",
-          Labels({Label{"error_type", "too_busy"}}),
-          "Total number of too busy errors"
-      ))
-    , notReadyCounter_(PrometheusService::counterInt(
-          "rpc_error_total_number",
-          Labels({Label{"error_type", "not_ready"}}),
-          "Total number of not ready replyes"
-      ))
-    , badSyntaxCounter_(PrometheusService::counterInt(
-          "rpc_error_total_number",
-          Labels({Label{"error_type", "bad_syntax"}}),
-          "Total number of bad syntax replyes"
-      ))
-    , unknownCommandCounter_(PrometheusService::counterInt(
-          "rpc_error_total_number",
-          Labels({Label{"error_type", "unknown_command"}}),
-          "Total number of unknown command replyes"
-      ))
-    , internalErrorCounter_(PrometheusService::counterInt(
-          "rpc_error_total_number",
-          Labels({Label{"error_type", "internal_error"}}),
-          "Total number of internal errors"
-      ))
+    : tooBusyCounter_(
+          PrometheusService::counterInt(
+              "rpc_error_total_number",
+              Labels({Label{"error_type", "too_busy"}}),
+              "Total number of too busy errors"
+          )
+      )
+    , notReadyCounter_(
+          PrometheusService::counterInt(
+              "rpc_error_total_number",
+              Labels({Label{"error_type", "not_ready"}}),
+              "Total number of not ready replyes"
+          )
+      )
+    , badSyntaxCounter_(
+          PrometheusService::counterInt(
+              "rpc_error_total_number",
+              Labels({Label{"error_type", "bad_syntax"}}),
+              "Total number of bad syntax replyes"
+          )
+      )
+    , unknownCommandCounter_(
+          PrometheusService::counterInt(
+              "rpc_error_total_number",
+              Labels({Label{"error_type", "unknown_command"}}),
+              "Total number of unknown command replyes"
+          )
+      )
+    , internalErrorCounter_(
+          PrometheusService::counterInt(
+              "rpc_error_total_number",
+              Labels({Label{"error_type", "internal_error"}}),
+              "Total number of internal errors"
+          )
+      )
     , workQueue_(std::cref(wq))
     , startupTime_{std::chrono::system_clock::now()}
 {

src/rpc/Errors.cpp

@@ -89,7 +89,7 @@ getErrorInfo(ClioError code)
         {.code = ClioError::RpcMalformedAuthorizedCredentials,
          .error = "malformedAuthorizedCredentials",
          .message = "Malformed authorized credentials."},
-
+        {.code = ClioError::RpcEntryNotFound, .error = "entryNotFound", .message = "Entry Not Found."},
         // special system errors
         {.code = ClioError::RpcInvalidApiVersion, .error = JS(invalid_API_version), .message = "Invalid API version."},
         {.code = ClioError::RpcCommandIsMissing,

src/rpc/Errors.hpp

@@ -43,6 +43,7 @@ enum class ClioError {
     RpcFieldNotFoundTransaction = 5006,
     RpcMalformedOracleDocumentId = 5007,
     RpcMalformedAuthorizedCredentials = 5008,
+    RpcEntryNotFound = 5009,
 
     // special system errors start with 6000
     RpcInvalidApiVersion = 6000,

src/rpc/README.md

@@ -17,7 +17,6 @@ See [tests/unit/rpc](https://github.com/XRPLF/clio/tree/develop/tests/unit/rpc)
 Handlers need to fulfil the requirements specified by the `SomeHandler` concept (see `rpc/common/Concepts.hpp`):
 
 - Expose types:
-
   - `Input` - The POD struct which acts as input for the handler
 
   - `Output` - The POD struct which acts as output of a valid handler invocation

src/rpc/RPCCenter.cpp

@@ -30,6 +30,7 @@ std::unordered_set<std::string_view> const&
 handledRpcs()
 {
     static std::unordered_set<std::string_view> const kHANDLED_RPCS = {
+        // clang-format off
         "account_channels",
         "account_currencies",
         "account_info",
@@ -64,7 +65,9 @@ handledRpcs()
         "tx",
         "subscribe",
         "unsubscribe",
+        "vault_info",
         "version",
+        // clang-format on
     };
     return kHANDLED_RPCS;
 }

src/rpc/RPCEngine.hpp

@@ -37,7 +37,6 @@
 #include <boost/iterator/transform_iterator.hpp>
 #include <boost/json.hpp>
 #include <boost/json/object.hpp>
-#include <fmt/core.h>
 #include <fmt/format.h>
 #include <xrpl/protocol/ErrorCodes.h>
 
@@ -158,49 +157,55 @@ public:
             return forwardingProxy_.forward(ctx);
         }
 
-        if (not ctx.isAdmin and responseCache_ and responseCache_->shouldCache(ctx.method)) {
-            auto updater =
-                [this, &ctx](boost::asio::yield_context
-                ) -> std::expected<util::ResponseExpirationCache::EntryData, util::ResponseExpirationCache::Error> {
-                auto result = buildResponseImpl(ctx);
-
-                auto const extracted =
-                    [&result]() -> std::expected<boost::json::object, util::ResponseExpirationCache::Error> {
-                    if (result.response.has_value()) {
-                        return std::move(result.response).value();
-                    }
-                    return std::unexpected{util::ResponseExpirationCache::Error{
-                        .status = std::move(result.response).error(), .warnings = std::move(result.warnings)
-                    }};
-                }();
-
-                if (extracted.has_value()) {
-                    return util::ResponseExpirationCache::EntryData{
-                        .lastUpdated = std::chrono::steady_clock::now(), .response = std::move(extracted).value()
-                    };
-                }
-                return std::unexpected{std::move(extracted).error()};
-            };
-
-            auto result = responseCache_->getOrUpdate(
-                ctx.yield,
-                ctx.method,
-                std::move(updater),
-                [&ctx](util::ResponseExpirationCache::EntryData const& entry) {
-                    return not ctx.isAdmin and not entry.response.contains("error");
-                }
-            );
-            if (result.has_value()) {
-                return Result{std::move(result).value()};
-            }
-
-            auto error = std::move(result).error();
-            Result errorResult{std::move(error.status)};
-            errorResult.warnings = std::move(error.warnings);
-            return errorResult;
+        if (not ctx.isAdmin and responseCache_) {
+            if (auto res = responseCache_->get(ctx.method); res.has_value())
+                return Result{std::move(res).value()};
         }
 
-        return buildResponseImpl(ctx);
+        if (backend_->isTooBusy()) {
+            LOG(log_.error()) << "Database is too busy. Rejecting request";
+            notifyTooBusy();  // TODO: should we add ctx.method if we have it?
+            return Result{Status{RippledError::rpcTOO_BUSY}};
+        }
+
+        auto const method = handlerProvider_->getHandler(ctx.method);
+        if (!method) {
+            notifyUnknownCommand();
+            return Result{Status{RippledError::rpcUNKNOWN_COMMAND}};
+        }
+
+        try {
+            LOG(perfLog_.debug()) << ctx.tag() << " start executing rpc `" << ctx.method << '`';
+
+            auto const context = Context{
+                .yield = ctx.yield,
+                .session = ctx.session,
+                .isAdmin = ctx.isAdmin,
+                .clientIp = ctx.clientIp,
+                .apiVersion = ctx.apiVersion
+            };
+            auto v = (*method).process(ctx.params, context);
+
+            LOG(perfLog_.debug()) << ctx.tag() << " finish executing rpc `" << ctx.method << '`';
+
+            if (not v) {
+                notifyErrored(ctx.method);
+            } else if (not ctx.isAdmin and responseCache_) {
+                responseCache_->put(ctx.method, v.result->as_object());
+            }
+
+            return Result{std::move(v)};
+        } catch (data::DatabaseTimeout const& t) {
+            LOG(log_.error()) << "Database timeout";
+            notifyTooBusy();
+
+            return Result{Status{RippledError::rpcTOO_BUSY}};
+        } catch (std::exception const& ex) {
+            LOG(log_.error()) << ctx.tag() << "Caught exception: " << ex.what();
+            notifyInternalError();
+
+            return Result{Status{RippledError::rpcINTERNAL}};
+        }
     }
 
     /**

src/rpc/RPCHelpers.cpp

@@ -45,7 +45,7 @@
 #include <boost/json/value_to.hpp>
 #include <boost/lexical_cast.hpp>
 #include <boost/lexical_cast/bad_lexical_cast.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/basics/Slice.h>
 #include <xrpl/basics/StringUtilities.h>
 #include <xrpl/basics/base_uint.h>
@@ -1283,9 +1283,10 @@ postProcessOrderBook(
             } else {
                 saTakerGetsFunded = saOwnerFundsLimit;
                 offerJson["taker_gets_funded"] = toBoostJson(saTakerGetsFunded.getJson(ripple::JsonOptions::none));
-                offerJson["taker_pays_funded"] =
-                    toBoostJson(std::min(saTakerPays, ripple::multiply(saTakerGetsFunded, dirRate, saTakerPays.issue()))
-                                    .getJson(ripple::JsonOptions::none));
+                offerJson["taker_pays_funded"] = toBoostJson(
+                    std::min(saTakerPays, ripple::multiply(saTakerGetsFunded, dirRate, saTakerPays.issue()))
+                        .getJson(ripple::JsonOptions::none)
+                );
             }
 
             ripple::STAmount const saOwnerPays = (ripple::parityRate == offerRate)

src/rpc/RPCHelpers.hpp

@@ -28,6 +28,7 @@
 #include "data/BackendInterface.hpp"
 #include "data/Types.hpp"
 #include "rpc/Errors.hpp"
+#include "rpc/JS.hpp"
 #include "rpc/common/Types.hpp"
 #include "util/JsonUtils.hpp"
 #include "util/Taggable.hpp"
@@ -41,7 +42,8 @@
 #include <boost/regex.hpp>
 #include <boost/regex/v5/regex_fwd.hpp>
 #include <boost/regex/v5/regex_match.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
+#include <xrpl/basics/Number.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/json/json_value.h>
 #include <xrpl/protocol/AccountID.h>
@@ -50,9 +52,12 @@
 #include <xrpl/protocol/Indexes.h>
 #include <xrpl/protocol/Issue.h>
 #include <xrpl/protocol/Keylet.h>
+#include <xrpl/protocol/LedgerFormats.h>
 #include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/MPTIssue.h>
 #include <xrpl/protocol/PublicKey.h>
 #include <xrpl/protocol/Rate.h>
+#include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/STAmount.h>
 #include <xrpl/protocol/STBase.h>
 #include <xrpl/protocol/STLedgerEntry.h>
@@ -60,9 +65,11 @@
 #include <xrpl/protocol/STTx.h>
 #include <xrpl/protocol/SecretKey.h>
 #include <xrpl/protocol/Seed.h>
+#include <xrpl/protocol/Serializer.h>
 #include <xrpl/protocol/TxMeta.h>
 #include <xrpl/protocol/UintTypes.h>
 #include <xrpl/protocol/XRPAmount.h>
+#include <xrpl/protocol/jss.h>
 
 #include <chrono>
 #include <cstddef>

src/rpc/WorkQueue.cpp

@@ -46,7 +46,8 @@ WorkQueue::OneTimeCallable::operator()()
         called_ = true;
     }
 }
-WorkQueue::OneTimeCallable::operator bool() const
+WorkQueue::OneTimeCallable::
+operator bool() const
 {
     return func_.operator bool();
 }

src/rpc/common/Checkers.hpp

@@ -24,7 +24,7 @@
 
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <optional>
 #include <string>

src/rpc/common/MetaProcessors.hpp

@@ -25,7 +25,7 @@
 #include "rpc/common/Types.hpp"
 
 #include <boost/json/value.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <cstddef>
 #include <functional>
@@ -107,8 +107,8 @@ public:
     template <SomeRequirement... Requirements>
     explicit IfType(Requirements&&... requirements)
         : processor_(
-              [... r = std::forward<Requirements>(requirements
-               )](boost::json::value& j, std::string_view key) -> MaybeError {
+              [... r = std::forward<Requirements>(requirements)](boost::json::value& j, std::string_view key)
+                  -> MaybeError {
                   std::optional<Status> firstFailure = std::nullopt;
 
                   // the check logic is the same as fieldspec

src/rpc/common/Validators.cpp

@@ -28,7 +28,7 @@
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/basics/StringUtilities.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/protocol/AccountID.h>

src/rpc/common/Validators.hpp

@@ -26,7 +26,7 @@
 #include <boost/json/array.hpp>
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/protocol/ErrorCodes.h>
 
@@ -457,7 +457,10 @@ public:
 checkIsU32Numeric(std::string_view sv);
 
 template <class HexType>
-    requires(std::is_same_v<HexType, ripple::uint160> || std::is_same_v<HexType, ripple::uint192> || std::is_same_v<HexType, ripple::uint256>)
+    requires(
+        std::is_same_v<HexType, ripple::uint160> || std::is_same_v<HexType, ripple::uint192> ||
+        std::is_same_v<HexType, ripple::uint256>
+    )
 MaybeError
 makeHexStringValidator(boost::json::value const& value, std::string_view key)
 {

src/rpc/common/impl/APIVersionParser.cpp

@@ -23,7 +23,7 @@
 #include "util/log/Logger.hpp"
 
 #include <boost/json/object.hpp>
-#include <fmt/core.h>
+#include <fmt/format.h>
 
 #include <cstdint>
 #include <expected>

src/rpc/common/impl/HandlerProvider.cpp

@@ -60,6 +60,7 @@
 #include "rpc/handlers/TransactionEntry.hpp"
 #include "rpc/handlers/Tx.hpp"
 #include "rpc/handlers/Unsubscribe.hpp"
+#include "rpc/handlers/VaultInfo.hpp"
 #include "rpc/handlers/VersionHandler.hpp"
 #include "util/config/ConfigDefinition.hpp"
 
@@ -114,6 +115,7 @@ ProductionHandlerProvider::ProductionHandlerProvider(
           {"tx", {.handler = TxHandler{backend, etl}}},
           {"subscribe", {.handler = SubscribeHandler{backend, amendmentCenter, subscriptionManager}}},
           {"unsubscribe", {.handler = UnsubscribeHandler{subscriptionManager}}},
+          {"vault_info", {.handler = VaultInfoHandler{backend}}},
           {"version", {.handler = VersionHandler{config}}},
       }
 {