chore: Commits for 2.5.0-b3 (#2197)

Merge right after: https://github.com/XRPLF/clio/pull/2178
Waits for: https://github.com/XRPLF/rippled/pull/5462

.github/actions/build_clio/action.yml

@@ -25,5 +25,5 @@ runs:
         cd build
         cmake \
           --build . \
-          --parallel ${{ steps.number_of_threads.outputs.threads_number }} \
+          --parallel "${{ steps.number_of_threads.outputs.threads_number }}" \
           --target ${{ inputs.targets }}

.github/actions/build_docker_image/action.yml

@@ -5,9 +5,6 @@ inputs:
   images:
     description: Name of the images to use as a base name
     required: true
-  dockerhub_repo:
-    description: DockerHub repository name
-    required: true
   push_image:
     description: Whether to push the image to the registry (true/false)
     required: true
@@ -20,15 +17,19 @@ inputs:
   platforms:
     description: Platforms to build the image for (e.g. linux/amd64,linux/arm64)
     required: true
-  description:
+
+  dockerhub_repo:
+    description: DockerHub repository name
+    required: false
+  dockerhub_description:
     description: Short description of the image
-    required: true
+    required: false
 
 runs:
   using: composite
   steps:
     - name: Login to DockerHub
-      if: ${{ inputs.push_image == 'true' }}
+      if: ${{ inputs.push_image == 'true' && inputs.dockerhub_repo != '' }}
       uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
       with:
         username: ${{ env.DOCKERHUB_USER }}
@@ -54,7 +55,7 @@ runs:
         tags: ${{ inputs.tags }}
 
     - name: Build and push
-      uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
+      uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
       with:
         context: ${{ inputs.directory }}
         platforms: ${{ inputs.platforms }}
@@ -62,11 +63,11 @@ runs:
         tags: ${{ steps.meta.outputs.tags }}
 
     - name: Update DockerHub description
-      if: ${{ inputs.push_image == 'true' }}
+      if: ${{ inputs.push_image == 'true' && inputs.dockerhub_repo != '' }}
       uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2
       with:
         username: ${{ env.DOCKERHUB_USER }}
         password: ${{ env.DOCKERHUB_PW }}
         repository: ${{ inputs.dockerhub_repo }}
-        short-description: ${{ inputs.description }}
+        short-description: ${{ inputs.dockerhub_description }}
         readme-filepath: ${{ inputs.directory }}/README.md

.github/actions/create_issue/action.yml

@@ -15,7 +15,7 @@ inputs:
   assignees:
     description: Comma-separated list of assignees
     required: true
-    default: "godexsoft,kuznetsss,PeterChen13579"
+    default: "godexsoft,kuznetsss,PeterChen13579,mathbunnyru"
 
 outputs:
   created_issue_id:
@@ -36,6 +36,6 @@ runs:
           --title '${{ inputs.title }}' \
           --body-file ./issue.md \
           > create_issue.log
-        created_issue=$(cat create_issue.log | sed 's|.*/||')
+        created_issue="$(sed 's|.*/||' create_issue.log)"
         echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
         rm create_issue.log issue.md

.github/actions/generate/action.yml

@@ -49,7 +49,7 @@ runs:
     - name: Run conan
       shell: bash
       env:
-        BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '' }}"
+        BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '*' }}"
         CODE_COVERAGE: "${{ inputs.code_coverage == 'true' && 'True' || 'False' }}"
         STATIC_OPTION: "${{ inputs.static == 'true' && 'True' || 'False' }}"
         INTEGRATION_TESTS_OPTION: "${{ inputs.build_integration_tests == 'true' && 'True' || 'False' }}"
@@ -59,21 +59,21 @@ runs:
         conan \
           install .. \
           -of . \
-          -b $BUILD_OPTION \
-          -s build_type=${{ inputs.build_type }} \
-          -o clio:static="${STATIC_OPTION}" \
-          -o clio:tests=True \
-          -o clio:integration_tests="${INTEGRATION_TESTS_OPTION}" \
-          -o clio:lint=False \
-          -o clio:coverage="${CODE_COVERAGE}" \
-          -o clio:time_trace="${TIME_TRACE}" \
-          --profile ${{ inputs.conan_profile }}
+          -b "$BUILD_OPTION" \
+          -s "build_type=${{ inputs.build_type }}" \
+          -o "&:static=${STATIC_OPTION}" \
+          -o "&:tests=True" \
+          -o "&:integration_tests=${INTEGRATION_TESTS_OPTION}" \
+          -o "&:lint=False" \
+          -o "&:coverage=${CODE_COVERAGE}" \
+          -o "&:time_trace=${TIME_TRACE}" \
+          --profile:all "${{ inputs.conan_profile }}"
 
     - name: Run cmake
       shell: bash
       env:
         BUILD_TYPE: "${{ inputs.build_type }}"
-        SANITIZER_OPTION: |
+        SANITIZER_OPTION: |-
           ${{ inputs.sanitizer == 'tsan' && '-Dsan=thread' ||
               inputs.sanitizer == 'ubsan' && '-Dsan=undefined' ||
               inputs.sanitizer == 'asan' && '-Dsan=address' ||
@@ -83,6 +83,6 @@ runs:
         cmake \
           -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
           -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
-          ${SANITIZER_OPTION} \
+          "${SANITIZER_OPTION}" \
           .. \
           -G Ninja

.github/actions/get_number_of_threads/action.yml

@@ -30,7 +30,7 @@ runs:
       id: number_of_threads_export
       shell: bash
       run: |
-        num_of_threads=${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }}
-        shift_by=${{ inputs.subtract_threads }}
-        shifted=$((num_of_threads - shift_by))
+        num_of_threads="${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }}"
+        shift_by="${{ inputs.subtract_threads }}"
+        shifted="$((num_of_threads - shift_by))"
         echo "num=$(( shifted > 1 ? shifted : 1 ))" >> $GITHUB_OUTPUT

.github/actions/git_common_ancestor/action.yml

@@ -13,4 +13,4 @@ runs:
       id: find_common_ancestor
       shell: bash
       run: |
-        echo "commit=$(git merge-base --fork-point origin/develop)" >> $GITHUB_OUTPUT
+        echo "commit=\"$(git merge-base --fork-point origin/develop)\"" >> $GITHUB_OUTPUT

.github/actions/prepare_runner/action.yml

@@ -18,13 +18,13 @@ runs:
           ca-certificates \
           ccache \
           clang-build-analyzer \
-          conan@1 \
+          conan \
           gh \
           jq \
           llvm@14 \
           ninja \
           pkg-config
-        echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
+        echo "/opt/homebrew/opt/conan@2/bin" >> $GITHUB_PATH
 
     - name: Install CMake 3.31.6 on mac
       if: ${{ runner.os == 'macOS' }}
@@ -38,7 +38,7 @@ runs:
         FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4"
 
         mkdir -p /tmp/homebrew-formula
-        curl -s -L $FORMULA_URL -o /tmp/homebrew-formula/cmake.rb
+        curl -s -L "$FORMULA_URL" -o /tmp/homebrew-formula/cmake.rb
 
         echo "$FORMULA_EXPECTED_SHA256  /tmp/homebrew-formula/cmake.rb" | shasum -a 256 -c
 
@@ -48,21 +48,21 @@ runs:
     - name: Fix git permissions on Linux
       if: ${{ runner.os == 'Linux' }}
       shell: bash
-      run: git config --global --add safe.directory $PWD
+      run: git config --global --add safe.directory "$PWD"
 
     - name: Set env variables for macOS
       if: ${{ runner.os == 'macOS' }}
       shell: bash
       run: |
         echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
-        echo "CONAN_USER_HOME=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "CONAN_HOME=${{ github.workspace }}/.conan2" >> $GITHUB_ENV
 
     - name: Set env variables for Linux
       if: ${{ runner.os == 'Linux' }}
       shell: bash
       run: |
         echo "CCACHE_DIR=/root/.ccache" >> $GITHUB_ENV
-        echo "CONAN_USER_HOME=/root/" >> $GITHUB_ENV
+        echo "CONAN_HOME=/root/.conan2" >> $GITHUB_ENV
 
     - name: Set CCACHE_DISABLE=1
       if: ${{ inputs.disable_ccache == 'true' }}
@@ -73,5 +73,5 @@ runs:
     - name: Create directories
       shell: bash
       run: |
-        mkdir -p $CCACHE_DIR
-        mkdir -p $CONAN_USER_HOME/.conan
+        mkdir -p "$CCACHE_DIR"
+        mkdir -p "$CONAN_HOME"

.github/actions/restore_cache/action.yml

@@ -3,7 +3,7 @@ description: Find and restores conan and ccache cache
 
 inputs:
   conan_dir:
-    description: Path to .conan directory
+    description: Path to Conan directory
     required: true
   conan_profile:
     description: Conan profile name
@@ -41,10 +41,10 @@ runs:
       id: conan_hash
       shell: bash
       run: |
-        conan info . -j info.json -o clio:tests=True
-        packages_info=$(cat info.json | jq '.[] | "\(.display_name): \(.id)"' | grep -v 'clio')
+        conan graph info . --format json --out-file info.json -o '&:tests=True' --profile:all ${{ inputs.conan_profile }}
+        packages_info="$(cat info.json | jq -r '.graph.nodes[]?.ref' | grep -v 'clio')"
         echo "$packages_info"
-        hash=$(echo "$packages_info" | shasum -a 256 | cut -d ' ' -f 1)
+        hash="$(echo "$packages_info" | shasum -a 256 | cut -d ' ' -f 1)"
         rm info.json
         echo "hash=$hash" >> $GITHUB_OUTPUT
 
@@ -52,7 +52,7 @@ runs:
       uses: actions/cache/restore@v4
       id: conan_cache
       with:
-        path: ${{ inputs.conan_dir }}/data
+        path: ${{ inputs.conan_dir }}/p
         key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ steps.conan_hash.outputs.hash }}
 
     - name: Restore ccache cache

.github/actions/save_cache/action.yml

@@ -42,13 +42,13 @@ runs:
       if: ${{ inputs.conan_cache_hit != 'true' }}
       shell: bash
       run: |
-        conan remove "*" -s -b -f
+        conan cache clean --source --build --temp
 
     - name: Save conan cache
       if: ${{ inputs.conan_cache_hit != 'true' }}
       uses: actions/cache/save@v4
       with:
-        path: ${{ inputs.conan_dir }}/data
+        path: ${{ inputs.conan_dir }}/p
         key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ inputs.conan_hash }}
 
     - name: Save ccache cache

.github/actions/setup_conan/action.yml

@@ -12,22 +12,11 @@ runs:
     - name: Create conan profile on macOS
       if: ${{ runner.os == 'macOS' }}
       shell: bash
-      env:
-        CONAN_PROFILE: ${{ inputs.conan_profile }}
       run: |
-        echo "Creating $CONAN_PROFILE conan profile"
-        conan profile new $CONAN_PROFILE --detect --force
-        conan profile update settings.compiler.libcxx=libc++ $CONAN_PROFILE
-        conan profile update settings.compiler.cppstd=20 $CONAN_PROFILE
-        conan profile update env.CXXFLAGS=-DBOOST_ASIO_DISABLE_CONCEPTS $CONAN_PROFILE
-        conan profile update "conf.tools.build:cxxflags+=[\"-DBOOST_ASIO_DISABLE_CONCEPTS\"]" $CONAN_PROFILE
+        conan profile detect --name "${{ inputs.conan_profile }}" --force
+        sed -i '' 's/compiler.cppstd=[^ ]*/compiler.cppstd=20/' "${{ env.CONAN_HOME }}/profiles/${{ inputs.conan_profile }}"
 
-    - name: Add conan-non-prod artifactory
+    - name: Add artifactory remote
       shell: bash
       run: |
-        if [[ -z $(conan remote list | grep conan-non-prod) ]]; then
-            echo "Adding conan-non-prod"
-            conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
-        else
-            echo "Conan-non-prod is available"
-        fi
+        conan remote add --index 0 --force ripple http://18.143.149.228:8081/artifactory/api/conan/dev

.github/scripts/prepare-release-artifacts.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex -o pipefail
+
+BINARY_NAME="clio_server"
+
+ARTIFACTS_DIR="$1"
+if [ -z "${ARTIFACTS_DIR}" ]; then
+    echo "Usage: $0 <artifacts_directory>"
+    exit 1
+fi
+
+cd "${ARTIFACTS_DIR}" || exit 1
+
+for artifact_name in $(ls); do
+    pushd "${artifact_name}" || exit 1
+    zip -r "../${artifact_name}.zip" ./${BINARY_NAME}
+    popd || exit 1
+
+    rm "${artifact_name}/${BINARY_NAME}"
+    rm -r "${artifact_name}"
+
+    sha256sum "./${artifact_name}.zip" > "./${artifact_name}.zip.sha256sum"
+done

.github/workflows/build.yml

@@ -5,6 +5,25 @@ on:
     branches: [master, release/*, develop]
   pull_request:
     branches: [master, release/*, develop]
+    paths:
+      - .github/workflows/build.yml
+
+      - .github/workflows/build_and_test.yml
+      - .github/workflows/build_impl.yml
+      - .github/workflows/test_impl.yml
+      - .github/workflows/upload_coverage_report.yml
+
+      - ".github/actions/**"
+      - "!.github/actions/build_docker_image/**"
+      - "!.github/actions/create_issue/**"
+
+      - CMakeLists.txt
+      - conanfile.py
+      - "cmake/**"
+      - "src/**"
+      - "tests/**"
+
+      - docs/config-description.md
   workflow_dispatch:
 
 concurrency:
@@ -80,23 +99,23 @@ jobs:
         shell: bash
         run: |
           repoConfigFile=docs/config-description.md
-          if ! [ -f ${repoConfigFile} ]; then
+          if ! [ -f "${repoConfigFile}" ]; then
             echo "Config Description markdown file is missing in docs folder"
             exit 1
           fi
 
           chmod +x ./clio_server
           configDescriptionFile=config_description_new.md
-          ./clio_server -d ${configDescriptionFile}
+          ./clio_server -d "${configDescriptionFile}"
 
-          configDescriptionHash=$(sha256sum ${configDescriptionFile} | cut -d' ' -f1)
-          repoConfigHash=$(sha256sum ${repoConfigFile} | cut -d' ' -f1)
+          configDescriptionHash=$(sha256sum "${configDescriptionFile}" | cut -d' ' -f1)
+          repoConfigHash=$(sha256sum "${repoConfigFile}" | cut -d' ' -f1)
 
-          if [ ${configDescriptionHash} != ${repoConfigHash} ]; then
+          if [ "${configDescriptionHash}" != "${repoConfigHash}" ]; then
             echo "Markdown file is not up to date"
             diff -u "${repoConfigFile}" "${configDescriptionFile}"
-            rm -f ${configDescriptionFile}
+            rm -f "${configDescriptionFile}"
             exit 1
           fi
-          rm -f ${configDescriptionFile}
+          rm -f "${configDescriptionFile}"
           exit 0

.github/workflows/build_clio_docker_image.yml

@@ -57,7 +57,7 @@ jobs:
         if: ${{ inputs.clio_server_binary_url != null }}
         shell: bash
         run: |
-          wget ${{inputs.clio_server_binary_url}} -P ./docker/clio/artifact/
+          wget "${{inputs.clio_server_binary_url}}" -P ./docker/clio/artifact/
           if [ "$(sha256sum ./docker/clio/clio_server | awk '{print $1}')" != "${{inputs.binary_sha256}}" ]; then
             echo "Binary sha256 sum doesn't match"
             exit 1
@@ -85,16 +85,16 @@ jobs:
       - name: Build Docker image
         uses: ./.github/actions/build_docker_image
         env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
           DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           images: |
-            rippleci/clio
             ghcr.io/xrplf/clio
-          dockerhub_repo: rippleci/clio
+            rippleci/clio
           push_image: ${{ inputs.publish_image }}
           directory: docker/clio
           tags: ${{ inputs.tags }}
           platforms: linux/amd64
-          description: Clio is an XRP Ledger API server.
+          dockerhub_repo: rippleci/clio
+          dockerhub_description: Clio is an XRP Ledger API server.

.github/workflows/build_impl.yml

@@ -92,7 +92,7 @@ jobs:
         uses: ./.github/actions/restore_cache
         id: restore_cache
         with:
-          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          conan_dir: ${{ env.CONAN_HOME }}
           conan_profile: ${{ inputs.conan_profile }}
           ccache_dir: ${{ env.CCACHE_DIR }}
           build_type: ${{ inputs.build_type }}
@@ -172,7 +172,7 @@ jobs:
         if: ${{ !inputs.disable_cache && github.ref == 'refs/heads/develop' }}
         uses: ./.github/actions/save_cache
         with:
-          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          conan_dir: ${{ env.CONAN_HOME }}
           conan_hash: ${{ steps.restore_cache.outputs.conan_hash }}
           conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
           ccache_dir: ${{ env.CCACHE_DIR }}

.github/workflows/clang-tidy.yml

@@ -7,9 +7,10 @@ on:
   pull_request:
     branches: [develop]
     paths:
-      - .clang_tidy
       - .github/workflows/clang-tidy.yml
 
+      - .clang_tidy
+
 concurrency:
   # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -48,7 +49,7 @@ jobs:
         uses: ./.github/actions/restore_cache
         id: restore_cache
         with:
-          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          conan_dir: ${{ env.CONAN_HOME }}
           ccache_dir: ${{ env.CCACHE_DIR }}
           conan_profile: ${{ env.CONAN_PROFILE }}
 
@@ -68,13 +69,14 @@ jobs:
         shell: bash
         id: run_clang_tidy
         run: |
-          run-clang-tidy-19 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
+          run-clang-tidy-19 -p build -j "${{ steps.number_of_threads.outputs.threads_number }}" -fix -quiet 1>output.txt
 
-      - name: Fix local includes
+      - name: Fix local includes and clang-format style
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
-        continue-on-error: true
         shell: bash
-        run: pre-commit run --all-files fix-local-includes
+        run: |
+          pre-commit run --all-files fix-local-includes || true
+          pre-commit run --all-files clang-format || true
 
       - name: Print issues found
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
@@ -85,7 +87,7 @@ jobs:
           rm output.txt
 
       - name: Create an issue
-        if: ${{ steps.run_clang_tidy.outcome != 'success' }}
+        if: ${{ steps.run_clang_tidy.outcome != 'success' && github.event_name != 'pull_request' }}
         id: create_issue
         uses: ./.github/actions/create_issue
         env:
@@ -98,7 +100,7 @@ jobs:
             List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/
 
       - uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec # v6.3.0
-        if: ${{ steps.run_clang_tidy.outcome != 'success' }}
+        if: ${{ steps.run_clang_tidy.outcome != 'success' && github.event_name != 'pull_request' }}
         with:
           gpg_private_key: ${{ secrets.ACTIONS_GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.ACTIONS_GPG_PASSPHRASE }}
@@ -106,7 +108,7 @@ jobs:
           git_commit_gpgsign: true
 
       - name: Create PR with fixes
-        if: ${{ steps.run_clang_tidy.outcome != 'success' }}
+        if: ${{ steps.run_clang_tidy.outcome != 'success' && github.event_name != 'pull_request' }}
         uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
         env:
           GH_REPO: ${{ github.repository }}
@@ -119,7 +121,7 @@ jobs:
           delete-branch: true
           title: "style: clang-tidy auto fixes"
           body: "Fixes #${{ steps.create_issue.outputs.created_issue_id }}. Please review and commit clang-tidy fixes."
-          reviewers: "godexsoft,kuznetsss,PeterChen13579"
+          reviewers: "godexsoft,kuznetsss,PeterChen13579,mathbunnyru"
 
       - name: Fail the job
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}

.github/workflows/clang-tidy_on_fix_merged.yml

@@ -18,8 +18,8 @@ jobs:
         id: check
         shell: bash
         run: |
-          passed=$(if [[ $(git log -1 --pretty=format:%s | grep 'style: clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi)
-          echo "passed=$passed" >> $GITHUB_OUTPUT
+          passed=$(if [[ "$(git log -1 --pretty=format:%s | grep 'style: clang-tidy auto fixes')" ]]; then echo 'true' ; else echo 'false' ; fi)
+          echo "passed=\"$passed\"" >> $GITHUB_OUTPUT
 
       - name: Run clang-tidy workflow
         if: ${{ contains(steps.check.outputs.passed, 'true') }}

.github/workflows/docs.yml

@@ -5,23 +5,14 @@ on:
     branches: [develop]
   workflow_dispatch:
 
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
 concurrency:
   # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
+  build:
     runs-on: ubuntu-latest
-    continue-on-error: true
     container:
       image: ghcr.io/xrplf/clio-ci:latest
 
@@ -31,10 +22,16 @@ jobs:
         with:
           lfs: true
 
-      - name: Build docs
-        run: |
-          mkdir -p build_docs && cd build_docs
-          cmake ../docs && cmake --build . --target docs
+      - name: Create build directory
+        run: mkdir build_docs
+
+      - name: Configure CMake
+        working-directory: build_docs
+        run: cmake ../docs
+
+      - name: Build
+        working-directory: build_docs
+        run: cmake --build . --target docs
 
       - name: Setup Pages
         uses: actions/configure-pages@v5
@@ -45,6 +42,19 @@ jobs:
           path: build_docs/html
           name: docs-develop
 
+  deploy:
+    needs: build
+    permissions:
+      pages: write
+      id-token: write
+
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    runs-on: ubuntu-latest
+
+    steps:
       - name: Deploy to GitHub Pages
         id: deployment
         uses: actions/deploy-pages@v4

.github/workflows/nightly.yml

@@ -6,8 +6,17 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - ".github/workflows/nightly.yml"
-      - ".github/workflows/build_clio_docker_image.yml"
+      - .github/workflows/nightly.yml
+
+      - .github/workflows/release_impl.yml
+      - .github/workflows/build_and_test.yml
+      - .github/workflows/build_impl.yml
+      - .github/workflows/test_impl.yml
+      - .github/workflows/build_clio_docker_image.yml
+
+      - ".github/actions/**"
+      - "!.github/actions/code_coverage/**"
+      - .github/scripts/prepare-release-artifacts.sh
 
 concurrency:
   # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
@@ -26,16 +35,25 @@ jobs:
             conan_profile: default_apple_clang
             build_type: Release
             static: false
+            sanitizer: "false"
           - os: heavy
             conan_profile: gcc
             build_type: Release
             static: true
             container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
+            sanitizer: "false"
           - os: heavy
             conan_profile: gcc
             build_type: Debug
             static: true
             container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
+            sanitizer: "false"
+          - os: heavy
+            conan_profile: gcc.ubsan
+            build_type: Release
+            static: false
+            container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
+            sanitizer: "ubsan"
 
     uses: ./.github/workflows/build_and_test.yml
     with:
@@ -45,9 +63,10 @@ jobs:
       build_type: ${{ matrix.build_type }}
       static: ${{ matrix.static }}
       run_unit_tests: true
-      run_integration_tests: ${{ matrix.os != 'macos15' }}
+      run_integration_tests: true
       upload_clio_server: true
       disable_cache: true
+      sanitizer: ${{ matrix.sanitizer }}
 
   analyze_build_time:
     name: Analyze Build Time
@@ -56,13 +75,10 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # TODO: Enable when we have at least ubuntu 22.04
-          # as ClangBuildAnalyzer requires relatively modern glibc
-          #
-          # - os: heavy
-          #   conan_profile: clang
-          #   container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
-          #   static: true
+          - os: heavy
+            conan_profile: clang
+            container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
+            static: true
           - os: macos15
             conan_profile: default_apple_clang
             container: ""
@@ -88,7 +104,14 @@ jobs:
       overwrite_release: true
       title: "Clio development (nightly) build"
       version: nightly
-      notes_header_file: nightly_notes.md
+      header: >
+        # Release notes
+
+        > **Note:** Please remember that this is a development release and it is not recommended for production use.
+
+        Changelog (including previous releases): <https://github.com/XRPLF/clio/commits/nightly>
+      generate_changelog: false
+      draft: false
 
   build_and_publish_docker_image:
     uses: ./.github/workflows/build_clio_docker_image.yml

.github/workflows/nightly_notes.md

@@ -1,7 +0,0 @@
-# Release notes
-
-> **Note:** Please remember that this is a development release and it is not recommended for production use.
-
-Changelog (including previous releases): <https://github.com/XRPLF/clio/commits/nightly>
-
-## SHA256 checksums

.github/workflows/pre-commit-autoupdate.yml

@@ -23,8 +23,8 @@ jobs:
           python-version: 3.x
 
       - run: pip install pre-commit
-      - run: pre-commit autoupdate
-      - run: pre-commit run --all-files
+      - run: pre-commit autoupdate --freeze
+      - run: pre-commit run --all-files || true
 
       - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
         if: always()
@@ -33,7 +33,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         with:
           branch: update/pre-commit-hooks
-          title: Update pre-commit hooks
-          commit-message: "style: update pre-commit hooks"
+          title: "style: Update pre-commit hooks"
+          commit-message: "style: Update pre-commit hooks"
           body: Update versions of pre-commit hooks to latest version.
           reviewers: "godexsoft,kuznetsss,PeterChen13579,mathbunnyru"

.github/workflows/release.yml

@@ -0,0 +1,56 @@
+name: Create release
+
+on:
+  push:
+    tags:
+      - "*.*.*"
+  pull_request:
+    paths:
+      - .github/workflows/release.yml
+
+concurrency:
+  # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-and-test:
+    name: Build and Test
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: macos15
+            conan_profile: default_apple_clang
+            build_type: Release
+            static: false
+          - os: heavy
+            conan_profile: gcc
+            build_type: Release
+            static: true
+            container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
+
+    uses: ./.github/workflows/build_and_test.yml
+    with:
+      runs_on: ${{ matrix.os }}
+      container: ${{ matrix.container }}
+      conan_profile: ${{ matrix.conan_profile }}
+      build_type: ${{ matrix.build_type }}
+      static: ${{ matrix.static }}
+      run_unit_tests: true
+      run_integration_tests: true
+      upload_clio_server: true
+      disable_cache: true
+
+  release:
+    needs: build-and-test
+    uses: ./.github/workflows/release_impl.yml
+    with:
+      overwrite_release: false
+      title: "${{ github.ref_name}}"
+      version: "${{ github.ref_name }}"
+      header: >
+        # Introducing Clio version ${{ github.ref_name }}
+      generate_changelog: true
+      draft: true

.github/workflows/release_impl.yml

@@ -18,14 +18,26 @@ on:
         required: true
         type: string
 
-      notes_header_file:
-        description: "Release notes header file"
+      header:
+        description: "Release notes header"
         required: true
         type: string
 
+      generate_changelog:
+        description: "Generate changelog"
+        required: false
+        type: boolean
+
+      draft:
+        description: "Create a draft release"
+        required: false
+        type: boolean
+
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: heavy
+    container:
+      image: ghcr.io/xrplf/clio-ci:latest
     env:
       GH_REPO: ${{ github.repository }}
       GH_TOKEN: ${{ github.token }}
@@ -35,29 +47,55 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+          disable_ccache: true
 
       - uses: actions/download-artifact@v4
         with:
           path: release_artifacts
           pattern: clio_server_*
 
-      - name: Prepare files
+      - name: Create release notes
+        shell: bash
+        run: |
+          printf '%s\n' "${{ inputs.header }}" > "${RUNNER_TEMP}/release_notes.md"
+
+      - name: Generate changelog
+        shell: bash
+        if: ${{ inputs.generate_changelog }}
+        run: |
+          LAST_TAG=$(gh release view --json tagName -q .tagName)
+          LAST_TAG_COMMIT=$(git rev-parse $LAST_TAG)
+          BASE_COMMIT=$(git merge-base HEAD $LAST_TAG_COMMIT)
+          git-cliff "${BASE_COMMIT}..HEAD" --ignore-tags "nightly|-b"
+          cat CHANGELOG.md >> "${RUNNER_TEMP}/release_notes.md"
+
+      - name: Prepare release artifacts
+        shell: bash
+        run: .github/scripts/prepare-release-artifacts.sh release_artifacts
+
+      - name: Append sha256 checksums
         shell: bash
         working-directory: release_artifacts
         run: |
-          cp ${{ github.workspace }}/.github/workflows/${{ inputs.notes_header_file }} "${RUNNER_TEMP}/release_notes.md"
-          echo '' >> "${RUNNER_TEMP}/release_notes.md"
-          echo '```' >> "${RUNNER_TEMP}/release_notes.md"
+          {
+            echo '## SHA256 checksums'
+            echo
+            echo '```'
+            cat *.sha256sum
+            echo '```'
+          } >> "${RUNNER_TEMP}/release_notes.md"
 
-          for d in $(ls); do
-            archive_name=$(ls $d)
-            mv ${d}/${archive_name} ./
-            rm -r $d
-            sha256sum ./$archive_name > ./${archive_name}.sha256sum
-            cat ./$archive_name.sha256sum >> "${RUNNER_TEMP}/release_notes.md"
-          done
-
-          echo '```' >> "${RUNNER_TEMP}/release_notes.md"
+      - name: Upload release notes
+        uses: actions/upload-artifact@v4
+        with:
+          name: release_notes_${{ inputs.version }}
+          path: "${RUNNER_TEMP}/release_notes.md"
 
       - name: Remove current release and tag
         if: ${{ github.event_name != 'pull_request' && inputs.overwrite_release }}
@@ -74,5 +112,6 @@ jobs:
             ${{ inputs.overwrite_release && '--prerelease' || '' }} \
             --title "${{ inputs.title }}" \
             --target $GITHUB_SHA \
+            ${{ inputs.draft && '--draft' || '' }} \
             --notes-file "${RUNNER_TEMP}/release_notes.md" \
             ./release_artifacts/clio_server*

.github/workflows/sanitizers.yml

@@ -6,7 +6,23 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - ".github/workflows/sanitizers.yml"
+      - .github/workflows/sanitizers.yml
+
+      - .github/workflows/build_and_test.yml
+      - .github/workflows/build_impl.yml
+      - .github/workflows/test_impl.yml
+
+      - ".github/actions/**"
+      - "!.github/actions/build_docker_image/**"
+      - "!.github/actions/create_issue/**"
+      - .github/scripts/execute-tests-under-sanitizer
+
+      - CMakeLists.txt
+      - conanfile.py
+      - "cmake/**"
+      # We don't run sanitizer on code change, because it takes too long
+      # - "src/**"
+      # - "tests/**"
 
 concurrency:
   # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags

.github/workflows/test_impl.yml

@@ -46,6 +46,10 @@ jobs:
 
     if: inputs.run_unit_tests
 
+    env:
+      # TODO: remove when we have fixed all currently existing issues from sanitizers
+      SANITIZER_IGNORE_ERRORS: ${{ inputs.sanitizer != 'false' && inputs.sanitizer != 'ubsan' }}
+
     steps:
       - name: Clean workdir
         if: ${{ runner.os == 'macOS' }}
@@ -64,15 +68,15 @@ jobs:
         run: chmod +x ./clio_tests
 
       - name: Run clio_tests (regular)
-        if: inputs.sanitizer == 'false'
+        if: env.SANITIZER_IGNORE_ERRORS == 'false'
         run: ./clio_tests
 
-      - name: Run clio_tests (sanitizer)
-        if: inputs.sanitizer != 'false'
+      - name: Run clio_tests (sanitizer errors ignored)
+        if: env.SANITIZER_IGNORE_ERRORS == 'true'
         run: ./.github/scripts/execute-tests-under-sanitizer ./clio_tests
 
       - name: Check for sanitizer report
-        if: inputs.sanitizer != 'false'
+        if: env.SANITIZER_IGNORE_ERRORS == 'true'
         shell: bash
         id: check_report
         run: |
@@ -83,16 +87,15 @@ jobs:
           fi
 
       - name: Upload sanitizer report
-        if: inputs.sanitizer != 'false' && steps.check_report.outputs.found_report == 'true'
+        if: env.SANITIZER_IGNORE_ERRORS == 'true' && steps.check_report.outputs.found_report == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.conan_profile }}_report
           path: .sanitizer-report/*
           include-hidden-files: true
 
-      # TODO: enable when we have fixed all currently existing issues from sanitizers
       - name: Create an issue
-        if: false && inputs.sanitizer != 'false' && steps.check_report.outputs.found_report == 'true'
+        if: false && env.SANITIZER_IGNORE_ERRORS == 'true' && steps.check_report.outputs.found_report == 'true'
         uses: ./.github/actions/create_issue
         env:
           GH_TOKEN: ${{ github.token }}
@@ -114,7 +117,7 @@ jobs:
 
     services:
       scylladb:
-        image: "scylladb/scylla"
+        image: ${{ inputs.container != '' && 'scylladb/scylla' || '' }}
         options: >-
           --health-cmd "cqlsh -e 'describe cluster'"
           --health-interval 10s
@@ -126,13 +129,37 @@ jobs:
         if: ${{ runner.os == 'macOS' }}
         uses: kuznetsss/workspace-cleanup@80b9863b45562c148927c3d53621ef354e5ae7ce # v1.0
 
+      - name: Spin up scylladb
+        if: ${{ runner.os == 'macOS' }}
+        timeout-minutes: 3
+        run: |
+          docker rm --force scylladb || true
+          docker run \
+            --detach \
+            --name scylladb \
+            --health-cmd "cqlsh -e 'describe cluster'" \
+            --health-interval 10s \
+            --health-timeout 5s \
+            --health-retries 5 \
+            --publish 9042:9042 \
+            --memory 16G \
+            scylladb/scylla
+
+          until [ "$(docker inspect -f '{{.State.Health.Status}}' scylladb)" == "healthy" ]; do
+            sleep 5
+          done
+
       - uses: actions/download-artifact@v4
         with:
           name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
 
-      # To be enabled back once docker in mac runner arrives
-      # https://github.com/XRPLF/clio/issues/1400
       - name: Run clio_integration_tests
         run: |
           chmod +x ./clio_integration_tests
-          ./clio_integration_tests --backend_host=scylladb
+          ./clio_integration_tests ${{ runner.os != 'macOS' && '--backend_host=scylladb' || '' }}
+
+      - name: Show docker logs and stop scylladb
+        if: ${{ always() && runner.os == 'macOS' }}
+        run: |
+          docker logs scylladb
+          docker rm --force scylladb || true

.github/workflows/update_docker_ci.yml

@@ -3,19 +3,23 @@ name: Update CI docker image
 on:
   pull_request:
     paths:
+      - .github/workflows/update_docker_ci.yml
+
+      - ".github/actions/build_docker_image/**"
+
       - "docker/ci/**"
       - "docker/compilers/**"
-      - .github/workflows/update_docker_ci.yml
-      - ".github/actions/build_docker_image/**"
+      - "docker/tools/**"
   push:
     branches: [develop]
     paths:
-      # CI image must update when either its Dockerfile changes
-      # or any compilers changed and were pushed by hand
+      - .github/workflows/update_docker_ci.yml
+
+      - ".github/actions/build_docker_image/**"
+
       - "docker/ci/**"
       - "docker/compilers/**"
-      - .github/workflows/update_docker_ci.yml
-      - ".github/actions/build_docker_image/**"
+      - "docker/tools/**"
   workflow_dispatch:
 
 concurrency:
@@ -24,22 +28,117 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_and_push:
-    name: Build and push docker image
+  gcc:
+    name: Build and push GCC docker image
     runs-on: [self-hosted, heavy]
 
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
+        with:
+          files: "docker/compilers/gcc/**"
+
+      - uses: ./.github/actions/build_docker_image
+        # Skipping this build for now, because CI environment is not stable
+        if: false && steps.changed-files.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+        with:
+          images: |
+            ghcr.io/xrplf/clio-gcc
+            rippleci/clio_gcc
+          push_image: ${{ github.event_name != 'pull_request' }}
+          directory: docker/compilers/gcc
+          tags: |
+            type=raw,value=latest
+            type=raw,value=12
+            type=raw,value=12.3.0
+            type=raw,value=${{ github.sha }}
+          platforms: linux/amd64,linux/arm64
+          dockerhub_repo: rippleci/clio_gcc
+          dockerhub_description: GCC compiler for XRPLF/clio.
+
+  clang:
+    name: Build and push Clang docker image
+    runs-on: [self-hosted, heavy]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
+        with:
+          files: "docker/compilers/clang/**"
+
+      - uses: ./.github/actions/build_docker_image
+        if: steps.changed-files.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+        with:
+          images: |
+            ghcr.io/xrplf/clio-clang
+            rippleci/clio_clang
+          push_image: ${{ github.event_name != 'pull_request' }}
+          directory: docker/compilers/clang
+          tags: |
+            type=raw,value=latest
+            type=raw,value=16
+            type=raw,value=${{ github.sha }}
+          platforms: linux/amd64,linux/arm64
+          dockerhub_repo: rippleci/clio_clang
+          dockerhub_description: Clang compiler for XRPLF/clio.
+
+  tools:
+    name: Build and push tools docker image
+    runs-on: [self-hosted, heavy]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
+        with:
+          files: "docker/tools/**"
+
+      - uses: ./.github/actions/build_docker_image
+        if: steps.changed-files.outputs.any_changed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          images: |
+            ghcr.io/xrplf/clio-tools
+          push_image: ${{ github.event_name != 'pull_request' }}
+          directory: docker/tools
+          tags: |
+            type=raw,value=latest
+            type=raw,value=${{ github.sha }}
+          platforms: linux/amd64,linux/arm64
+
+  ci:
+    name: Build and push CI docker image
+    runs-on: [self-hosted, heavy]
+    needs: [gcc, clang, tools]
+
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/build_docker_image
         env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
           DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           images: |
             rippleci/clio_ci
             ghcr.io/xrplf/clio-ci
-          dockerhub_repo: rippleci/clio_ci
           push_image: ${{ github.event_name != 'pull_request' }}
           directory: docker/ci
           tags: |
@@ -47,4 +146,5 @@ jobs:
             type=raw,value=gcc_12_clang_16
             type=raw,value=${{ github.sha }}
           platforms: linux/amd64,linux/arm64
-          description: CI image for XRPLF/clio.
+          dockerhub_repo: rippleci/clio_ci
+          dockerhub_description: CI image for XRPLF/clio.

.github/workflows/upload_coverage_report.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Upload coverage report
         if: ${{ hashFiles('build/coverage_report.xml') != '' }}
-        uses: codecov/codecov-action@ad3126e916f78f00edff4ed0317cf185271ccc2d # v5.4.2
+        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
         with:
           files: build/coverage_report.xml
           fail_ci_if_error: true

.hadolint.yml

@@ -0,0 +1,6 @@
+---
+ignored:
+  - DL3003
+  - DL3007
+  - DL3008
+  - DL3013

.pre-commit-config.yaml

@@ -7,50 +7,72 @@
 # Common tasks
 #
 # - Run on all files:   pre-commit run --all-files
-# - Register git hooks: pre-commit install --install-hooks
+# - Register git hooks: pre-commit install --hook-type pre-commit --hook-type pre-push
 #
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
+exclude: ^docs/doxygen-awesome-theme/
+
 repos:
   # `pre-commit sample-config` default hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # frozen: v5.0.0
     hooks:
       - id: check-added-large-files
       - id: check-executables-have-shebangs
       - id: check-shebang-scripts-are-executable
       - id: end-of-file-fixer
-        exclude: ^docs/doxygen-awesome-theme/
       - id: trailing-whitespace
-        exclude: ^docs/doxygen-awesome-theme/
 
   # Autoformat: YAML, JSON, Markdown, etc.
   - repo: https://github.com/rbubley/mirrors-prettier
-    rev: v3.5.3
+    rev: 787fb9f542b140ba0b2aced38e6a3e68021647a3 # frozen: v3.5.3
     hooks:
       - id: prettier
-        exclude: ^docs/doxygen-awesome-theme/
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.44.0
+    rev: 586c3ea3f51230da42bab657c6a32e9e66c364f0 # frozen: v0.44.0
     hooks:
       - id: markdownlint-fix
         exclude: LICENSE.md
 
-  - repo: https://github.com/crate-ci/typos
-    rev: v1.31.2
+  - repo: https://github.com/hadolint/hadolint
+    rev: c3dc18df7a501f02a560a2cc7ba3c69a85ca01d3 # frozen: v2.13.1-beta
     hooks:
-      - id: typos
+      - id: hadolint-docker
+        # hadolint-docker is a special hook that runs hadolint in a Docker container
+        # Docker is not installed in the environment where pre-commit is run
+        stages: [manual]
+        entry: hadolint/hadolint:v2.12.1-beta hadolint
 
+  - repo: https://github.com/codespell-project/codespell
+    rev: 63c8f8312b7559622c0d82815639671ae42132ac # frozen: v2.4.1
+    hooks:
+      - id: codespell
+        args:
+          [
+            --write-changes,
+            --ignore-words=pre-commit-hooks/codespell_ignore.txt,
+          ]
+
+  # Running fix-local-includes before clang-format
+  # to ensure that the include order is correct.
+  - repo: local
+    hooks:
+      - id: fix-local-includes
+        name: Fix Local Includes
+        entry: pre-commit-hooks/fix-local-includes.sh
+        types: [c++]
+        language: script
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v19.1.7
+    rev: f9a52e87b6cdcb01b0a62b8611d9ba9f2dad0067 # frozen: v19.1.7
     hooks:
       - id: clang-format
         args: [--style=file]
         types: [c++]
 
   - repo: https://github.com/cheshirekow/cmake-format-precommit
-    rev: v0.6.13
+    rev: e2c2116d86a80e72e7146a06e68b7c228afc6319 # frozen: v0.6.13
     hooks:
       - id: cmake-format
         additional_dependencies: [PyYAML]
@@ -75,44 +97,12 @@ repos:
         name: Check Doxygen Documentation
         entry: pre-commit-hooks/check-doxygen-docs.sh
         types: [text]
-        language: system
-        pass_filenames: false
-      - id: fix-local-includes
-        name: Fix Local Includes
-        entry: pre-commit-hooks/fix-local-includes.sh
-        types: [c++]
-        language: system
+        language: script
         pass_filenames: false
       - id: verify-commits
         name: Verify Commits
         entry: pre-commit-hooks/verify-commits.sh
-        types: [text]
-        language: system
-        pass_filenames: false
-
-  - repo: local
-    hooks:
-      - id: lfs-post-checkout
-        name: LFS Post Checkout
-        entry: pre-commit-hooks/lfs/post-checkout
-        types: [text]
-        stages: [post-checkout]
-        language: system
-      - id: lfs-post-commit
-        name: LFS Post Commit
-        entry: pre-commit-hooks/lfs/post-commit
-        types: [text]
-        stages: [post-commit]
-        language: system
-      - id: lfs-post-merge
-        name: LFS Post Merge
-        entry: pre-commit-hooks/lfs/post-merge
-        types: [text]
-        stages: [post-merge]
-        language: system
-      - id: lfs-pre-push
-        name: LFS Pre Push
-        entry: pre-commit-hooks/lfs/pre-push
-        types: [text]
+        always_run: true
         stages: [pre-push]
-        language: system
+        language: script
+        pass_filenames: false

CONTRIBUTING.md

@@ -17,6 +17,16 @@ To contribute, please:
 
 > **Note:** Please read the [Style guide](#style-guide).
 
+### `git lfs` hooks
+
+Install `git lfs` hooks using the following command:
+
+```bash
+git lfs install
+```
+
+> **Note:** You need to install Git LFS hooks before installing `pre-commit` hooks.
+
 ### `pre-commit` hooks
 
 To ensure code quality and style, we use [`pre-commit`](https://pre-commit.com/).
@@ -25,7 +35,7 @@ Run the following command to enable `pre-commit` hooks that help with Clio devel
 
 ```bash
 pip3 install pre-commit
-pre-commit install
+pre-commit install --hook-type pre-commit --hook-type pre-push
 ```
 
 `pre-commit` takes care of running each tool in [`.pre-commit-config.yaml`](https://github.com/XRPLF/clio/blob/develop/.pre-commit-config.yaml) in a separate environment.

_typos.toml

@@ -1,22 +0,0 @@
-[default]
-# This allows to ignore account ids in tests and private keys
-# More info: https://github.com/crate-ci/typos/issues/415
-extend-ignore-re = [
-    "[a-z-A-Z0-9]{33}",
-    "[a-z-A-Z0-9]{34}",
-    "[a-z-A-Z0-9]{64}",
-]
-
-[default.extend-identifiers]
-# (S)tring
-tring = "tring"
-trings = "trings"
-
-ASSERTs = "ASSERTs"
-EXCLUDEs = "EXCLUDEs"
-
-ser = "ser"
-
-[default.extend-words]
-strat = "strat"
-datas = "datas"

benchmarks/util/async/ExecutionContextBenchmarks.cpp

@@ -191,8 +191,9 @@ generateData()
     constexpr auto kTOTAL = 10'000;
     std::vector<uint64_t> data;
     data.reserve(kTOTAL);
+    util::MTRandomGenerator randomGenerator;
     for (auto i = 0; i < kTOTAL; ++i)
-        data.push_back(util::Random::uniform(1, 100'000'000));
+        data.push_back(randomGenerator.uniform(1, 100'000'000));
 
     return data;
 }

cliff.toml

@@ -8,15 +8,24 @@
 [changelog]
 # template for the changelog header
 header = """
-# Changelog\n
-All notable changes to this project will be documented in this file.\n
 """
 # template for the changelog body
 # https://keats.github.io/tera/docs/#introduction
+
 body = """
 {% if version %}\
-    ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
+  Version {{ version | trim_start_matches(pat="v") }} of Clio, an XRP Ledger API server optimized for HTTP and WebSocket API calls, is now available.
 {% else %}\
+  Clio, an XRP Ledger API server optimized for HTTP and WebSocket API calls, is under active development.
+{% endif %}\
+
+<!-- Please, remove one of the 2 following lines -->
+This release adds new features and bug fixes.
+This release adds bug fixes.
+\
+{% if version %}
+    ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
+{% else %}
     ## [unreleased]
 {% endif %}\
 {% for group, commits in commits | filter(attribute="merge_commit", value=false) | group_by(attribute="group") %}
@@ -24,7 +33,7 @@ body = """
     {% for commit in commits %}
         - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
             {% if commit.breaking %}[**breaking**] {% endif %}\
-            {{ commit.message | upper_first }} {% if commit.remote.username %}by @{{ commit.remote.username }}{% endif %}\
+            {{ commit.message | upper_first }}{% if commit.remote.username %} by @{{ commit.remote.username }}{% endif %}\
     {% endfor %}
 {% endfor %}\n
 """

cmake/Settings.cmake

@@ -26,7 +26,7 @@ set(COMPILER_FLAGS
     # TODO: Address these and others in https://github.com/XRPLF/clio/issues/1273
 )
 
-# TODO: reenable when we change CI #884 if (is_gcc AND NOT lint) list(APPEND COMPILER_FLAGS -Wduplicated-branches
+# TODO: re-enable when we change CI #884 if (is_gcc AND NOT lint) list(APPEND COMPILER_FLAGS -Wduplicated-branches
 # -Wduplicated-cond -Wlogical-op -Wuseless-cast ) endif ()
 
 if (is_clang)

conanfile.py

@@ -2,10 +2,10 @@ from conan import ConanFile
 from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
 
 
-class Clio(ConanFile):
+class ClioConan(ConanFile):
     name = 'clio'
     license = 'ISC'
-    author = 'Alex Kremer <akremer@ripple.com>, John Freeman <jfreeman@ripple.com>'
+    author = 'Alex Kremer <akremer@ripple.com>, John Freeman <jfreeman@ripple.com>, Ayaz Salikhov <asalikhov@ripple.com>'
     url = 'https://github.com/xrplf/clio'
     description = 'Clio RPC server'
     settings = 'os', 'compiler', 'build_type', 'arch'
@@ -28,10 +28,10 @@ class Clio(ConanFile):
         'boost/1.83.0',
         'cassandra-cpp-driver/2.17.0',
         'fmt/10.1.1',
-        'protobuf/3.21.9',
+        'protobuf/3.21.12',
         'grpc/1.50.1',
         'openssl/1.1.1v',
-        'xrpl/2.4.0',
+        'xrpl/2.5.0-rc1',
         'zlib/1.3.1',
         'libbacktrace/cci.20210118'
     ]
@@ -100,6 +100,10 @@ class Clio(ConanFile):
         tc.variables['benchmark'] = self.options.benchmark
         tc.variables['snapshot'] = self.options.snapshot
         tc.variables['time_trace'] = self.options.time_trace
+
+        if self.settings.compiler == 'clang' and self.settings.compiler.version == 16:
+            tc.extra_cxxflags = ["-DBOOST_ASIO_DISABLE_CONCEPTS"]
+
         tc.generate()
 
     def build(self):

docker/ci/Dockerfile

@@ -1,34 +1,69 @@
-FROM rippleci/clio_clang:16
-ARG DEBIAN_FRONTEND=noninteractive
-ARG TARGETARCH
+# TODO: change this when we are able to push gcc image to ghcr.io
+FROM rippleci/clio_gcc:12.3.0 AS clio-gcc
+FROM ghcr.io/xrplf/clio-tools:latest AS clio-tools
 
-SHELL ["/bin/bash", "-c"]
+FROM ghcr.io/xrplf/clio-clang:16
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# Using root by default is not very secure but github checkout action doesn't work with any other user
+# https://github.com/actions/checkout/issues/956
+# And Github Actions doc recommends using root
+# https://docs.github.com/en/actions/sharing-automations/creating-actions/dockerfile-support-for-github-actions#user
+
+# hadolint ignore=DL3002
 USER root
 WORKDIR /root
 
-ENV CCACHE_VERSION=4.10.2 \
-    LLVM_TOOLS_VERSION=19 \
-    GH_VERSION=2.40.0 \
-    DOXYGEN_VERSION=1.12.0 \
-    CLANG_BUILD_ANALYZER_VERSION=1.6.0
+ARG LLVM_TOOLS_VERSION=19
 
 # Add repositories
-RUN apt-get -qq update \
-    && apt-get -qq install -y --no-install-recommends --no-install-suggests gnupg wget curl software-properties-common \
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+        curl \
+        gnupg \
+        wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
     && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM_TOOLS_VERSION} main" >> /etc/apt/sources.list \
-    && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+    && wget --progress=dot:giga -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 
 # Install packages
-RUN apt update -qq \
-    && apt install -y --no-install-recommends --no-install-suggests python3 python3-pip git git-lfs make ninja-build flex bison jq graphviz \
-    clang-tidy-${LLVM_TOOLS_VERSION} clang-tools-${LLVM_TOOLS_VERSION} \
-    && pip3 install -q --upgrade --no-cache-dir pip && pip3 install -q --no-cache-dir conan==1.62 gcovr cmake==3.31.6 pre-commit \
-    && apt-get clean && apt remove -y software-properties-common
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+        bison \
+        clang-tidy-${LLVM_TOOLS_VERSION} \
+        clang-tools-${LLVM_TOOLS_VERSION} \
+        flex \
+        git \
+        git-lfs \
+        graphviz \
+        jq \
+        make \
+        ninja-build \
+        python3 \
+        python3-pip \
+        zip \
+    && pip3 install -q --upgrade --no-cache-dir pip \
+    && pip3 install -q --no-cache-dir \
+        cmake==3.31.6 \
+        conan==2.17.0 \
+        gcovr \
+        pre-commit \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install gcc-12 and make ldconfig aware of the new libstdc++ location (for gcc)
 # Note: Clang is using libc++ instead
-COPY --from=rippleci/clio_gcc:12.3.0 /gcc12.deb /
-RUN apt update && apt-get install -y binutils libc6-dev \
+COPY --from=clio-gcc /gcc12.deb /
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+        binutils \
+        libc6-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
     && dpkg -i /gcc12.deb \
     && rm -rf /gcc12.deb \
     && ldconfig
@@ -42,72 +77,29 @@ RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
     && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
     && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100
 
-WORKDIR /tmp
-
-# Install ccache from source
-RUN wget "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz" \
-    && tar xf "ccache-${CCACHE_VERSION}.tar.gz" \
-    && cd "ccache-${CCACHE_VERSION}" \
-    && mkdir build && cd build \
-    && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install \
-    && rm -rf /tmp/* /var/tmp/*
-
-# Install doxygen from source
-RUN wget "https://github.com/doxygen/doxygen/releases/download/Release_${DOXYGEN_VERSION//./_}/doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
-    && tar xf "doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
-    && cd "doxygen-${DOXYGEN_VERSION}" \
-    && mkdir build && cd build \
-    && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install \
-    && rm -rf /tmp/* /var/tmp/*
-
-# Install ClangBuildAnalyzer
-RUN wget "https://github.com/aras-p/ClangBuildAnalyzer/releases/download/v${CLANG_BUILD_ANALYZER_VERSION}/ClangBuildAnalyzer-linux" \
-    && chmod +x ClangBuildAnalyzer-linux \
-    && mv ClangBuildAnalyzer-linux /usr/bin/ClangBuildAnalyzer \
-    && rm -rf /tmp/* /var/tmp/*
-
-# Install gh
-RUN wget "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz" \
-    && tar xf gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
-    && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh \
-    && rm -rf /tmp/* /var/tmp/*
+COPY --from=clio-tools \
+    /usr/local/bin/ccache \
+    /usr/local/bin/doxygen \
+    /usr/local/bin/ClangBuildAnalyzer \
+    /usr/local/bin/git-cliff \
+    /usr/local/bin/gh \
+    /usr/local/bin/
 
 WORKDIR /root
-# Using root by default is not very secure but github checkout action doesn't work with any other user
-# https://github.com/actions/checkout/issues/956
-# And Github Actions doc recommends using root
-# https://docs.github.com/en/actions/sharing-automations/creating-actions/dockerfile-support-for-github-actions#user
 
 # Setup conan
-RUN conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+RUN conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev
 
-# Note: intentionally leaving cppstd=20
-RUN conan profile new gcc --detect \
-    && conan profile update settings.compiler=gcc gcc \
-    && conan profile update settings.compiler.version=12 gcc \
-    && conan profile update settings.compiler.cppstd=20 gcc \
-    && conan profile update settings.compiler.libcxx=libstdc++11 gcc \
-    && conan profile update env.CC=/usr/bin/gcc-12 gcc \
-    && conan profile update env.CXX=/usr/bin/g++-12 gcc \
-    && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/gcc-12\", \"cpp\": \"/usr/bin/g++-12\"}" gcc
+WORKDIR /root/.conan2/profiles
 
-RUN conan profile new clang --detect \
-    && conan profile update settings.compiler=clang clang \
-    && conan profile update settings.compiler.version=16 clang \
-    && conan profile update settings.compiler.cppstd=20 clang \
-    && conan profile update settings.compiler.libcxx=libc++ clang \
-    && conan profile update env.CC=/usr/bin/clang-16 clang \
-    && conan profile update env.CXX=/usr/bin/clang++-16 clang \
-    && conan profile update env.CXXFLAGS="-DBOOST_ASIO_DISABLE_CONCEPTS" clang \
-    && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/clang-16\", \"cpp\": \"/usr/bin/clang++-16\"}" clang
+COPY conan/clang.profile ./clang
+COPY conan/sanitizer_template.profile ./clang.asan
+COPY conan/sanitizer_template.profile ./clang.tsan
+COPY conan/sanitizer_template.profile ./clang.ubsan
 
-RUN echo "include(gcc)" >> .conan/profiles/default
+COPY conan/gcc.profile ./gcc
+COPY conan/sanitizer_template.profile ./gcc.asan
+COPY conan/sanitizer_template.profile ./gcc.tsan
+COPY conan/sanitizer_template.profile ./gcc.ubsan
 
-COPY conan/gcc.asan /root/.conan/profiles
-COPY conan/gcc.tsan /root/.conan/profiles
-COPY conan/gcc.ubsan /root/.conan/profiles
-COPY conan/clang.asan /root/.conan/profiles
-COPY conan/clang.tsan /root/.conan/profiles
-COPY conan/clang.ubsan /root/.conan/profiles
+WORKDIR /root

docker/ci/README.md

@@ -5,13 +5,17 @@ It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but c
 
 The image is based on Ubuntu 20.04 and contains:
 
+- ccache 4.11.3
 - clang 16.0.6
-- gcc 12.3
+- ClangBuildAnalyzer 1.6.0
+- conan 2.17.0
 - doxygen 1.12
-- gh 2.40
-- ccache 4.10.2
-- conan 1.62
+- gcc 12.3.0
+- gh 2.74
+- git-cliff 2.9.1
 - and some other useful tools
 
-Conan is set up to build Clio without any additional steps. There are two preset conan profiles: `clang` and `gcc` to use corresponding compiler. By default conan is setup to use `gcc`.
-Sanitizer builds for `ASAN`, `TSAN` and `UBSAN` are enabled via conan profiles for each of the supported compilers. These can be selected using the following pattern (all lowercase): `[compiler].[sanitizer]` (e.g. `--profile gcc.tsan`).
+Conan is set up to build Clio without any additional steps.
+There are two preset conan profiles: `clang` and `gcc` to use corresponding compiler.
+`ASan`, `TSan` and `UBSan` sanitizer builds are enabled via conan profiles for each of the supported compilers.
+These can be selected using the following pattern (all lowercase): `[compiler].[sanitizer]` (e.g. `--profile:all gcc.tsan`).

docker/ci/conan/clang.asan

@@ -1,9 +0,0 @@
-include(clang)
-
-[options]
-boost:extra_b2_flags="cxxflags=\"-fsanitize=address\" linkflags=\"-fsanitize=address\""
-boost:without_stacktrace=True
-[env]
-CFLAGS="-fsanitize=address"
-CXXFLAGS="-fsanitize=address"
-LDFLAGS="-fsanitize=address"

docker/ci/conan/clang.profile

@@ -0,0 +1,11 @@
+[settings]
+arch=x86_64
+build_type=Release
+compiler=clang
+compiler.cppstd=20
+compiler.libcxx=libc++
+compiler.version=16
+os=Linux
+
+[conf]
+tools.build:compiler_executables={'c': '/usr/bin/clang-16', 'cpp': '/usr/bin/clang++-16'}

docker/ci/conan/clang.tsan

@@ -1,9 +0,0 @@
-include(clang)
-
-[options]
-boost:extra_b2_flags="cxxflags=\"-fsanitize=thread\" linkflags=\"-fsanitize=thread\""
-boost:without_stacktrace=True
-[env]
-CFLAGS="-fsanitize=thread"
-CXXFLAGS="-fsanitize=thread"
-LDFLAGS="-fsanitize=thread"

docker/ci/conan/clang.ubsan

@@ -1,9 +0,0 @@
-include(clang)
-
-[options]
-boost:extra_b2_flags="cxxflags=\"-fsanitize=undefined\" linkflags=\"-fsanitize=undefined\""
-boost:without_stacktrace=True
-[env]
-CFLAGS="-fsanitize=undefined"
-CXXFLAGS="-fsanitize=undefined"
-LDFLAGS="-fsanitize=undefined"

docker/ci/conan/gcc.asan

@@ -1,9 +0,0 @@
-include(gcc)
-
-[options]
-boost:extra_b2_flags="cxxflags=\"-fsanitize=address\" linkflags=\"-fsanitize=address\""
-boost:without_stacktrace=True
-[env]
-CFLAGS="-fsanitize=address"
-CXXFLAGS="-fsanitize=address"
-LDFLAGS="-fsanitize=address"

docker/ci/conan/gcc.profile

@@ -0,0 +1,11 @@
+[settings]
+arch=x86_64
+build_type=Release
+compiler=gcc
+compiler.cppstd=20
+compiler.libcxx=libstdc++11
+compiler.version=12
+os=Linux
+
+[conf]
+tools.build:compiler_executables={'c': '/usr/bin/gcc-12', 'cpp': '/usr/bin/g++-12'}

docker/ci/conan/gcc.tsan

@@ -1,9 +0,0 @@
-include(gcc)
-
-[options]
-boost:extra_b2_flags="cxxflags=\"-fsanitize=thread\" linkflags=\"-fsanitize=thread\""
-boost:without_stacktrace=True
-[env]
-CFLAGS="-fsanitize=thread"
-CXXFLAGS="-fsanitize=thread"
-LDFLAGS="-fsanitize=thread"

docker/ci/conan/gcc.ubsan

@@ -1,9 +0,0 @@
-include(gcc)
-
-[options]
-boost:extra_b2_flags="cxxflags=\"-fsanitize=undefined\" linkflags=\"-fsanitize=undefined\""
-boost:without_stacktrace=True
-[env]
-CFLAGS="-fsanitize=undefined"
-CXXFLAGS="-fsanitize=undefined"
-LDFLAGS="-fsanitize=undefined"

docker/ci/conan/sanitizer_template.profile

@@ -0,0 +1,15 @@
+{% set compiler, sani = profile_name.split('.') %}
+
+{% set sanitizer_opt_map = {'asan': 'address', 'tsan': 'thread', 'ubsan': 'undefined'} %}
+{% set sanitizer = sanitizer_opt_map[sani] %}
+
+include({{ compiler }})
+
+[options]
+boost/*:extra_b2_flags="cxxflags=\"-fsanitize={{ sanitizer }}\" linkflags=\"-fsanitize={{ sanitizer }}\""
+boost/*:without_stacktrace=True
+
+[conf]
+tools.build:cflags+=["-fsanitize={{ sanitizer }}"]
+tools.build:cxxflags+=["-fsanitize={{ sanitizer }}"]
+tools.build:exelinkflags+=["-fsanitize={{ sanitizer }}"]

docker/clio/Dockerfile

@@ -2,14 +2,17 @@ FROM ubuntu:22.04
 
 COPY ./clio_server /opt/clio/bin/clio_server
 
-RUN ln -s /opt/clio/bin/clio_server /usr/local/bin/clio_server && \
-    mkdir -p /opt/clio/etc/ && \
-    mkdir -p /opt/clio/log/ && \
-    groupadd -g 10001 clio && \
-    useradd -u 10000 -g 10001 -s /bin/bash clio && \
-    chown clio:clio /opt/clio/log && \
-    apt update && \
-    apt install -y libatomic1
+RUN ln -s /opt/clio/bin/clio_server /usr/local/bin/clio_server \
+    && mkdir -p /opt/clio/etc/ \
+    && mkdir -p /opt/clio/log/ \
+    && groupadd -g 10001 clio \
+    && useradd -u 10000 -g 10001 -s /bin/bash clio \
+    && chown clio:clio /opt/clio/log \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+        libatomic1 \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 USER clio
 ENTRYPOINT ["/opt/clio/bin/clio_server"]

docker/compilers/clang-16/Dockerfile

@@ -1,19 +0,0 @@
-FROM ubuntu:focal
-ARG DEBIAN_FRONTEND=noninteractive
-ARG TARGETARCH
-
-SHELL ["/bin/bash", "-c"]
-USER root
-WORKDIR /root
-
-ENV CLANG_VERSION=16
-
-RUN apt update -qq \
-    && apt install -qq -y --no-install-recommends --no-install-suggests \
-    wget software-properties-common gnupg
-
-RUN wget https://apt.llvm.org/llvm.sh \
-    && chmod +x llvm.sh \
-    && ./llvm.sh ${CLANG_VERSION} \
-    && rm -rf llvm.sh \
-    && apt-get install -y libc++-16-dev libc++abi-16-dev

docker/compilers/clang/Dockerfile

@@ -0,0 +1,30 @@
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+SHELL ["/bin/bash", "-c"]
+
+# hadolint ignore=DL3002
+USER root
+WORKDIR /root
+
+ARG CLANG_VERSION=16
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+        wget \
+        software-properties-common \
+        gnupg \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN wget --progress=dot:giga https://apt.llvm.org/llvm.sh \
+    && chmod +x llvm.sh \
+    && ./llvm.sh ${CLANG_VERSION} \
+    && rm -rf llvm.sh \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+        libc++-${CLANG_VERSION}-dev \
+        libc++abi-${CLANG_VERSION}-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*

docker/compilers/clang/README.md

@@ -0,0 +1,3 @@
+# Clang compiler
+
+This image contains clang compiler to build <https://github.com/XRPLF/clio>.

docker/compilers/gcc/Dockerfile

@@ -1,49 +1,60 @@
-FROM ubuntu:focal as build
+FROM ubuntu:20.04 AS build
 
 ARG DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
 ARG UBUNTU_VERSION=20.04
 ARG GCC_VERSION=12.3.0
-ARG BUILD_VERSION=1
+ARG BUILD_VERSION=2
 
-RUN apt update && apt install -y wget build-essential file flex libz-dev libzstd-dev
-RUN wget https://gcc.gnu.org/pub/gcc/releases/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz \
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+        build-essential \
+        file \
+        flex \
+        libz-dev \
+        libzstd-dev \
+        software-properties-common \
+        wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN wget --progress=dot:giga https://gcc.gnu.org/pub/gcc/releases/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz \
     && tar xf gcc-$GCC_VERSION.tar.gz \
     && cd /gcc-$GCC_VERSION && ./contrib/download_prerequisites
 
 RUN mkdir /${TARGETARCH}-gcc-12
 WORKDIR /${TARGETARCH}-gcc-12
 RUN /gcc-$GCC_VERSION/configure \
-    --with-pkgversion="clio-build-$BUILD_VERSION https://github.com/XRPLF/clio" \
-    --enable-languages=c,c++ \
-    --prefix=/usr \
-    --with-gcc-major-version-only \
-    --program-suffix=-12 \
-    --enable-shared \
-    --enable-linker-build-id \
-    --libexecdir=/usr/lib \
-    --without-included-gettext \
-    --enable-threads=posix \
-    --libdir=/usr/lib \
-    --disable-nls \
-    --enable-clocale=gnu \
-    --enable-libstdcxx-backtrace=yes \
-    --enable-libstdcxx-debug \
-    --enable-libstdcxx-time=yes \
-    --with-default-libstdcxx-abi=new \
-    --enable-gnu-unique-object \
-    --disable-vtable-verify \
-    --enable-plugin \
-    --enable-default-pie \
-    --with-system-zlib \
-    --enable-libphobos-checking=release \
-    --with-target-system-zlib=auto \
-    --disable-werror \
-    --enable-cet \
-    --disable-multilib \
-    --without-cuda-driver \
-    --enable-checking=release \
-    && make -j`nproc` \
+        --with-pkgversion="clio-build-$BUILD_VERSION https://github.com/XRPLF/clio" \
+        --enable-languages=c,c++ \
+        --prefix=/usr \
+        --with-gcc-major-version-only \
+        --program-suffix=-12 \
+        --enable-shared \
+        --enable-linker-build-id \
+        --libexecdir=/usr/lib \
+        --without-included-gettext \
+        --enable-threads=posix \
+        --libdir=/usr/lib \
+        --disable-nls \
+        --enable-clocale=gnu \
+        --enable-libstdcxx-backtrace=yes \
+        --enable-libstdcxx-debug \
+        --enable-libstdcxx-time=yes \
+        --with-default-libstdcxx-abi=new \
+        --enable-gnu-unique-object \
+        --disable-vtable-verify \
+        --enable-plugin \
+        --enable-default-pie \
+        --with-system-zlib \
+        --enable-libphobos-checking=release \
+        --with-target-system-zlib=auto \
+        --disable-werror \
+        --enable-cet \
+        --disable-multilib \
+        --without-cuda-driver \
+        --enable-checking=release \
+    && make -j "$(nproc)" \
     && make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION \
     && mkdir -p /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64 \
     && mv /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/lib64/libstdc++.so.6.0.30-gdb.py /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64/libstdc++.so.6.0.30-gdb.py
@@ -58,11 +69,16 @@ RUN mkdir /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/DEBIAN \
     && dpkg-deb --build --root-owner-group /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION /gcc12.deb
 
 # Create final image
-FROM ubuntu:focal as gcc
+FROM ubuntu:20.04
 COPY --from=build /gcc12.deb /
 
 # Make gcc-12 available but also leave gcc12.deb for others to copy if needed
-RUN apt update && apt-get install -y binutils libc6-dev \
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+        binutils \
+        libc6-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* \
     && dpkg -i /gcc12.deb
 
 RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \

docker/compilers/gcc/README.md

@@ -0,0 +1,3 @@
+# gcc compiler
+
+This image contains gcc compiler to build <https://github.com/XRPLF/clio>.

docker/tools/Dockerfile

@@ -0,0 +1,64 @@
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG TARGETARCH
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+        bison \
+        build-essential \
+        cmake \
+        flex \
+        ninja-build \
+        software-properties-common \
+        wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /tmp
+
+ARG CCACHE_VERSION=4.11.3
+RUN wget --progress=dot:giga "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz" \
+    && tar xf "ccache-${CCACHE_VERSION}.tar.gz" \
+    && cd "ccache-${CCACHE_VERSION}" \
+    && mkdir build \
+    && cd build \
+    && cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=False .. \
+    && cmake --build . --target install \
+    && rm -rf /tmp/* /var/tmp/*
+
+ARG DOXYGEN_VERSION=1.12.0
+RUN wget --progress=dot:giga "https://github.com/doxygen/doxygen/releases/download/Release_${DOXYGEN_VERSION//./_}/doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
+    && tar xf "doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
+    && cd "doxygen-${DOXYGEN_VERSION}" \
+    && mkdir build \
+    && cd build \
+    && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
+    && cmake --build . --target install \
+    && rm -rf /tmp/* /var/tmp/*
+
+ARG CLANG_BUILD_ANALYZER_VERSION=1.6.0
+RUN wget --progress=dot:giga "https://github.com/aras-p/ClangBuildAnalyzer/archive/refs/tags/v${CLANG_BUILD_ANALYZER_VERSION}.tar.gz" \
+    && tar xf "v${CLANG_BUILD_ANALYZER_VERSION}.tar.gz" \
+    && cd "ClangBuildAnalyzer-${CLANG_BUILD_ANALYZER_VERSION}" \
+    && mkdir build \
+    && cd build \
+    && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
+    && cmake --build . --target install \
+    && rm -rf /tmp/* /var/tmp/*
+
+ARG GIT_CLIFF_VERSION=2.9.1
+RUN wget --progress=dot:giga "https://github.com/orhun/git-cliff/releases/download/v${GIT_CLIFF_VERSION}/git-cliff-${GIT_CLIFF_VERSION}-x86_64-unknown-linux-musl.tar.gz" \
+    && tar xf git-cliff-${GIT_CLIFF_VERSION}-x86_64-unknown-linux-musl.tar.gz \
+    && mv git-cliff-${GIT_CLIFF_VERSION}/git-cliff /usr/local/bin/git-cliff \
+    && rm -rf /tmp/* /var/tmp/*
+
+ARG GH_VERSION=2.74.0
+RUN wget --progress=dot:giga "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz" \
+    && tar xf gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
+    && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/local/bin/gh \
+    && rm -rf /tmp/* /var/tmp/*
+
+WORKDIR /root

docs/Doxyfile

@@ -22,6 +22,7 @@ WARNINGS               = ${LINT}
 WARN_NO_PARAMDOC       = ${LINT}
 WARN_IF_INCOMPLETE_DOC = ${LINT}
 WARN_IF_UNDOCUMENTED   = ${LINT}
+WARN_AS_ERROR          = ${WARN_AS_ERROR}
 
 GENERATE_LATEX         = NO
 GENERATE_HTML          = YES

docs/build-clio.md

@@ -1,12 +1,13 @@
 # How to build Clio
 
-Clio is built with [CMake](https://cmake.org/) and uses [Conan](https://conan.io/) for managing dependencies. It is written in C++20 and therefore requires a modern compiler.
+`Clio` is built with [CMake](https://cmake.org/) and uses [Conan](https://conan.io/) for managing dependencies.
+`Clio` is written in C++23 and therefore requires a modern compiler.
 
 ## Minimum Requirements
 
 - [Python 3.7](https://www.python.org/downloads/)
-- [Conan 1.55](https://conan.io/downloads.html)
-- [CMake 3.20](https://cmake.org/download/)
+- [Conan 1.55, <2.0](https://conan.io/downloads.html)
+- [CMake 3.20, <4.0](https://cmake.org/download/)
 - [**Optional**] [GCovr](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html): needed for code coverage generation
 - [**Optional**] [CCache](https://ccache.dev/): speeds up compilation if you are going to compile Clio often
 
@@ -18,41 +19,41 @@ Clio is built with [CMake](https://cmake.org/) and uses [Conan](https://conan.io
 
 ### Conan Configuration
 
-Clio does not require anything other than `compiler.cppstd=20` in your (`~/.conan/profiles/default`) Conan profile.
+Clio requires `compiler.cppstd=20` in your Conan profile (`~/.conan/profiles/default`).
 
 > [!NOTE]
 > Although Clio is built using C++23, it's required to set `compiler.cppstd=20` for the time being as some of Clio's dependencies are not yet capable of building under C++23.
 
-**Mac example**:
+**Mac apple-clang 16 example**:
 
 ```text
 [settings]
-os=Macos
-os_build=Macos
 arch=armv8
-arch_build=armv8
-compiler=apple-clang
-compiler.version=15
-compiler.libcxx=libc++
 build_type=Release
+compiler=apple-clang
 compiler.cppstd=20
+compiler.libcxx=libc++
+compiler.version=16
+os=Macos
+
 [conf]
-tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]
+tools.build:cxxflags+=["-Wno-missing-template-arg-list-after-template-kw"]
 ```
 
-**Linux example**:
+**Linux gcc-12 example**:
 
 ```text
 [settings]
-os=Linux
-os_build=Linux
 arch=x86_64
-arch_build=x86_64
-compiler=gcc
-compiler.version=12
-compiler.libcxx=libstdc++11
 build_type=Release
+compiler=gcc
 compiler.cppstd=20
+compiler.libcxx=libstdc++11
+compiler.version=12
+os=Linux
+
+[conf]
+tools.build:compiler_executables={'c': '/usr/bin/gcc-12', 'cpp': '/usr/bin/g++-12'}
 ```
 
 #### Artifactory
@@ -60,7 +61,7 @@ compiler.cppstd=20
 Make sure artifactory is setup with Conan.
 
 ```sh
-conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev
 ```
 
 Now you should be able to download the prebuilt `xrpl` package on some platforms.
@@ -80,18 +81,21 @@ Navigate to Clio's root directory and run:
 
 ```sh
 mkdir build && cd build
-conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False
+conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True'
+# You can also add -GNinja to use Ninja build system instead of Make
 cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
 cmake --build . --parallel 8 # or without the number if you feel extra adventurous
 ```
 
 > [!TIP]
-> You can omit the `-o tests=True` if you don't want to build `clio_tests`.
+> You can omit the `-o '&:tests=True'` if you don't want to build `clio_tests`.
 
 If successful, `conan install` will find the required packages and `cmake` will do the rest. You should see `clio_server` and `clio_tests` in the `build` directory (the current directory).
 
 > [!TIP]
-> To generate a Code Coverage report, include `-o coverage=True` in the `conan install` command above, along with `-o tests=True` to enable tests. After running the `cmake` commands, execute `make clio_tests-ccov`. The coverage report will be found at `clio_tests-llvm-cov/index.html`.
+> To generate a Code Coverage report, include `-o '&:coverage=True'` in the `conan install` command above, along with `-o '&:tests=True'` to enable tests.
+> After running the `cmake` commands, execute `make clio_tests-ccov`.
+> The coverage report will be found at `clio_tests-llvm-cov/index.html`.
 
 <!-- markdownlint-disable-line MD028 -->
 
@@ -100,15 +104,15 @@ If successful, `conan install` will find the required packages and `cmake` will
 
 ### Generating API docs for Clio
 
-The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen on your system.
+The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen 1.12.0 on your system.
 
 To generate the API docs:
 
-1. First, include `-o docs=True` in the conan install command. For example:
+1. First, include `-o '&:docs=True'` in the conan install command. For example:
 
    ```sh
    mkdir build && cd build
-   conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False -o docs=True
+   conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True' -o '&:docs=True'
    ```
 
 2. Once that has completed successfully, run the `cmake` command and add the `--target docs` option:
@@ -132,7 +136,7 @@ It is also possible to build Clio using [Docker](https://www.docker.com/) if you
 docker run -it ghcr.io/xrplf/clio-ci:latest
 git clone https://github.com/XRPLF/clio
 mkdir build && cd build
-conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False
+conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True'
 cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
 cmake --build . --parallel 8 # or without the number if you feel extra adventurous
 ```
@@ -148,7 +152,8 @@ If you wish to develop against a `rippled` instance running in standalone mode t
 
 Sometimes, during development, you need to build against a custom version of `libxrpl`. (For example, you may be developing compatibility for a proposed amendment that is not yet merged to the main `rippled` codebase.) To build Clio with compatibility for a custom fork or branch of `rippled`, follow these steps:
 
-1. First, pull/clone the appropriate `rippled` fork and switch to the branch you want to build. For example, the following example uses an in-development build with [XLS-33d Multi-Purpose Tokens](https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens):
+1. First, pull/clone the appropriate `rippled` fork and switch to the branch you want to build.
+   The following example uses an in-development build with [XLS-33d Multi-Purpose Tokens](https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens):
 
    ```sh
    git clone https://github.com/shawnxie999/rippled/
@@ -168,14 +173,15 @@ Sometimes, during development, you need to build against a custom version of `li
 
    ```py
    # ... (excerpt from conanfile.py)
-       requires = [
-       'boost/1.82.0',
+   requires = [
+       'boost/1.83.0',
        'cassandra-cpp-driver/2.17.0',
        'fmt/10.1.1',
        'protobuf/3.21.9',
        'grpc/1.50.1',
-       'openssl/1.1.1u',
+       'openssl/1.1.1v',
        'xrpl/2.3.0-b1@my/feature', # Update this line
+       'zlib/1.3.1',
        'libbacktrace/cci.20210118'
    ]
    ```
@@ -188,14 +194,15 @@ Sometimes, during development, you need to build against a custom version of `li
 
 The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 19.0.
 
-Clang-tidy can be run by Cmake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command:
+Clang-tidy can be run by CMake when building the project.
+To achieve this, you just need to provide the option `-o '&:lint=True'` for the `conan install` command:
 
 ```sh
-conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=True
+conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True' -o '&:lint=True'
 ```
 
-By default Cmake will try to find `clang-tidy` automatically in your system.
-To force Cmake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example:
+By default CMake will try to find `clang-tidy` automatically in your system.
+To force CMake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example:
 
 ```sh
 export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@19/bin/clang-tidy

docs/examples/infrastructure/README.md

@@ -4,17 +4,17 @@
 > This is only an example of Grafana dashboard for Clio. It was created for demonstration purposes only and may contain errors.
 > Clio team would not recommend to relate on data from this dashboard or use it for monitoring your Clio instances.
 
-This directory contains an example of docker based infrastructure to collect and visualise metrics from clio.
+This directory contains an example of docker based infrastructure to collect and visualize metrics from clio.
 
 The structure of the directory:
 
 - `compose.yaml`
-  Docker-compose file with Prometheus and Grafana set up.
+  Docker Compose file with Prometheus and Grafana set up.
 - `prometheus.yaml`
   Defines metrics collection from Clio and Prometheus itself.
-  Demonstrates how to setup Clio target and Clio's admin authorisation in Prometheus.
+  Demonstrates how to setup Clio target and Clio's admin authorization in Prometheus.
 - `grafana/clio_dashboard.json`
-  Json file containing preconfigured dashboard in Grafana format.
+  Json file containing pre-configured dashboard in Grafana format.
 - `grafana/dashboard_local.yaml`
   Grafana configuration file defining the directory to search for dashboards json files.
 - `grafana/datasources.yaml`
@@ -22,9 +22,9 @@ The structure of the directory:
 
 ## How to try
 
-1. Make sure you have `docker` and `docker-compose` installed.
-2. Run `docker-compose up -d` from this directory. It will start docker containers with Prometheus and Grafana.
+1. Make sure you have Docker (with `Docker Compose`) installed.
+2. Run `docker compose up -d` from this directory. It will start docker containers with Prometheus and Grafana.
 3. Open [http://localhost:3000/dashboards](http://localhost:3000/dashboards). Grafana login `admin`, password `grafana`.
-   There will be preconfigured Clio dashboard.
+   There will be pre-configured Clio dashboard.
 
 If Clio is not running yet launch Clio to see metrics. Some of the metrics may appear only after requests to Clio.

docs/run-clio.md

@@ -4,10 +4,10 @@
 
 - Access to a Cassandra cluster or ScyllaDB cluster. Can be local or remote.
 
-  > [!IMPORTANT]
-  > There are some key considerations when using **ScyllaDB**. By default, Scylla reserves all free RAM on a machine for itself. If you are running `rippled` or other services on the same machine, restrict its memory usage using the `--memory` argument.
-  >
-  > See [ScyllaDB in a Shared Environment](https://docs.scylladb.com/getting-started/scylla-in-a-shared-environment/) to learn more.
+> [!IMPORTANT]
+> There are some key considerations when using **ScyllaDB**. By default, Scylla reserves all free RAM on a machine for itself. If you are running `rippled` or other services on the same machine, restrict its memory usage using the `--memory` argument.
+>
+> See [ScyllaDB in a Shared Environment](https://docs.scylladb.com/getting-started/scylla-in-a-shared-environment/) to learn more.
 
 - Access to one or more `rippled` nodes. Can be local or remote.
 
@@ -92,4 +92,4 @@ To completely disable Prometheus metrics add `"prometheus": { "enabled": false }
 
 It is important to know that Clio responds to Prometheus request only if they are admin requests. If you are using the admin password feature, the same password should be provided in the Authorization header of Prometheus requests.
 
-You can find an example docker-compose file, with Prometheus and Grafana configs, in [examples/infrastructure](../docs/examples/infrastructure/).
+You can find an example Docker Compose file, with Prometheus and Grafana configs, in [examples/infrastructure](../docs/examples/infrastructure/).

pre-commit-hooks/check-doxygen-docs.sh

@@ -61,6 +61,7 @@ pushd ${DOCDIR} > /dev/null 2>&1
 cat ${ROOT}/docs/Doxyfile | \
 sed \
     -e "s/\${LINT}/YES/" \
+    -e "s/\${WARN_AS_ERROR}/NO/" \
     -e "s!\${SOURCE}!${ROOT}!" \
     -e "s/\${USE_DOT}/NO/" \
     -e "s/\${EXCLUDES}/impl/" \

pre-commit-hooks/codespell_ignore.txt

@@ -0,0 +1,9 @@
+ser
+onWs
+datas
+AtLeast
+AtMost
+compiletime
+tring
+trings
+strat

pre-commit-hooks/fix-local-includes.sh

@@ -2,31 +2,43 @@
 
 # Note: This script is intended to be run from the root of the repository.
 #
-# This script checks will fix local includes in the C++ code.
+# This script will fix local includes in the C++ code for a given file.
+# Usage: ./pre-commit-hooks/fix-local-includes.sh <file1> <file2> ...
 
-# paths to fix include statements
-sources="src tests"
-
-echo "+ Fixing local includes..."
-
-function grep_code {
-    grep -l "${1}" ${sources} -r --include \*.hpp --include \*.cpp
-}
+files="$@"
+echo "+ Fixing includes in $files..."
 
 GNU_SED=$(sed --version 2>&1 | grep -q 'GNU' && echo true || echo false)
 
 if [[ "$GNU_SED" == "false" ]]; then # macOS sed
-    # make all includes to be <...> style
-    grep_code '#include ".*"' | xargs sed -i '' -E 's|#include "(.*)"|#include <\1>|g'
-
-    # make local includes to be "..." style
     main_src_dirs=$(find ./src -maxdepth 1 -type d -exec basename {} \; | tr '\n' '|' | sed 's/|$//' | sed 's/|/\\|/g')
-    grep_code "#include <\($main_src_dirs\)/.*>" | xargs sed -i '' -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g"
 else
-    # make all includes to be <...> style
-    grep_code '#include ".*"' | xargs sed -i -E 's|#include "(.*)"|#include <\1>|g'
-
-    # make local includes to be "..." style
-    main_src_dirs=$(find ./src -maxdepth 1 -type d  -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g')
-    grep_code "#include <\($main_src_dirs\)/.*>" | xargs sed -i -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g"
+    main_src_dirs=$(find ./src -maxdepth 1 -type d -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g')
 fi
+
+fix_includes() {
+    file_path="$1"
+
+    file_path_all_global="${file_path}.tmp.global"
+    file_path_fixed="${file_path}.tmp.fixed"
+
+    # Make all includes to be <...> style
+    sed -E 's|#include "(.*)"|#include <\1>|g' "$file_path" > "$file_path_all_global"
+
+    # Make local includes to be "..." style
+    sed -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g" "$file_path_all_global" > "$file_path_fixed"
+    rm "$file_path_all_global"
+
+    # Check if the temporary file is different from the original file
+    if ! cmp -s "$file_path" "$file_path_fixed"; then
+        # Replace the original file with the temporary file
+        mv "$file_path_fixed" "$file_path"
+    else
+        # Remove the temporary file if it's the same as the original
+        rm "$file_path_fixed"
+    fi
+}
+
+for file in $files; do
+    fix_includes "$file"
+done

pre-commit-hooks/lfs/post-checkout

@@ -1,3 +0,0 @@
-#!/bin/sh
-command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-checkout' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
-git lfs post-checkout "$@"

pre-commit-hooks/lfs/post-commit

@@ -1,3 +0,0 @@
-#!/bin/sh
-command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-commit' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
-git lfs post-commit "$@"

pre-commit-hooks/lfs/post-merge

@@ -1,3 +0,0 @@
-#!/bin/sh
-command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-merge' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
-git lfs post-merge "$@"

pre-commit-hooks/lfs/pre-push

@@ -1,3 +0,0 @@
-#!/bin/sh
-command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'pre-push' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
-git lfs pre-push "$@"

pre-commit-hooks/verify-commits.sh

@@ -39,16 +39,14 @@ verify_tag_signed() {
     fi
 }
 
-while read local_ref local_oid remote_ref remote_oid; do
-    # Check some things if we're pushing a branch called "release/"
-    if echo "$remote_ref" | grep ^refs\/heads\/release\/ &> /dev/null ; then
-        version=$(git tag --points-at HEAD)
-        echo "Looks like you're trying to push a $version release..."
-        echo "Making sure you've signed and tagged it."
-        if verify_commit_signed && verify_tag && verify_tag_signed ; then
-            : # Ok, I guess you can push
-        else
-            exit 1
-        fi
+# Check some things if we're pushing a branch called "release/"
+if echo "$PRE_COMMIT_REMOTE_BRANCH" | grep ^refs\/heads\/release\/ &> /dev/null ; then
+    version=$(git tag --points-at HEAD)
+    echo "Looks like you're trying to push a $version release..."
+    echo "Making sure you've signed and tagged it."
+    if verify_commit_signed && verify_tag && verify_tag_signed ; then
+        : # Ok, I guess you can push
+    else
+        exit 1
     fi
-done
+fi

src/app/CliArgs.cpp

@@ -21,7 +21,7 @@
 
 #include "migration/MigrationApplication.hpp"
 #include "util/build/Build.hpp"
-#include "util/newconfig/ConfigDescription.hpp"
+#include "util/config/ConfigDescription.hpp"
 
 #include <boost/program_options/options_description.hpp>
 #include <boost/program_options/parsers.hpp>

src/app/ClioApplication.cpp

@@ -36,15 +36,18 @@
 #include "rpc/RPCEngine.hpp"
 #include "rpc/WorkQueue.hpp"
 #include "rpc/common/impl/HandlerProvider.hpp"
+#include "util/Random.hpp"
+#include "util/async/context/BasicExecutionContext.hpp"
 #include "util/build/Build.hpp"
+#include "util/config/ConfigDefinition.hpp"
 #include "util/log/Logger.hpp"
-#include "util/newconfig/ConfigDefinition.hpp"
 #include "util/prometheus/Prometheus.hpp"
 #include "web/AdminVerificationStrategy.hpp"
 #include "web/RPCServerHandler.hpp"
 #include "web/Server.hpp"
 #include "web/dosguard/DOSGuard.hpp"
 #include "web/dosguard/IntervalSweepHandler.hpp"
+#include "web/dosguard/Weights.hpp"
 #include "web/dosguard/WhitelistHandler.hpp"
 #include "web/ng/RPCServerHandler.hpp"
 #include "web/ng/Server.hpp"
@@ -102,9 +105,14 @@ ClioApplication::run(bool const useNgWebServer)
     // This is not the only io context in the application.
     boost::asio::io_context ioc{threads};
 
+    // Similarly we need a context to run ETLng on
+    // In the future we can remove the raw ioc and use ctx instead
+    util::async::CoroExecutionContext ctx{threads};
+
     // Rate limiter, to prevent abuse
     auto whitelistHandler = web::dosguard::WhitelistHandler{config_};
-    auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler};
+    auto const dosguardWeights = web::dosguard::Weights::make(config_);
+    auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler, dosguardWeights};
     auto sweepHandler = web::dosguard::IntervalSweepHandler{config_, ioc, dosGuard};
     auto cache = data::LedgerCache{};
 
@@ -137,14 +145,19 @@ ClioApplication::run(bool const useNgWebServer)
     // The server uses the balancer to forward RPCs to a rippled node.
     // The balancer itself publishes to streams (transactions_proposed and accounts_proposed)
     auto balancer = [&] -> std::shared_ptr<etlng::LoadBalancerInterface> {
-        if (config_.get<bool>("__ng_etl"))
-            return etlng::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
+        if (config_.get<bool>("__ng_etl")) {
+            return etlng::LoadBalancer::makeLoadBalancer(
+                config_, ioc, backend, subscriptions, std::make_unique<util::MTRandomGenerator>(), ledgers
+            );
+        }
 
-        return etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
+        return etl::LoadBalancer::makeLoadBalancer(
+            config_, ioc, backend, subscriptions, std::make_unique<util::MTRandomGenerator>(), ledgers
+        );
     }();
 
     // ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes
-    auto etl = etl::ETLService::makeETLService(config_, ioc, backend, subscriptions, balancer, ledgers);
+    auto etl = etl::ETLService::makeETLService(config_, ioc, ctx, backend, subscriptions, balancer, ledgers);
 
     auto workQueue = rpc::WorkQueue::makeWorkQueue(config_);
     auto counters = rpc::Counters::makeCounters(workQueue);
@@ -158,7 +171,7 @@ ClioApplication::run(bool const useNgWebServer)
         RPCEngineType::makeRPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
 
     if (useNgWebServer or config_.get<bool>("server.__ng_web_server")) {
-        web::ng::RPCServerHandler<RPCEngineType> handler{config_, backend, rpcEngine, etl};
+        web::ng::RPCServerHandler<RPCEngineType> handler{config_, backend, rpcEngine, etl, dosGuard};
 
         auto expectedAdminVerifier = web::makeAdminVerificationStrategy(config_);
         if (not expectedAdminVerifier.has_value()) {
@@ -176,7 +189,7 @@ ClioApplication::run(bool const useNgWebServer)
 
         httpServer->onGet("/metrics", MetricsHandler{adminVerifier});
         httpServer->onGet("/health", HealthCheckHandler{});
-        auto requestHandler = RequestHandler{adminVerifier, handler, dosGuard};
+        auto requestHandler = RequestHandler{adminVerifier, handler};
         httpServer->onPost("/", requestHandler);
         httpServer->onWs(std::move(requestHandler));
 
@@ -199,7 +212,7 @@ ClioApplication::run(bool const useNgWebServer)
     }
 
     // Init the web server
-    auto handler = std::make_shared<web::RPCServerHandler<RPCEngineType>>(config_, backend, rpcEngine, etl);
+    auto handler = std::make_shared<web::RPCServerHandler<RPCEngineType>>(config_, backend, rpcEngine, etl, dosGuard);
 
     auto const httpServer = web::makeHttpServer(config_, ioc, dosGuard, handler);
 

src/app/ClioApplication.hpp

@@ -21,7 +21,7 @@
 
 #include "app/Stopper.hpp"
 #include "util/SignalsHandler.hpp"
-#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/config/ConfigDefinition.hpp"
 
 namespace app {
 

src/app/VerifyConfig.hpp

@@ -19,8 +19,8 @@
 
 #pragma once
 
-#include "util/newconfig/ConfigDefinition.hpp"
-#include "util/newconfig/ConfigFileJson.hpp"
+#include "util/config/ConfigDefinition.hpp"
+#include "util/config/ConfigFileJson.hpp"
 
 #include <cstdlib>
 #include <iostream>

src/app/WebHandlers.hpp

@@ -147,7 +147,6 @@ class RequestHandler {
     util::Logger webServerLog_{"WebServer"};
     std::shared_ptr<web::AdminVerificationStrategy> adminVerifier_;
     std::reference_wrapper<RpcHandlerType> rpcHandler_;
-    std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
 
 public:
     /**
@@ -155,14 +154,9 @@ public:
      *
      * @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access.
      * @param rpcHandler The RPC handler to use for handling the request.
-     * @param dosguard The DOSGuardInterface to use for checking the connection.
      */
-    RequestHandler(
-        std::shared_ptr<web::AdminVerificationStrategy> adminVerifier,
-        RpcHandlerType& rpcHandler,
-        web::dosguard::DOSGuardInterface& dosguard
-    )
-        : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler), dosguard_(dosguard)
+    RequestHandler(std::shared_ptr<web::AdminVerificationStrategy> adminVerifier, RpcHandlerType& rpcHandler)
+        : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler)
     {
     }
 
@@ -183,21 +177,6 @@ public:
         boost::asio::yield_context yield
     )
     {
-        if (not dosguard_.get().request(connectionMetadata.ip())) {
-            auto error = rpc::makeError(rpc::RippledError::rpcSLOW_DOWN);
-
-            if (not request.isHttp()) {
-                try {
-                    auto requestJson = boost::json::parse(request.message());
-                    if (requestJson.is_object() && requestJson.as_object().contains("id"))
-                        error["id"] = requestJson.as_object().at("id");
-                    error["request"] = request.message();
-                } catch (std::exception const&) {
-                    error["request"] = request.message();
-                }
-            }
-            return web::ng::Response{boost::beast::http::status::service_unavailable, error, request};
-        }
         LOG(webServerLog_.info()) << connectionMetadata.tag()
                                   << "Received request from ip = " << connectionMetadata.ip()
                                   << " - posting to WorkQueue";
@@ -207,20 +186,7 @@ public:
         });
 
         try {
-            auto response = rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield);
-
-            if (not dosguard_.get().add(connectionMetadata.ip(), response.message().size())) {
-                auto jsonResponse = boost::json::parse(response.message()).as_object();
-                jsonResponse["warning"] = "load";
-                if (jsonResponse.contains("warnings") && jsonResponse["warnings"].is_array()) {
-                    jsonResponse["warnings"].as_array().push_back(rpc::makeWarning(rpc::WarnRpcRateLimit));
-                } else {
-                    jsonResponse["warnings"] = boost::json::array{rpc::makeWarning(rpc::WarnRpcRateLimit)};
-                }
-                response.setMessage(jsonResponse);
-            }
-
-            return response;
+            return rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield);
         } catch (std::exception const&) {
             return web::ng::Response{
                 boost::beast::http::status::internal_server_error,

src/cluster/ClusterCommunicationService.cpp

@@ -36,6 +36,7 @@
 #include <chrono>
 #include <ctime>
 #include <memory>
+#include <string>
 #include <utility>
 #include <vector>
 
@@ -111,9 +112,12 @@ ClusterCommunicationService::selfData() const
     return result;
 }
 
-std::vector<ClioNode>
+std::expected<std::vector<ClioNode>, std::string>
 ClusterCommunicationService::clusterData() const
 {
+    if (not isHealthy_) {
+        return std::unexpected{"Service is not healthy"};
+    }
     std::vector<ClioNode> result;
     boost::asio::spawn(strand_, [this, &result](boost::asio::yield_context) {
         result = otherNodesData_;
@@ -127,7 +131,13 @@ ClusterCommunicationService::doRead(boost::asio::yield_context yield)
 {
     otherNodesData_.clear();
 
-    auto const expectedResult = backend_->fetchClioNodesData(yield);
+    BackendInterface::ClioNodesDataFetchResult expectedResult;
+    try {
+        expectedResult = backend_->fetchClioNodesData(yield);
+    } catch (...) {
+        expectedResult = std::unexpected{"Failed to fecth Clio nodes data"};
+    }
+
     if (!expectedResult.has_value()) {
         LOG(log_.error()) << "Failed to fetch nodes data";
         isHealthy_ = false;

src/cluster/ClusterCommunicationService.hpp

@@ -34,6 +34,7 @@
 
 #include <chrono>
 #include <memory>
+#include <string>
 #include <vector>
 
 namespace cluster {
@@ -125,9 +126,9 @@ public:
     /**
      * @brief Get the data of all nodes in the cluster (including self).
      *
-     * @return The data of all nodes in the cluster.
+     * @return The data of all nodes in the cluster or error if the service is not healthy.
      */
-    std::vector<ClioNode>
+    std::expected<std::vector<ClioNode>, std::string>
     clusterData() const override;
 
 private:

src/cluster/ClusterCommunicationServiceInterface.hpp

@@ -21,6 +21,8 @@
 
 #include "cluster/ClioNode.hpp"
 
+#include <expected>
+#include <string>
 #include <vector>
 
 namespace cluster {
@@ -43,9 +45,9 @@ public:
     /**
      * @brief Get the data of all nodes in the cluster (including self).
      *
-     * @return The data of all nodes in the cluster.
+     * @return The data of all nodes in the cluster or error if the service is not healthy.
      */
-    [[nodiscard]] virtual std::vector<ClioNode>
+    [[nodiscard]] virtual std::expected<std::vector<ClioNode>, std::string>
     clusterData() const = 0;
 };
 

src/data/AmendmentCenter.hpp

@@ -137,6 +137,14 @@ struct Amendments {
     REGISTER(fixInvalidTxFlags);
     REGISTER(fixFrozenLPTokenTransfer);
     REGISTER(DeepFreeze);
+    REGISTER(PermissionDelegation);
+    REGISTER(fixPayChanCancelAfter);
+    REGISTER(Batch);
+    REGISTER(PermissionedDEX);
+    REGISTER(SingleAssetVault);
+    REGISTER(TokenEscrow);
+    REGISTER(fixAMMv1_3);
+    REGISTER(fixEnforceNFTokenTrustlineV2);
 
     // Obsolete but supported by libxrpl
     REGISTER(CryptoConditionsSuite);

src/data/BackendFactory.hpp

@@ -23,8 +23,8 @@
 #include "data/CassandraBackend.hpp"
 #include "data/LedgerCacheInterface.hpp"
 #include "data/cassandra/SettingsProvider.hpp"
+#include "util/config/ConfigDefinition.hpp"
 #include "util/log/Logger.hpp"
-#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>

src/data/BackendInterface.hpp

@@ -569,13 +569,17 @@ public:
     virtual std::optional<std::string>
     fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const = 0;
 
+    /** @brief Return type for fetchClioNodesData() method */
+    using ClioNodesDataFetchResult =
+        std::expected<std::vector<std::pair<boost::uuids::uuid, std::string>>, std::string>;
+
     /**
      * @brief Fetches the data of all nodes in the cluster.
      *
      * @param yield The coroutine context
      *@return The data of all nodes in the cluster.
      */
-    [[nodiscard]] virtual std::expected<std::vector<std::pair<boost::uuids::uuid, std::string>>, std::string>
+    [[nodiscard]] virtual ClioNodesDataFetchResult
     fetchClioNodesData(boost::asio::yield_context yield) const = 0;
 
     /**

src/data/CMakeLists.txt

@@ -5,6 +5,7 @@ target_sources(
           BackendCounters.cpp
           BackendInterface.cpp
           LedgerCache.cpp
+          LedgerHeaderCache.cpp
           cassandra/impl/Future.cpp
           cassandra/impl/Cluster.cpp
           cassandra/impl/Batch.cpp

src/data/CassandraBackend.hpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "data/DBHelpers.hpp"
 #include "data/LedgerCacheInterface.hpp"
+#include "data/LedgerHeaderCache.hpp"
 #include "data/Types.hpp"
 #include "data/cassandra/Concepts.hpp"
 #include "data/cassandra/Handle.hpp"
@@ -62,6 +63,8 @@
 #include <utility>
 #include <vector>
 
+class CacheBackendCassandraTest;
+
 namespace data::cassandra {
 
 /**
@@ -71,21 +74,27 @@ namespace data::cassandra {
  *
  * @tparam SettingsProviderType The settings provider type to use
  * @tparam ExecutionStrategyType The execution strategy type to use
+ * @tparam FetchLedgerCacheType The ledger header cache type to use
  */
-template <SomeSettingsProvider SettingsProviderType, SomeExecutionStrategy ExecutionStrategyType>
+template <
+    SomeSettingsProvider SettingsProviderType,
+    SomeExecutionStrategy ExecutionStrategyType,
+    typename FetchLedgerCacheType = FetchLedgerCache>
 class BasicCassandraBackend : public BackendInterface {
     util::Logger log_{"Backend"};
 
     SettingsProviderType settingsProvider_;
     Schema<SettingsProviderType> schema_;
-
     std::atomic_uint32_t ledgerSequence_ = 0u;
+    friend class ::CacheBackendCassandraTest;
 
 protected:
     Handle handle_;
 
     // have to be mutable because BackendInterface constness :(
     mutable ExecutionStrategyType executor_;
+    // TODO: move to interface level
+    mutable FetchLedgerCacheType ledgerCache_{};
 
 public:
     /**
@@ -129,7 +138,6 @@ public:
             LOG(log_.error()) << error;
             throw std::runtime_error(error);
         }
-
         LOG(log_.info()) << "Created (revamped) CassandraBackend";
     }
 
@@ -263,11 +271,16 @@ public:
     std::optional<ripple::LedgerHeader>
     fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const override
     {
+        if (auto const lock = ledgerCache_.get(); lock.has_value() && lock->seq == sequence)
+            return lock->ledger;
+
         auto const res = executor_.read(yield, schema_->selectLedgerBySeq, sequence);
         if (res) {
             if (auto const& result = res.value(); result) {
                 if (auto const maybeValue = result.template get<std::vector<unsigned char>>(); maybeValue) {
-                    return util::deserializeHeader(ripple::makeSlice(*maybeValue));
+                    auto const header = util::deserializeHeader(ripple::makeSlice(*maybeValue));
+                    ledgerCache_.put(FetchLedgerCache::CacheEntry{header, sequence});
+                    return header;
                 }
 
                 LOG(log_.error()) << "Could not fetch ledger by sequence - no rows";

src/data/LedgerHeaderCache.cpp

@@ -0,0 +1,46 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/LedgerHeaderCache.hpp"
+
+#include "util/Mutex.hpp"
+
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+
+namespace data {
+
+FetchLedgerCache::FetchLedgerCache() = default;
+
+void
+FetchLedgerCache::put(CacheEntry const& cacheEntry)
+{
+    auto lock = mutex_.lock<std::unique_lock>();
+    *lock = cacheEntry;
+}
+
+std::optional<FetchLedgerCache::CacheEntry>
+FetchLedgerCache::get() const
+{
+    auto const lock = mutex_.lock<std::shared_lock>();
+    return lock.get();
+}
+
+}  // namespace data

src/data/LedgerHeaderCache.hpp

@@ -0,0 +1,85 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/Mutex.hpp"
+
+#include <xrpl/protocol/LedgerHeader.h>
+
+#include <cstdint>
+#include <optional>
+#include <shared_mutex>
+
+namespace data {
+
+/**
+ * @brief A simple cache holding one `ripple::LedgerHeader` to reduce DB lookups.
+ *
+ * Used internally by backend implementations. When a ledger header is
+ * fetched via `FetchLedgerBySeq` (often triggered by RPC commands),
+ * the result can be stored here. Subsequent requests for the same ledger
+ * sequence can proceed to retrieve the header from this cache, avoiding unnecessary
+ * database reads and improving performance.
+ */
+class FetchLedgerCache {
+public:
+    FetchLedgerCache();
+
+    /**
+     * @brief Struct to store ledger header cache entry and the sequence it belongs to
+     */
+    struct CacheEntry {
+        ripple::LedgerHeader ledger;
+        uint32_t seq{};
+
+        /**
+         * @brief Comparing CacheEntry. Used in testing for EXPECT_CALL
+         *
+         * @param other The other cacheEntry to compare
+         * @return true if two CacheEntry is the same, false otherwise
+         */
+        bool
+        operator==(CacheEntry const& other) const
+        {
+            return ledger.hash == other.ledger.hash && seq == other.seq;
+        }
+    };
+
+    /**
+     * @brief Put CacheEntry into thread-safe container
+     *
+     * @param cacheEntry The Cache to store into thread-safe container.
+     */
+    void
+    put(CacheEntry const& cacheEntry);
+
+    /**
+     * @brief Read CacheEntry from thread-safe container.
+     *
+     * @return Optional CacheEntry, depending on if it exists in thread-safe container or not.
+     */
+    std::optional<CacheEntry>
+    get() const;
+
+private:
+    mutable util::Mutex<std::optional<CacheEntry>, std::shared_mutex> mutex_;
+};
+
+}  // namespace data

src/data/cassandra/SettingsProvider.cpp

@@ -22,7 +22,7 @@
 #include "data/cassandra/Types.hpp"
 #include "data/cassandra/impl/Cluster.hpp"
 #include "util/Constants.hpp"
-#include "util/newconfig/ObjectView.hpp"
+#include "util/config/ObjectView.hpp"
 
 #include <cerrno>
 #include <chrono>

src/data/cassandra/SettingsProvider.hpp

@@ -21,7 +21,7 @@
 
 #include "data/cassandra/Types.hpp"
 #include "data/cassandra/impl/Cluster.hpp"
-#include "util/newconfig/ObjectView.hpp"
+#include "util/config/ObjectView.hpp"
 
 #include <cstdint>
 #include <optional>

src/etl/CacheLoader.hpp

@@ -26,6 +26,7 @@
 #include "etl/impl/CursorFromAccountProvider.hpp"
 #include "etl/impl/CursorFromDiffProvider.hpp"
 #include "etl/impl/CursorFromFixDiffNumProvider.hpp"
+#include "etlng/CacheLoaderInterface.hpp"
 #include "util/Assert.hpp"
 #include "util/async/context/BasicExecutionContext.hpp"
 #include "util/log/Logger.hpp"
@@ -33,6 +34,7 @@
 #include <cstdint>
 #include <functional>
 #include <memory>
+#include <utility>
 
 namespace etl {
 
@@ -46,7 +48,7 @@ namespace etl {
  * @tparam ExecutionContextType The type of the execution context to use
  */
 template <typename ExecutionContextType = util::async::CoroExecutionContext>
-class CacheLoader {
+class CacheLoader : public etlng::CacheLoaderInterface {
     using CacheLoaderType = impl::CacheLoaderImpl<data::LedgerCacheInterface>;
 
     util::Logger log_{"ETL"};
@@ -67,10 +69,13 @@ public:
      */
     CacheLoader(
         util::config::ClioConfigDefinition const& config,
-        std::shared_ptr<BackendInterface> const& backend,
+        std::shared_ptr<BackendInterface> backend,
         data::LedgerCacheInterface& cache
     )
-        : backend_{backend}, cache_{cache}, settings_{makeCacheLoaderSettings(config)}, ctx_{settings_.numThreads}
+        : backend_{std::move(backend)}
+        , cache_{cache}
+        , settings_{makeCacheLoaderSettings(config)}
+        , ctx_{settings_.numThreads}
     {
     }
 
@@ -83,7 +88,7 @@ public:
      * @param seq The sequence number to load cache for
      */
     void
-    load(uint32_t const seq)
+    load(uint32_t const seq) override
     {
         ASSERT(not cache_.get().isFull(), "Cache must not be full. seq = {}", seq);
 
@@ -129,7 +134,7 @@ public:
      * @brief Requests the loader to stop asap
      */
     void
-    stop() noexcept
+    stop() noexcept override
     {
         if (loader_ != nullptr)
             loader_->stop();
@@ -139,7 +144,7 @@ public:
      * @brief Waits for the loader to finish background work
      */
     void
-    wait() noexcept
+    wait() noexcept override
     {
         if (loader_ != nullptr)
             loader_->wait();

src/etl/CacheLoaderSettings.cpp

@@ -19,7 +19,7 @@
 
 #include "etl/CacheLoaderSettings.hpp"
 
-#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/config/ConfigDefinition.hpp"
 
 #include <boost/algorithm/string/predicate.hpp>
 

src/etl/CacheLoaderSettings.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/config/ConfigDefinition.hpp"
 
 #include <cstddef>
 

src/etl/ETLService.cpp

@@ -20,14 +20,36 @@
 #include "etl/ETLService.hpp"
 
 #include "data/BackendInterface.hpp"
+#include "etl/CacheLoader.hpp"
 #include "etl/CorruptionDetector.hpp"
+#include "etl/ETLState.hpp"
+#include "etl/LoadBalancer.hpp"
 #include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etl/SystemState.hpp"
+#include "etl/impl/AmendmentBlockHandler.hpp"
+#include "etl/impl/ExtractionDataPipe.hpp"
+#include "etl/impl/Extractor.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "etl/impl/LedgerLoader.hpp"
+#include "etl/impl/LedgerPublisher.hpp"
+#include "etl/impl/Transformer.hpp"
+#include "etlng/ETLService.hpp"
+#include "etlng/ETLServiceInterface.hpp"
+#include "etlng/LoadBalancer.hpp"
 #include "etlng/LoadBalancerInterface.hpp"
+#include "etlng/impl/LedgerPublisher.hpp"
+#include "etlng/impl/TaskManagerProvider.hpp"
+#include "etlng/impl/ext/Cache.hpp"
+#include "etlng/impl/ext/Core.hpp"
+#include "etlng/impl/ext/MPT.hpp"
+#include "etlng/impl/ext/NFT.hpp"
+#include "etlng/impl/ext/Successor.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "util/Assert.hpp"
 #include "util/Constants.hpp"
+#include "util/async/AnyExecutionContext.hpp"
+#include "util/config/ConfigDefinition.hpp"
 #include "util/log/Logger.hpp"
-#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <xrpl/beast/core/CurrentThreadName.h>
@@ -45,6 +67,76 @@
 
 namespace etl {
 
+std::shared_ptr<etlng::ETLServiceInterface>
+ETLService::makeETLService(
+    util::config::ClioConfigDefinition const& config,
+    boost::asio::io_context& ioc,
+    util::async::AnyExecutionContext ctx,
+    std::shared_ptr<BackendInterface> backend,
+    std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+    std::shared_ptr<etlng::LoadBalancerInterface> balancer,
+    std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
+)
+{
+    std::shared_ptr<etlng::ETLServiceInterface> ret;
+
+    if (config.get<bool>("__ng_etl")) {
+        ASSERT(
+            std::dynamic_pointer_cast<etlng::LoadBalancer>(balancer), "LoadBalancer type must be etlng::LoadBalancer"
+        );
+
+        auto state = std::make_shared<etl::SystemState>();
+
+        auto fetcher = std::make_shared<etl::impl::LedgerFetcher>(backend, balancer);
+        auto extractor = std::make_shared<etlng::impl::Extractor>(fetcher);
+        auto publisher = std::make_shared<etlng::impl::LedgerPublisher>(ioc, backend, subscriptions, *state);
+        auto cacheLoader = std::make_shared<etl::CacheLoader<>>(config, backend, backend->cache());
+        auto cacheUpdater = std::make_shared<etlng::impl::CacheUpdater>(backend->cache());
+        auto amendmentBlockHandler = std::make_shared<etlng::impl::AmendmentBlockHandler>(ctx, *state);
+
+        auto loader = std::make_shared<etlng::impl::Loader>(
+            backend,
+            etlng::impl::makeRegistry(
+                *state,
+                etlng::impl::CacheExt{cacheUpdater},
+                etlng::impl::CoreExt{backend},
+                etlng::impl::SuccessorExt{backend, backend->cache()},
+                etlng::impl::NFTExt{backend},
+                etlng::impl::MPTExt{backend}
+            ),
+            amendmentBlockHandler
+        );
+
+        auto taskManagerProvider = std::make_shared<etlng::impl::TaskManagerProvider>(*ledgers, extractor, loader);
+
+        ret = std::make_shared<etlng::ETLService>(
+            ctx,
+            config,
+            backend,
+            balancer,
+            ledgers,
+            publisher,
+            cacheLoader,
+            cacheUpdater,
+            extractor,
+            loader,  // loader itself
+            loader,  // initial load observer
+            taskManagerProvider,
+            state
+        );
+    } else {
+        ASSERT(std::dynamic_pointer_cast<etl::LoadBalancer>(balancer), "LoadBalancer type must be etl::LoadBalancer");
+        ret = std::make_shared<etl::ETLService>(config, ioc, backend, subscriptions, balancer, ledgers);
+    }
+
+    // inject networkID into subscriptions, as transaction feed require it to inject CTID in response
+    if (auto const state = ret->getETLState(); state)
+        subscriptions->setNetworkID(state->networkID);
+
+    ret->run();
+    return ret;
+}
+
 // Database must be populated when this starts
 std::optional<uint32_t>
 ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)

src/etl/ETLService.hpp

@@ -22,7 +22,6 @@
 #include "data/BackendInterface.hpp"
 #include "etl/CacheLoader.hpp"
 #include "etl/ETLState.hpp"
-#include "etl/LoadBalancer.hpp"
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/SystemState.hpp"
 #include "etl/impl/AmendmentBlockHandler.hpp"
@@ -32,12 +31,12 @@
 #include "etl/impl/LedgerLoader.hpp"
 #include "etl/impl/LedgerPublisher.hpp"
 #include "etl/impl/Transformer.hpp"
-#include "etlng/ETLService.hpp"
 #include "etlng/ETLServiceInterface.hpp"
-#include "etlng/LoadBalancer.hpp"
 #include "etlng/LoadBalancerInterface.hpp"
+#include "etlng/impl/LedgerPublisher.hpp"
+#include "etlng/impl/TaskManagerProvider.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
-#include "util/Assert.hpp"
+#include "util/async/AnyExecutionContext.hpp"
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
@@ -150,6 +149,7 @@ public:
      *
      * @param config The configuration to use
      * @param ioc io context to run on
+     * @param ctx Execution context for asynchronous operations
      * @param backend BackendInterface implementation
      * @param subscriptions Subscription manager
      * @param balancer Load balancer to use
@@ -160,34 +160,12 @@ public:
     makeETLService(
         util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
+        util::async::AnyExecutionContext ctx,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
         std::shared_ptr<etlng::LoadBalancerInterface> balancer,
         std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
-    )
-    {
-        std::shared_ptr<etlng::ETLServiceInterface> ret;
-
-        if (config.get<bool>("__ng_etl")) {
-            ASSERT(
-                std::dynamic_pointer_cast<etlng::LoadBalancer>(balancer),
-                "LoadBalancer type must be etlng::LoadBalancer"
-            );
-            ret = std::make_shared<etlng::ETLService>(config, backend, subscriptions, balancer, ledgers);
-        } else {
-            ASSERT(
-                std::dynamic_pointer_cast<etl::LoadBalancer>(balancer), "LoadBalancer type must be etl::LoadBalancer"
-            );
-            ret = std::make_shared<etl::ETLService>(config, ioc, backend, subscriptions, balancer, ledgers);
-        }
-
-        // inject networkID into subscriptions, as transaction feed require it to inject CTID in response
-        if (auto const state = ret->getETLState(); state)
-            subscriptions->setNetworkID(state->networkID);
-
-        ret->run();
-        return ret;
-    }
+    );
 
     /**
      * @brief Stops components and joins worker thread.

src/etl/LoadBalancer.cpp

@@ -28,12 +28,15 @@
 #include "rpc/Errors.hpp"
 #include "util/Assert.hpp"
 #include "util/CoroutineGroup.hpp"
+#include "util/Profiler.hpp"
 #include "util/Random.hpp"
 #include "util/ResponseExpirationCache.hpp"
+#include "util/config/ArrayView.hpp"
+#include "util/config/ConfigDefinition.hpp"
+#include "util/config/ObjectView.hpp"
 #include "util/log/Logger.hpp"
-#include "util/newconfig/ArrayView.hpp"
-#include "util/newconfig/ConfigDefinition.hpp"
-#include "util/newconfig/ObjectView.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
@@ -58,6 +61,7 @@
 #include <vector>
 
 using namespace util::config;
+using util::prometheus::Labels;
 
 namespace etl {
 
@@ -67,12 +71,19 @@ LoadBalancer::makeLoadBalancer(
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+    std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
     std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
     SourceFactory sourceFactory
 )
 {
     return std::make_shared<LoadBalancer>(
-        config, ioc, std::move(backend), std::move(subscriptions), std::move(validatedLedgers), std::move(sourceFactory)
+        config,
+        ioc,
+        std::move(backend),
+        std::move(subscriptions),
+        std::move(randomGenerator),
+        std::move(validatedLedgers),
+        std::move(sourceFactory)
     );
 }
 
@@ -81,9 +92,38 @@ LoadBalancer::LoadBalancer(
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+    std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
     std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
     SourceFactory sourceFactory
 )
+    : randomGenerator_(std::move(randomGenerator))
+    , forwardingCounters_{
+          .successDuration = PrometheusService::counterInt(
+              "forwarding_duration_milliseconds_counter",
+              Labels({util::prometheus::Label{"status", "success"}}),
+              "The duration of processing successful forwarded requests"
+          ),
+          .failDuration = PrometheusService::counterInt(
+              "forwarding_duration_milliseconds_counter",
+              Labels({util::prometheus::Label{"status", "fail"}}),
+              "The duration of processing failed forwarded requests"
+          ),
+          .retries = PrometheusService::counterInt(
+              "forwarding_retries_counter",
+              Labels(),
+              "The number of retries before a forwarded request was successful. Initial attempt excluded"
+          ),
+          .cacheHit = PrometheusService::counterInt(
+              "forwarding_cache_hit_counter",
+              Labels(),
+              "The number of requests that we served from the cache"
+          ),
+          .cacheMiss = PrometheusService::counterInt(
+              "forwarding_cache_miss_counter",
+              Labels(),
+              "The number of requests that were not served from the cache"
+          )
+      }
 {
     auto const forwardingCacheTimeout = config.get<float>("forwarding.cache_timeout");
     if (forwardingCacheTimeout > 0.f) {
@@ -170,11 +210,6 @@ LoadBalancer::LoadBalancer(
     }
 }
 
-LoadBalancer::~LoadBalancer()
-{
-    sources_.clear();
-}
-
 std::vector<std::string>
 LoadBalancer::loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter)
 {
@@ -242,9 +277,11 @@ LoadBalancer::forwardToRippled(
     auto const cmd = boost::json::value_to<std::string>(request.at("command"));
 
     if (forwardingCache_ and forwardingCache_->shouldCache(cmd)) {
+        bool servedFromCache = true;
         auto updater =
-            [this, &request, &clientIp, isAdmin](boost::asio::yield_context yield
+            [this, &request, &clientIp, &servedFromCache, isAdmin](boost::asio::yield_context yield
             ) -> std::expected<util::ResponseExpirationCache::EntryData, util::ResponseExpirationCache::Error> {
+            servedFromCache = false;
             auto result = forwardToRippledImpl(request, clientIp, isAdmin, yield);
             if (result.has_value()) {
                 return util::ResponseExpirationCache::EntryData{
@@ -262,6 +299,9 @@ LoadBalancer::forwardToRippled(
             std::move(updater),
             [](util::ResponseExpirationCache::EntryData const& entry) { return not entry.response.contains("error"); }
         );
+        if (servedFromCache) {
+            ++forwardingCounters_.cacheHit.get();
+        }
         if (result.has_value()) {
             return std::move(result).value();
         }
@@ -288,7 +328,7 @@ void
 LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter)
 {
     ASSERT(not sources_.empty(), "ETL sources must be configured to execute functions.");
-    size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
+    size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
 
     size_t numAttempts = 0;
 
@@ -369,8 +409,10 @@ LoadBalancer::forwardToRippledImpl(
     boost::asio::yield_context yield
 )
 {
+    ++forwardingCounters_.cacheMiss.get();
+
     ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
-    std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
+    std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
 
     auto numAttempts = 0u;
 
@@ -379,11 +421,16 @@ LoadBalancer::forwardToRippledImpl(
     std::optional<boost::json::object> response;
     rpc::ClioError error = rpc::ClioError::EtlConnectionError;
     while (numAttempts < sources_.size()) {
-        auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield);
+        auto [res, duration] =
+            util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
+
         if (res) {
+            forwardingCounters_.successDuration.get() += duration;
             response = std::move(res).value();
             break;
         }
+        forwardingCounters_.failDuration.get() += duration;
+        ++forwardingCounters_.retries.get();
         error = std::max(error, res.error());  // Choose the best result between all sources
 
         sourceIdx = (sourceIdx + 1) % sources_.size();

src/etl/LoadBalancer.hpp

@@ -29,9 +29,11 @@
 #include "rpc/Errors.hpp"
 #include "util/Assert.hpp"
 #include "util/Mutex.hpp"
+#include "util/Random.hpp"
 #include "util/ResponseExpirationCache.hpp"
+#include "util/config/ConfigDefinition.hpp"
 #include "util/log/Logger.hpp"
-#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/prometheus/Counter.hpp"
 
 #include <boost/asio.hpp>
 #include <boost/asio/io_context.hpp>
@@ -87,12 +89,22 @@ private:
     std::optional<util::ResponseExpirationCache> forwardingCache_;
     std::optional<std::string> forwardingXUserValue_;
 
+    std::unique_ptr<util::RandomGeneratorInterface> randomGenerator_;
+
     std::vector<SourcePtr> sources_;
     std::optional<ETLState> etlState_;
     std::uint32_t downloadRanges_ =
         kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
 
-    // Using mutext instead of atomic_bool because choosing a new source to
+    struct ForwardingCounters {
+        std::reference_wrapper<util::prometheus::CounterInt> successDuration;
+        std::reference_wrapper<util::prometheus::CounterInt> failDuration;
+        std::reference_wrapper<util::prometheus::CounterInt> retries;
+        std::reference_wrapper<util::prometheus::CounterInt> cacheHit;
+        std::reference_wrapper<util::prometheus::CounterInt> cacheMiss;
+    } forwardingCounters_;
+
+    // Using mutex instead of atomic_bool because choosing a new source to
     // forward messages should be done with a mutual exclusion otherwise there will be a race condition
     util::Mutex<bool> hasForwardingSource_{false};
 
@@ -114,6 +126,7 @@ public:
      * @param ioc The io_context to run on
      * @param backend BackendInterface implementation
      * @param subscriptions Subscription manager
+     * @param randomGenerator A random generator to use for selecting sources
      * @param validatedLedgers The network validated ledgers datastructure
      * @param sourceFactory A factory function to create a source
      */
@@ -122,6 +135,7 @@ public:
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+        std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
         std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
         SourceFactory sourceFactory = makeSource
     );
@@ -133,7 +147,8 @@ public:
      * @param ioc The io_context to run on
      * @param backend BackendInterface implementation
      * @param subscriptions Subscription manager
-     * @param validatedLedgers The network validated ledgers datastructure
+     * @param randomGenerator A random generator to use for selecting sources
+     * @param validatedLedgers The network validated ledgers data structure
      * @param sourceFactory A factory function to create a source
      * @return A shared pointer to a new instance of LoadBalancer
      */
@@ -143,12 +158,11 @@ public:
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+        std::unique_ptr<util::RandomGeneratorInterface> randomGenerator,
         std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
         SourceFactory sourceFactory = makeSource
     );
 
-    ~LoadBalancer() override;
-
     /**
      * @brief Load the initial ledger, writing data to the queue.
      * @note This function will retry indefinitely until the ledger is downloaded.

src/etl/Source.cpp

@@ -26,7 +26,7 @@
 #include "etl/impl/SourceImpl.hpp"
 #include "etl/impl/SubscriptionSource.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
-#include "util/newconfig/ObjectView.hpp"
+#include "util/config/ObjectView.hpp"
 
 #include <boost/asio/io_context.hpp>
 

src/etl/Source.hpp

@@ -23,7 +23,7 @@
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Errors.hpp"
-#include "util/newconfig/ObjectView.hpp"
+#include "util/config/ObjectView.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>

src/etl/impl/LedgerPublisher.hpp

@@ -24,6 +24,7 @@
 #include "data/LedgerCacheInterface.hpp"
 #include "data/Types.hpp"
 #include "etl/SystemState.hpp"
+#include "etlng/LedgerPublisherInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "util/Assert.hpp"
 #include "util/log/Logger.hpp"
@@ -31,6 +32,7 @@
 #include "util/prometheus/Prometheus.hpp"
 
 #include <boost/asio/io_context.hpp>
+#include <boost/asio/post.hpp>
 #include <boost/asio/strand.hpp>
 #include <xrpl/basics/chrono.h>
 #include <xrpl/protocol/Fees.h>
@@ -66,7 +68,7 @@ namespace etl::impl {
  * includes reading all of the transactions from the database) is done from the application wide asio io_service, and a
  * strand is used to ensure ledgers are published in order.
  */
-class LedgerPublisher {
+class LedgerPublisher : public etlng::LedgerPublisherInterface {
     util::Logger log_{"ETL"};
 
     boost::asio::strand<boost::asio::io_context::executor_type> publishStrand_;
@@ -121,7 +123,7 @@ public:
         uint32_t ledgerSequence,
         std::optional<uint32_t> maxAttempts,
         std::chrono::steady_clock::duration attemptsDelay = std::chrono::seconds{1}
-    )
+    ) override
     {
         LOG(log_.info()) << "Attempting to publish ledger = " << ledgerSequence;
         size_t numAttempts = 0;
@@ -235,7 +237,7 @@ public:
      * @brief Get time passed since last publish, in seconds
      */
     std::uint32_t
-    lastPublishAgeSeconds() const
+    lastPublishAgeSeconds() const override
     {
         return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - getLastPublish())
             .count();
@@ -245,7 +247,7 @@ public:
      * @brief Get last publish time as a time point
      */
     std::chrono::time_point<std::chrono::system_clock>
-    getLastPublish() const
+    getLastPublish() const override
     {
         return std::chrono::time_point<std::chrono::system_clock>{std::chrono::seconds{lastPublishSeconds_.get().value()
         }};
@@ -255,7 +257,7 @@ public:
      * @brief Get time passed since last ledger close, in seconds
      */
     std::uint32_t
-    lastCloseAgeSeconds() const
+    lastCloseAgeSeconds() const override
     {
         std::shared_lock const lck(closeTimeMtx_);
         auto now = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch())
@@ -267,7 +269,7 @@ public:
     }
 
     /**
-     * @brief Get the sequence of the last schueduled ledger to publish, Be aware that the ledger may not have been
+     * @brief Get the sequence of the last scheduled ledger to publish, Be aware that the ledger may not have been
      * published to network
      */
     std::optional<uint32_t>

src/etl/impl/SourceImpl.hpp

@@ -198,7 +198,6 @@ public:
      *
      * @param sequence Sequence of the ledger to download
      * @param numMarkers Number of markers to generate for async calls
-     * @param cacheOnly Only insert into cache, not the DB; defaults to false
      * @return A std::pair of the data and a bool indicating whether the download was successful
      */
     std::pair<std::vector<std::string>, bool>

src/etlng/CMakeLists.txt

@@ -2,7 +2,8 @@ add_library(clio_etlng)
 
 target_sources(
   clio_etlng
-  PRIVATE LoadBalancer.cpp
+  PRIVATE ETLService.cpp
+          LoadBalancer.cpp
           Source.cpp
           impl/AmendmentBlockHandler.cpp
           impl/AsyncGrpcCall.cpp
@@ -14,6 +15,7 @@ target_sources(
           impl/TaskManager.cpp
           impl/ext/Cache.cpp
           impl/ext/Core.cpp
+          impl/ext/MPT.cpp
           impl/ext/NFT.cpp
           impl/ext/Successor.cpp
 )

src/etlng/CacheLoaderInterface.hpp

@@ -0,0 +1,53 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <cstdint>
+
+namespace etlng {
+
+/**
+ * @brief An interface for the Cache Loader
+ */
+struct CacheLoaderInterface {
+    virtual ~CacheLoaderInterface() = default;
+
+    /**
+     * @brief Load the cache with the most recent ledger data
+     *
+     * @param seq The sequence number of the ledger to load
+     */
+    virtual void
+    load(uint32_t const seq) = 0;
+
+    /**
+     * @brief Stop the cache loading process
+     */
+    virtual void
+    stop() noexcept = 0;
+
+    /**
+     * @brief Wait for all cache loading tasks to complete
+     */
+    virtual void
+    wait() noexcept = 0;
+};
+
+}  // namespace etlng

src/etlng/CacheUpdaterInterface.hpp

@@ -0,0 +1,66 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/Types.hpp"
+#include "etlng/Models.hpp"
+
+#include <cstdint>
+#include <vector>
+
+namespace etlng {
+
+/**
+ * @brief An interface for the Cache Updater
+ */
+struct CacheUpdaterInterface {
+    virtual ~CacheUpdaterInterface() = default;
+
+    /**
+     * @brief Update the cache with ledger data
+     * @param data The ledger data to update with
+     */
+    virtual void
+    update(model::LedgerData const& data) = 0;
+
+    /**
+     * @brief Update the cache with ledger objects at a specific sequence
+     * @param seq The ledger sequence number
+     * @param objs The ledger objects to update with
+     */
+    virtual void
+    update(uint32_t seq, std::vector<data::LedgerObject> const& objs) = 0;
+
+    /**
+     * @brief Update the cache with model objects at a specific sequence
+     * @param seq The ledger sequence number
+     * @param objs The model objects to update with
+     */
+    virtual void
+    update(uint32_t seq, std::vector<model::Object> const& objs) = 0;
+
+    /**
+     * @brief Mark the cache as fully loaded
+     */
+    virtual void
+    setFull() = 0;
+};
+
+}  // namespace etlng

src/etlng/ETLService.cpp

@@ -0,0 +1,265 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/ETLService.hpp"
+
+#include "data/BackendInterface.hpp"
+#include "data/LedgerCacheInterface.hpp"
+#include "data/Types.hpp"
+#include "etl/ETLState.hpp"
+#include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etl/SystemState.hpp"
+#include "etl/impl/AmendmentBlockHandler.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "etlng/CacheLoaderInterface.hpp"
+#include "etlng/CacheUpdaterInterface.hpp"
+#include "etlng/ExtractorInterface.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/LedgerPublisherInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
+#include "etlng/LoaderInterface.hpp"
+#include "etlng/MonitorInterface.hpp"
+#include "etlng/TaskManagerProviderInterface.hpp"
+#include "etlng/impl/AmendmentBlockHandler.hpp"
+#include "etlng/impl/CacheUpdater.hpp"
+#include "etlng/impl/Extraction.hpp"
+#include "etlng/impl/LedgerPublisher.hpp"
+#include "etlng/impl/Loading.hpp"
+#include "etlng/impl/Monitor.hpp"
+#include "etlng/impl/Registry.hpp"
+#include "etlng/impl/Scheduling.hpp"
+#include "etlng/impl/TaskManager.hpp"
+#include "etlng/impl/ext/Cache.hpp"
+#include "etlng/impl/ext/Core.hpp"
+#include "etlng/impl/ext/NFT.hpp"
+#include "etlng/impl/ext/Successor.hpp"
+#include "util/Assert.hpp"
+#include "util/Profiler.hpp"
+#include "util/async/AnyExecutionContext.hpp"
+#include "util/log/Logger.hpp"
+
+#include <boost/json/object.hpp>
+#include <boost/signals2/connection.hpp>
+#include <fmt/core.h>
+
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+
+namespace etlng {
+
+ETLService::ETLService(
+    util::async::AnyExecutionContext ctx,
+    std::reference_wrapper<util::config::ClioConfigDefinition const> config,
+    std::shared_ptr<data::BackendInterface> backend,
+    std::shared_ptr<LoadBalancerInterface> balancer,
+    std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers,
+    std::shared_ptr<LedgerPublisherInterface> publisher,
+    std::shared_ptr<CacheLoaderInterface> cacheLoader,
+    std::shared_ptr<CacheUpdaterInterface> cacheUpdater,
+    std::shared_ptr<ExtractorInterface> extractor,
+    std::shared_ptr<LoaderInterface> loader,
+    std::shared_ptr<InitialLoadObserverInterface> initialLoadObserver,
+    std::shared_ptr<etlng::TaskManagerProviderInterface> taskManagerProvider,
+    std::shared_ptr<etl::SystemState> state
+)
+    : ctx_(std::move(ctx))
+    , config_(config)
+    , backend_(std::move(backend))
+    , balancer_(std::move(balancer))
+    , ledgers_(std::move(ledgers))
+    , publisher_(std::move(publisher))
+    , cacheLoader_(std::move(cacheLoader))
+    , cacheUpdater_(std::move(cacheUpdater))
+    , extractor_(std::move(extractor))
+    , loader_(std::move(loader))
+    , initialLoadObserver_(std::move(initialLoadObserver))
+    , taskManagerProvider_(std::move(taskManagerProvider))
+    , state_(std::move(state))
+{
+    LOG(log_.info()) << "Creating ETLng...";
+}
+
+ETLService::~ETLService()
+{
+    stop();
+    LOG(log_.debug()) << "Destroying ETLng";
+}
+
+void
+ETLService::run()
+{
+    LOG(log_.info()) << "Running ETLng...";
+
+    // TODO: write-enabled node should start in readonly and do the 10 second dance to become a writer
+    mainLoop_.emplace(ctx_.execute([this] {
+        state_->isWriting =
+            not state_->isReadOnly;  // TODO: this is now needed because we don't have a mechanism for readonly or
+                                     // ETL writer node. remove later in favor of real mechanism
+
+        auto const rng = loadInitialLedgerIfNeeded();
+
+        LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
+        std::optional<uint32_t> const mostRecentValidated = ledgers_->getMostRecent();
+
+        if (not mostRecentValidated) {
+            LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
+                                "Exiting monitor loop";
+            return;
+        }
+
+        ASSERT(rng.has_value(), "Ledger range can't be null");
+        auto const nextSequence = rng->maxSequence + 1;
+
+        LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence;
+        startMonitor(nextSequence);
+
+        // TODO: we only want to run the full ETL task man if we are POSSIBLY a write node
+        // but definitely not in strict readonly
+        if (not state_->isReadOnly)
+            startLoading(nextSequence);
+    }));
+}
+
+void
+ETLService::stop()
+{
+    LOG(log_.info()) << "Stop called";
+
+    if (taskMan_)
+        taskMan_->stop();
+    if (monitor_)
+        monitor_->stop();
+}
+
+boost::json::object
+ETLService::getInfo() const
+{
+    boost::json::object result;
+
+    result["etl_sources"] = balancer_->toJson();
+    result["is_writer"] = static_cast<int>(state_->isWriting);
+    result["read_only"] = static_cast<int>(state_->isReadOnly);
+    auto last = publisher_->getLastPublish();
+    if (last.time_since_epoch().count() != 0)
+        result["last_publish_age_seconds"] = std::to_string(publisher_->lastPublishAgeSeconds());
+    return result;
+}
+
+bool
+ETLService::isAmendmentBlocked() const
+{
+    return state_->isAmendmentBlocked;
+}
+
+bool
+ETLService::isCorruptionDetected() const
+{
+    return state_->isCorruptionDetected;
+}
+
+std::optional<etl::ETLState>
+ETLService::getETLState() const
+{
+    return balancer_->getETLState();
+}
+
+std::uint32_t
+ETLService::lastCloseAgeSeconds() const
+{
+    return publisher_->lastCloseAgeSeconds();
+}
+
+std::optional<data::LedgerRange>
+ETLService::loadInitialLedgerIfNeeded()
+{
+    auto rng = backend_->hardFetchLedgerRangeNoThrow();
+    if (not rng.has_value()) {
+        LOG(log_.info()) << "Database is empty. Will download a ledger from the network.";
+
+        LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
+        if (auto const mostRecentValidated = ledgers_->getMostRecent(); mostRecentValidated.has_value()) {
+            auto const seq = *mostRecentValidated;
+            LOG(log_.info()) << "Ledger " << seq << " has been validated. Downloading... ";
+
+            auto [ledger, timeDiff] = ::util::timed<std::chrono::duration<double>>([this, seq]() {
+                return extractor_->extractLedgerOnly(seq).and_then([this, seq](auto&& data) {
+                    // TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar
+                    data.edgeKeys = balancer_->loadInitialLedger(seq, *initialLoadObserver_);
+
+                    // TODO: this should be interruptible for graceful shutdown
+                    return loader_->loadInitialLedger(data);
+                });
+            });
+
+            if (not ledger.has_value()) {
+                LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop";
+                return std::nullopt;
+            }
+
+            LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff;
+            LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
+
+            return backend_->hardFetchLedgerRangeNoThrow();
+        }
+
+        LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
+                            "Exiting monitor loop";
+        return std::nullopt;
+    }
+
+    LOG(log_.info()) << "Database already populated. Picking up from the tip of history";
+    cacheLoader_->load(rng->maxSequence);
+
+    return rng;
+}
+
+void
+ETLService::startMonitor(uint32_t seq)
+{
+    monitor_ = std::make_unique<impl::Monitor>(ctx_, backend_, ledgers_, seq);
+    monitorSubscription_ = monitor_->subscribe([this](uint32_t seq) {
+        log_.info() << "MONITOR got new seq from db: " << seq;
+
+        // FIXME: is this the best way?
+        if (not state_->isWriting) {
+            auto const diff = data::synchronousAndRetryOnTimeout([this, seq](auto yield) {
+                return backend_->fetchLedgerDiff(seq, yield);
+            });
+            cacheUpdater_->update(seq, diff);
+        }
+
+        publisher_->publish(seq, {});
+    });
+    monitor_->run();
+}
+
+void
+ETLService::startLoading(uint32_t seq)
+{
+    taskMan_ = taskManagerProvider_->make(ctx_, *monitor_, seq);
+    taskMan_->run(config_.get().get<std::size_t>("extractor_threads"));
+}
+
+}  // namespace etlng

src/etlng/ETLService.hpp

@@ -20,22 +20,27 @@
 #pragma once
 
 #include "data/BackendInterface.hpp"
-#include "data/LedgerCache.hpp"
 #include "data/Types.hpp"
-#include "etl/CacheLoader.hpp"
 #include "etl/ETLState.hpp"
-#include "etl/LedgerFetcherInterface.hpp"
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/SystemState.hpp"
 #include "etl/impl/AmendmentBlockHandler.hpp"
 #include "etl/impl/LedgerFetcher.hpp"
-#include "etl/impl/LedgerPublisher.hpp"
-#include "etlng/AmendmentBlockHandlerInterface.hpp"
+#include "etlng/CacheLoaderInterface.hpp"
+#include "etlng/CacheUpdaterInterface.hpp"
 #include "etlng/ETLServiceInterface.hpp"
 #include "etlng/ExtractorInterface.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/LedgerPublisherInterface.hpp"
 #include "etlng/LoadBalancerInterface.hpp"
+#include "etlng/LoaderInterface.hpp"
+#include "etlng/MonitorInterface.hpp"
+#include "etlng/TaskManagerInterface.hpp"
+#include "etlng/TaskManagerProviderInterface.hpp"
 #include "etlng/impl/AmendmentBlockHandler.hpp"
+#include "etlng/impl/CacheUpdater.hpp"
 #include "etlng/impl/Extraction.hpp"
+#include "etlng/impl/LedgerPublisher.hpp"
 #include "etlng/impl/Loading.hpp"
 #include "etlng/impl/Monitor.hpp"
 #include "etlng/impl/Registry.hpp"
@@ -45,15 +50,14 @@
 #include "etlng/impl/ext/Core.hpp"
 #include "etlng/impl/ext/NFT.hpp"
 #include "etlng/impl/ext/Successor.hpp"
-#include "feed/SubscriptionManagerInterface.hpp"
-#include "util/Assert.hpp"
-#include "util/Profiler.hpp"
-#include "util/async/context/BasicExecutionContext.hpp"
-#include "util/config/Config.hpp"
+#include "util/async/AnyExecutionContext.hpp"
+#include "util/async/AnyOperation.hpp"
+#include "util/config/ConfigDefinition.hpp"
 #include "util/log/Logger.hpp"
-#include "util/newconfig/ConfigDefinition.hpp"
 
+#include <boost/asio/io_context.hpp>
 #include <boost/json/object.hpp>
+#include <boost/signals2/connection.hpp>
 #include <fmt/core.h>
 #include <xrpl/basics/Blob.h>
 #include <xrpl/basics/base_uint.h>
@@ -65,15 +69,12 @@
 #include <xrpl/protocol/TxFormats.h>
 #include <xrpl/protocol/TxMeta.h>
 
-#include <chrono>
+#include <cstddef>
 #include <cstdint>
+#include <functional>
 #include <memory>
 #include <optional>
-#include <ranges>
-#include <stdexcept>
 #include <string>
-#include <tuple>
-#include <utility>
 
 namespace etlng {
 
@@ -93,191 +94,94 @@ namespace etlng {
 class ETLService : public ETLServiceInterface {
     util::Logger log_{"ETL"};
 
+    util::async::AnyExecutionContext ctx_;
+    std::reference_wrapper<util::config::ClioConfigDefinition const> config_;
     std::shared_ptr<BackendInterface> backend_;
-    std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions_;
-    std::shared_ptr<etlng::LoadBalancerInterface> balancer_;
+    std::shared_ptr<LoadBalancerInterface> balancer_;
     std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers_;
-    std::shared_ptr<etl::CacheLoader<>> cacheLoader_;
-
-    std::shared_ptr<etl::LedgerFetcherInterface> fetcher_;
+    std::shared_ptr<LedgerPublisherInterface> publisher_;
+    std::shared_ptr<CacheLoaderInterface> cacheLoader_;
+    std::shared_ptr<CacheUpdaterInterface> cacheUpdater_;
     std::shared_ptr<ExtractorInterface> extractor_;
+    std::shared_ptr<LoaderInterface> loader_;
+    std::shared_ptr<InitialLoadObserverInterface> initialLoadObserver_;
+    std::shared_ptr<etlng::TaskManagerProviderInterface> taskManagerProvider_;
+    std::shared_ptr<etl::SystemState> state_;
 
-    etl::SystemState state_;
-    util::async::CoroExecutionContext ctx_{8};
+    std::unique_ptr<MonitorInterface> monitor_;
+    std::unique_ptr<TaskManagerInterface> taskMan_;
 
-    std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler_;
-    std::shared_ptr<impl::Loader> loader_;
+    boost::signals2::scoped_connection monitorSubscription_;
 
-    std::optional<util::async::CoroExecutionContext::Operation<void>> mainLoop_;
+    std::optional<util::async::AnyOperation<void>> mainLoop_;
 
 public:
     /**
      * @brief Create an instance of ETLService.
      *
-     * @param config The configuration to use
-     * @param backend BackendInterface implementation
-     * @param subscriptions Subscription manager
-     * @param balancer Load balancer to use
-     * @param ledgers The network validated ledgers datastructure
+     * @param ctx The execution context for asynchronous operations
+     * @param config The Clio configuration definition
+     * @param backend Interface to the backend database
+     * @param balancer Load balancer for distributing work
+     * @param ledgers Interface for accessing network validated ledgers
+     * @param publisher Interface for publishing ledger data
+     * @param cacheLoader Interface for loading cache data
+     * @param cacheUpdater Interface for updating cache data
+     * @param extractor The extractor to use
+     * @param loader Interface for loading data
+     * @param initialLoadObserver The observer for initial data loading
+     * @param taskManagerProvider The provider of the task manager instance
+     * @param state System state tracking object
      */
     ETLService(
-        util::config::ClioConfigDefinition const& config,
-        std::shared_ptr<BackendInterface> backend,
-        std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
-        std::shared_ptr<etlng::LoadBalancerInterface> balancer,
-        std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers
-    )
-        : backend_(std::move(backend))
-        , subscriptions_(std::move(subscriptions))
-        , balancer_(std::move(balancer))
-        , ledgers_(std::move(ledgers))
-        , cacheLoader_(std::make_shared<etl::CacheLoader<>>(config, backend_, backend_->cache()))
-        , fetcher_(std::make_shared<etl::impl::LedgerFetcher>(backend_, balancer_))
-        , extractor_(std::make_shared<impl::Extractor>(fetcher_))
-        , amendmentBlockHandler_(std::make_shared<etlng::impl::AmendmentBlockHandler>(ctx_, state_))
-        , loader_(std::make_shared<impl::Loader>(
-              backend_,
-              fetcher_,
-              impl::makeRegistry(
-                  impl::CacheExt{backend_->cache()},
-                  impl::CoreExt{backend_},
-                  impl::SuccessorExt{backend_, backend_->cache()},
-                  impl::NFTExt{backend_}
-              ),
-              amendmentBlockHandler_
-          ))
-    {
-        LOG(log_.info()) << "Creating ETLng...";
-    }
+        util::async::AnyExecutionContext ctx,
+        std::reference_wrapper<util::config::ClioConfigDefinition const> config,
+        std::shared_ptr<data::BackendInterface> backend,
+        std::shared_ptr<LoadBalancerInterface> balancer,
+        std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers,
+        std::shared_ptr<LedgerPublisherInterface> publisher,
+        std::shared_ptr<CacheLoaderInterface> cacheLoader,
+        std::shared_ptr<CacheUpdaterInterface> cacheUpdater,
+        std::shared_ptr<ExtractorInterface> extractor,
+        std::shared_ptr<LoaderInterface> loader,
+        std::shared_ptr<InitialLoadObserverInterface> initialLoadObserver,
+        std::shared_ptr<etlng::TaskManagerProviderInterface> taskManagerProvider,
+        std::shared_ptr<etl::SystemState> state
+    );
 
-    ~ETLService() override
-    {
-        LOG(log_.debug()) << "Stopping ETLng";
-    }
+    ~ETLService() override;
 
     void
-    run() override
-    {
-        LOG(log_.info()) << "run() in ETLng...";
-
-        mainLoop_.emplace(ctx_.execute([this] {
-            auto const rng = loadInitialLedgerIfNeeded();
-
-            LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
-            std::optional<uint32_t> const mostRecentValidated = ledgers_->getMostRecent();
-
-            if (not mostRecentValidated) {
-                LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
-                                    "Exiting monitor loop";
-                return;
-            }
-
-            ASSERT(rng.has_value(), "Ledger range can't be null");
-            auto const nextSequence = rng->maxSequence + 1;
-
-            LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence;
-
-            auto scheduler = impl::makeScheduler(impl::ForwardScheduler{*ledgers_, nextSequence}
-                                                 // impl::BackfillScheduler{nextSequence - 1, nextSequence - 1000},
-                                                 // TODO lift limit and start with rng.minSeq
-            );
-
-            auto man = impl::TaskManager(ctx_, *scheduler, *extractor_, *loader_);
-
-            // TODO: figure out this: std::make_shared<impl::Monitor>(backend_, ledgers_, nextSequence)
-            man.run({});  // TODO: needs to be interruptible and fill out settings
-        }));
-    }
+    run() override;
 
     void
-    stop() override
-    {
-        LOG(log_.info()) << "Stop called";
-        // TODO: stop the service correctly
-    }
+    stop() override;
 
     boost::json::object
-    getInfo() const override
-    {
-        // TODO
-        return {{"ok", true}};
-    }
+    getInfo() const override;
 
     bool
-    isAmendmentBlocked() const override
-    {
-        // TODO
-        return false;
-    }
+    isAmendmentBlocked() const override;
 
     bool
-    isCorruptionDetected() const override
-    {
-        // TODO
-        return false;
-    }
+    isCorruptionDetected() const override;
 
     std::optional<etl::ETLState>
-    getETLState() const override
-    {
-        // TODO
-        return std::nullopt;
-    }
+    getETLState() const override;
 
     std::uint32_t
-    lastCloseAgeSeconds() const override
-    {
-        // TODO
-        return 0;
-    }
+    lastCloseAgeSeconds() const override;
 
 private:
     // TODO: this better be std::expected
     std::optional<data::LedgerRange>
-    loadInitialLedgerIfNeeded()
-    {
-        if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); not rng.has_value()) {
-            LOG(log_.info()) << "Database is empty. Will download a ledger from the network.";
+    loadInitialLedgerIfNeeded();
 
-            try {
-                LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
-                if (auto const mostRecentValidated = ledgers_->getMostRecent(); mostRecentValidated.has_value()) {
-                    auto const seq = *mostRecentValidated;
-                    LOG(log_.info()) << "Ledger " << seq << " has been validated. Downloading... ";
+    void
+    startMonitor(uint32_t seq);
 
-                    auto [ledger, timeDiff] = ::util::timed<std::chrono::duration<double>>([this, seq]() {
-                        return extractor_->extractLedgerOnly(seq).and_then([this, seq](auto&& data) {
-                            // TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar
-                            data.edgeKeys = balancer_->loadInitialLedger(seq, *loader_);
-
-                            // TODO: this should be interruptible for graceful shutdown
-                            return loader_->loadInitialLedger(data);
-                        });
-                    });
-
-                    LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff;
-                    LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
-
-                    if (ledger.has_value())
-                        return backend_->hardFetchLedgerRangeNoThrow();
-
-                    LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop";
-                } else {
-                    LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
-                                        "Exiting monitor loop";
-                }
-            } catch (std::runtime_error const& e) {
-                LOG(log_.fatal()) << "Failed to load initial ledger: " << e.what();
-                amendmentBlockHandler_->notifyAmendmentBlocked();
-            }
-        } else {
-            LOG(log_.info()) << "Database already populated. Picking up from the tip of history";
-            cacheLoader_->load(rng->maxSequence);
-
-            return rng;
-        }
-
-        return std::nullopt;
-    }
+    void
+    startLoading(uint32_t seq);
 };
+
 }  // namespace etlng

src/etlng/LedgerPublisherInterface.hpp

@@ -37,13 +37,38 @@ struct LedgerPublisherInterface {
      * @param seq The sequence number of the ledger
      * @param maxAttempts The maximum number of attempts to publish the ledger; no limit if nullopt
      * @param attemptsDelay The delay between attempts
+     * @return Whether the ledger was found in the database and published
      */
-    virtual void
+    virtual bool
     publish(
         uint32_t seq,
         std::optional<uint32_t> maxAttempts,
         std::chrono::steady_clock::duration attemptsDelay = std::chrono::seconds{1}
     ) = 0;
+
+    /**
+     * @brief Get last publish time as a time point
+     *
+     * @return A std::chrono::time_point representing the time of the last publish
+     */
+    virtual std::chrono::time_point<std::chrono::system_clock>
+    getLastPublish() const = 0;
+
+    /**
+     * @brief Get time passed since last ledger close, in seconds
+     *
+     * @return The number of seconds since the last ledger close as std::uint32_t
+     */
+    virtual std::uint32_t
+    lastCloseAgeSeconds() const = 0;
+
+    /**
+     * @brief Get time passed since last publish, in seconds
+     *
+     * @return The number of seconds since the last publish as std::uint32_t
+     */
+    virtual std::uint32_t
+    lastPublishAgeSeconds() const = 0;
 };
 
 }  // namespace etlng