feat: Implement MPT changes (#1147)

Implements https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens

.clang-tidy

@@ -142,7 +142,7 @@ CheckOptions:
   readability-braces-around-statements.ShortStatementLines: 2
   bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true
   bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc
-  misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*'
+  misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*;.*ranges_lower_bound\.h;time.h;stdlib.h'
 
 HeaderFilterRegex: '^.*/(src|tests)/.*\.(h|hpp)$'
 WarningsAsErrors: '*'

.clangd

@@ -5,4 +5,9 @@ Diagnostics:
   UnusedIncludes: Strict
   MissingIncludes: Strict
   Includes:
-    IgnoreHeader: ".*/(detail|impl)/.*"
+    IgnoreHeader:
+      - ".*/(detail|impl)/.*"
+      - ".*expected.*"
+      - ".*ranges_lower_bound.h"
+      - "time.h"
+      - "stdlib.h"

.githooks/check-docs

@@ -13,14 +13,36 @@ TMPDIR=${ROOT}/.cache/doxygen
 TMPFILE=${TMPDIR}/docs.log
 DOCDIR=${TMPDIR}/out
 
+# Check doxygen is at all installed
 if [ -z "$DOXYGEN" ]; then
     # No hard error if doxygen is not installed yet
     cat <<EOF
 
                                    WARNING
 -----------------------------------------------------------------------------
-        'doxygen' is required to check documentation.
-        Please install it for next time. For the time being it's on CI.
+        'doxygen' is required to check documentation. 
+        Please install it for next time.
+
+        Your changes may fail to pass CI once pushed.
+-----------------------------------------------------------------------------
+
+EOF
+    exit 0
+fi
+
+# Check version of doxygen is at least 1.12
+version=$($DOXYGEN --version | grep -o '[0-9\.]*')
+
+if [[ "1.12.0" > "$version" ]]; then
+    # No hard error if doxygen version is not the one we want - let CI deal with it
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+        A minimum of version 1.12 of `which doxygen` is required.
+        Your version is $version. Please upgrade it for next time.
+
+        Your changes may fail to pass CI once pushed.
 -----------------------------------------------------------------------------
 
 EOF

.githooks/check-format

@@ -5,6 +5,20 @@
 # This script checks the format of the code and cmake files.
 # In many cases it will automatically fix the issues and abort the commit.
 
+no_formatted_directories_staged() {
+    staged_directories=$(git diff-index --cached --name-only HEAD | awk -F/ '{print $1}')
+    for sd in $staged_directories; do
+        if [[ "$sd" =~ ^(benchmark|cmake|src|tests)$ ]]; then
+            return 1
+        fi
+    done
+    return 0
+}
+
+if no_formatted_directories_staged ; then
+    exit 0
+fi
+
 echo "+ Checking code format..."
 
 # paths to check and re-format
@@ -12,12 +26,12 @@ sources="src tests"
 formatter="clang-format -i"
 version=$($formatter --version | grep -o '[0-9\.]*')
 
-if [[ "18.0.0" > "$version" ]]; then
+if [[ "19.0.0" > "$version" ]]; then
     cat <<EOF
 
                                     ERROR
 -----------------------------------------------------------------------------
-            A minimum of version 18 of `which clang-format` is required.
+            A minimum of version 19 of `which clang-format` is required.
             Your version is $version.
             Please fix paths and run again.
 -----------------------------------------------------------------------------

.githooks/pre-commit

@@ -4,3 +4,4 @@
 
 source .githooks/check-format
 source .githooks/check-docs
+

.githooks/pre-push

@@ -1,3 +1,58 @@
 #!/bin/sh
+
+
+# git for-each-ref refs/tags  # see which tags are annotated and which are lightweight. Annotated tags are "tag" objects.
+# # Set these so your commits and tags are always signed
+# git config commit.gpgsign true
+# git config tag.gpgsign true
+
+verify_commit_signed() {
+    if git verify-commit HEAD &> /dev/null; then
+        :
+        # echo "HEAD commit seems signed..."
+    else
+        echo "HEAD commit isn't signed!"
+        exit 1
+    fi
+}
+
+verify_tag() {
+    if git describe --exact-match --tags HEAD &> /dev/null; then
+        : # You might be ok to push
+        # echo "Tag is annotated."
+        return 0
+    else
+        echo "Tag for [$version] not an annotated tag."
+        exit 1
+    fi
+}
+
+verify_tag_signed() {
+    if git verify-tag "$version" &> /dev/null ; then
+        : # ok, I guess we'll let you push
+        # echo "Tag appears signed"
+        return 0
+    else
+        echo "$version tag isn't signed"
+        echo "Sign it with [git tag -ams\"$version\" $version]"
+        exit 1
+    fi
+}
+
+while read local_ref local_oid remote_ref remote_oid; do
+    # Check some things if we're pushing a branch called "release/"
+    if echo "$remote_ref" | grep ^refs\/heads\/release\/ &> /dev/null ; then
+        version=$(git tag --points-at HEAD)
+        echo "Looks like you're trying to push a $version release..."
+        echo "Making sure you've signed and tagged it."
+        if verify_commit_signed && verify_tag && verify_tag_signed ; then
+            : # Ok, I guess you can push
+        else
+            exit 1
+        fi
+    fi
+done
+
 command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'pre-push' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+
 git lfs pre-push "$@"

.github/actions/build_docker_image/action.yml

@@ -0,0 +1,66 @@
+name: Build and push Docker image
+description: Build and push Docker image to DockerHub and GitHub Container Registry
+inputs:
+  image_name:
+    description: Name of the image to build
+    required: true
+  push_image:
+    description: Whether to push the image to the registry (true/false)
+    required: true
+  directory:
+    description: The directory containing the Dockerfile
+    required: true
+  tags:
+    description: Comma separated tags to apply to the image
+    required: true
+  platforms:
+    description: Platforms to build the image for (e.g. linux/amd64,linux/arm64)
+    required: true
+  description:
+    description: Short description of the image
+    required: true
+runs:
+  using: composite
+  steps:
+      - name: Login to DockerHub
+        if: ${{ inputs.push_image == 'true' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_PW }}
+
+      - name: Login to GitHub Container Registry
+        if: ${{ inputs.push_image == 'true' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ env.GITHUB_TOKEN }}
+
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/metadata-action@v5
+        id: meta
+        with:
+          images: ${{ inputs.image_name }}
+          tags: ${{ inputs.tags }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ inputs.directory }}
+          platforms: ${{ inputs.platforms }}
+          push: ${{ inputs.push_image == 'true' }}
+          tags: ${{ steps.meta.outputs.tags }}
+
+      - name: Update DockerHub description
+        if: ${{ inputs.push_image == 'true' }}
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_PW }}
+          repository: ${{ inputs.image_name }}
+          short-description: ${{ inputs.description }}
+          readme-filepath: ${{ inputs.directory }}/README.md
+

.github/actions/create_issue/action.yml

@@ -7,6 +7,14 @@ inputs:
   body:
     description: Issue body
     required: true
+  labels:
+    description: Comma-separated list of labels
+    required: true
+    default: 'bug'
+  assignees:
+    description: Comma-separated list of assignees
+    required: true
+    default: 'cindyyan317,godexsoft,kuznetsss'
 outputs:
   created_issue_id:
     description: Created issue id
@@ -19,7 +27,7 @@ runs:
         shell: bash
         run: |
           echo -e '${{ inputs.body }}' > issue.md
-          gh issue create --assignee 'cindyyan317,godexsoft,kuznetsss' --label bug --title '${{ inputs.title }}' --body-file ./issue.md > create_issue.log
+          gh issue create --assignee '${{ inputs.assignees }}' --label '${{ inputs.labels }}' --title '${{ inputs.title }}' --body-file ./issue.md > create_issue.log
           created_issue=$(cat create_issue.log | sed 's|.*/||')
           echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
           rm create_issue.log issue.md

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "04:00"
+      timezone: "Etc/GMT"
+    reviewers:
+      - "cindyyan317"
+      - "godexsoft"
+      - "kuznetsss"
+    commit-message:
+      prefix: "[CI] "
+    target-branch: "develop"

.github/scripts/update-libxrpl-version

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Note: This script is intended to be run from the root of the repository.
+#
+# This script modifies conanfile.py such that the specified version of libXRPL is used.
+
+if [[ -z "$1" ]]; then
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+            Version should be passed as first argument to the script.
+-----------------------------------------------------------------------------
+
+EOF
+    exit 1
+fi
+
+VERSION=$1
+GNU_SED=$(sed --version 2>&1 | grep -q 'GNU' && echo true || echo false)
+
+echo "+ Updating required libXRPL version to $VERSION"
+
+if [[ "$GNU_SED" == "false" ]]; then
+    sed -i '' -E "s|'xrpl/[a-zA-Z0-9\\.\\-]+'|'xrpl/$VERSION'|g" conanfile.py
+else
+    sed -i -E "s|'xrpl/[a-zA-Z0-9\\.\\-]+'|'xrpl/$VERSION'|g" conanfile.py
+fi

.github/workflows/build.yml

@@ -23,7 +23,7 @@ jobs:
         run: |
           ./.githooks/check-format --diff
         shell: bash
-  
+ 
   check_docs:
     name: Check documentation
     runs-on: ubuntu-20.04
@@ -211,6 +211,7 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
+
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests

.github/workflows/build_clio_docker_image.yml

@@ -0,0 +1,95 @@
+name: Build and publish Clio docker image
+on:
+  workflow_call:
+    inputs:
+      tags:
+        required: true
+        type: string
+        description: Comma separated tags for docker image
+      artifact_name:
+        type: string
+        description: Name of Github artifact to put into docker image
+      strip_binary:
+        type: boolean
+        description: Whether to strip clio binary
+        default: true
+      publish_image:
+        type: boolean
+        description: Whether to publish docker image
+        required: true
+
+  workflow_dispatch:
+    inputs:
+      tags:
+        required: true
+        type: string
+        description: Comma separated tags for docker image
+      clio_server_binary_url:
+        required: true
+        type: string
+        description: Url to download clio_server binary from
+      binary_sha256:
+        required: true
+        type: string
+        description: sha256 hash of the binary
+      strip_binary:
+        type: boolean
+        description: Whether to strip clio binary
+        default: true
+
+jobs:
+  build_and_publish_image:
+    name: Build and publish image
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download Clio binary from artifact
+        if: ${{ inputs.artifact_name != null }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.artifact_name }}
+          path: ./docker/clio/artifact/
+
+      - name: Download Clio binary from url
+        if: ${{ inputs.clio_server_binary_url != null }}
+        shell: bash
+        run: |
+          wget ${{inputs.clio_server_binary_url}} -P ./docker/clio/artifact/
+          if [ "$(sha256sum ./docker/clio/clio_server | awk '{print $1}')" != "${{inputs.binary_sha256}}" ]; then
+            echo "Binary sha256 sum doesn't match"
+            exit 1
+          fi
+      - name: Unpack binary
+        shell: bash
+        run: |
+          sudo apt update && sudo apt install -y tar unzip
+          cd docker/clio/artifact
+          artifact=$(find . -type f)
+          if [[ $artifact == *.zip ]]; then
+            unzip $artifact
+          elif [[ $artifact == *.tar.gz ]]; then
+            tar -xvf $artifact
+          fi
+          mv clio_server ../
+          cd ../
+          rm -rf ./artifact
+
+      - name: Strip binary
+        if: ${{ inputs.strip_binary }}
+        shell: bash
+        run: strip ./docker/clio/clio_server
+
+      - name: Build Docker image
+        uses: ./.github/actions/build_docker_image
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          image_name: rippleci/clio
+          push_image: ${{ inputs.publish_image }}
+          directory: docker/clio
+          tags: ${{ inputs.tags }}
+          platforms: linux/amd64
+          description: Clio is an XRP Ledger API server.

.github/workflows/check_libxrpl.yml

@@ -0,0 +1,91 @@
+name: Check new libXRPL
+on:
+  repository_dispatch:
+    types: [check_libxrpl]
+
+jobs:
+  build:
+    name: Build Clio / `libXRPL ${{ github.event.client_payload.version }}`
+    runs-on: [self-hosted, heavy]
+    container:
+      image: rippleci/clio_ci:latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            fetch-depth: 0
+
+      - name: Update libXRPL version requirement
+        shell: bash
+        run: |
+          ./.github/scripts/update-libxrpl-version ${{ github.event.client_payload.version }}
+
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+            disable_ccache: true
+
+      - name: Setup conan
+        uses: ./.github/actions/setup_conan
+        id: conan
+        with:
+          conan_profile: gcc
+
+      - name: Run conan and cmake
+        uses: ./.github/actions/generate
+        with:
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
+          build_type: Release
+
+      - name: Build Clio
+        uses: ./.github/actions/build_clio
+
+      - name: Strip tests
+        run: strip build/clio_tests
+
+      - name: Upload clio_tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: clio_tests_check_libxrpl
+          path: build/clio_tests
+
+  run_tests:
+    name: Run tests
+    needs: build
+    runs-on: [self-hosted, heavy]
+    container:
+      image: rippleci/clio_ci:latest
+
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: clio_tests_check_libxrpl
+
+      - name: Run clio_tests
+        run: |
+          chmod +x ./clio_tests
+          ./clio_tests
+
+  create_issue_on_failure:
+    name: Create an issue on failure
+    needs: [build, run_tests]
+    if: ${{ always() && contains(needs.*.result, 'failure') }}
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create an issue
+        uses: ./.github/actions/create_issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          labels: 'compatibility,bug'
+          title: 'Proposed libXRPL check failed'
+          body: >
+            Clio build or tests failed against `libXRPL ${{ github.event.client_payload.version }}`.
+
+            Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/

.github/workflows/check_pr_title.yml

@@ -0,0 +1,18 @@
+name: Check PR title
+on:
+  pull_request:
+    types: [opened, edited, reopened, synchronize]
+    branches: [develop]
+
+jobs:
+  check_title:
+    runs-on: ubuntu-20.04
+    # permissions:
+    #   pull-requests: write
+    steps:
+      - uses: ytanikin/PRConventionalCommits@1.3.0
+        with:
+          task_types: '["build","feat","fix","docs","test","ci","style","refactor","perf","chore"]'
+          add_label: false
+          # Turned off labelling because it leads to an error, see https://github.com/ytanikin/PRConventionalCommits/issues/19
+          # custom_labels: '{"build":"build", "feat":"enhancement", "fix":"bug", "docs":"documentation", "test":"testability", "ci":"ci", "style":"refactoring", "refactor":"refactoring", "perf":"performance", "chore":"tooling"}'

.github/workflows/clang-tidy.yml

@@ -60,7 +60,7 @@ jobs:
         shell: bash
         id: run_clang_tidy
         run: |
-          run-clang-tidy-18 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
+          run-clang-tidy-19 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
 
       - name: Check format
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
@@ -89,7 +89,7 @@ jobs:
 
             List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/
 
-      - uses: crazy-max/ghaction-import-gpg@v5
+      - uses: crazy-max/ghaction-import-gpg@v6
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
         with:
           gpg_private_key: ${{ secrets.ACTIONS_GPG_PRIVATE_KEY }}
@@ -99,7 +99,7 @@ jobs:
 
       - name: Create PR with fixes
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v7
         env:
           GH_REPO: ${{ github.repository }}
           GH_TOKEN: ${{ github.token }}
@@ -109,7 +109,7 @@ jobs:
           branch: "clang_tidy/autofix"
           branch-suffix: timestamp
           delete-branch: true
-          title: "[CI] clang-tidy auto fixes"
+          title: "style: clang-tidy auto fixes"
           body: "Fixes #${{ steps.create_issue.outputs.created_issue_id }}. Please review and commit clang-tidy fixes."
           reviewers: "cindyyan317,godexsoft,kuznetsss"
 

.github/workflows/clang-tidy_on_fix_merged.yml

@@ -17,7 +17,7 @@ jobs:
         id: check
         shell: bash
         run: |
-          passed=$(if [[ $(git log -1 --pretty=format:%s | grep '\[CI\] clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi)
+          passed=$(if [[ $(git log -1 --pretty=format:%s | grep 'style: clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi)
           echo "passed=$passed" >> $GITHUB_OUTPUT
 
       - name: Run clang-tidy workflow

.github/workflows/docs.yml

@@ -34,7 +34,7 @@ jobs:
           cmake ../docs && cmake --build . --target docs
       
       - name: Setup Pages
-        uses: actions/configure-pages@v4
+        uses: actions/configure-pages@v5
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

.github/workflows/nightly.yml

@@ -3,6 +3,10 @@ on:
   schedule:
     - cron:  '0 5 * * 1-5'
   workflow_dispatch:
+  pull_request:
+    paths:
+      - '.github/workflows/nightly.yml'
+      - '.github/workflows/build_clio_docker_image.yml'
 
 jobs:
   build:
@@ -13,12 +17,15 @@ jobs:
         include:
           - os: macos14
             build_type: Release
+            static: false
           - os: heavy
             build_type: Release
+            static: true
             container:
               image: rippleci/clio_ci:latest
           - os: heavy
             build_type: Debug
+            static: true
             container:
               image: rippleci/clio_ci:latest
     runs-on: [self-hosted, "${{ matrix.os }}"]
@@ -50,6 +57,7 @@ jobs:
           conan_profile: ${{ steps.conan.outputs.conan_profile }}
           conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
           build_type: ${{ matrix.build_type }}
+          static: ${{ matrix.static }}
 
       - name: Build Clio
         uses: ./.github/actions/build_clio
@@ -130,6 +138,7 @@ jobs:
           ./clio_integration_tests --backend_host=scylladb
 
   nightly_release:
+    if: ${{ github.event_name != 'pull_request' }}
     needs: run_tests
     runs-on: ubuntu-20.04
     env:
@@ -143,13 +152,13 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           path: nightly_release
+          pattern: clio_server_*
 
       - name: Prepare files
         shell: bash
         run: |
           cp ${{ github.workspace }}/.github/workflows/nightly_notes.md "${RUNNER_TEMP}/nightly_notes.md"
           cd nightly_release
-          rm -r clio_*tests*
           for d in $(ls); do
             archive_name=$(ls $d)
             mv ${d}/${archive_name} ./
@@ -172,9 +181,21 @@ jobs:
             --target $GITHUB_SHA --notes-file "${RUNNER_TEMP}/nightly_notes.md" \
             ./nightly_release/clio_server*
 
+  build_and_publish_docker_image:
+    uses: ./.github/workflows/build_clio_docker_image.yml
+    needs: run_tests
+    secrets: inherit
+    with:
+      tags: |
+          type=raw,value=nightly
+          type=raw,value=${{ github.sha }}
+      artifact_name: clio_server_Linux_Release
+      strip_binary: true
+      publish_image: ${{ github.event_name != 'pull_request' }}
+
   create_issue_on_failure:
-    needs: [build, run_tests, nightly_release]
-    if: ${{ always() && contains(needs.*.result, 'failure') }}
+    needs: [build, run_tests, nightly_release, build_and_publish_docker_image]
+    if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name != 'pull_request' }}
     runs-on: ubuntu-20.04
     permissions:
       contents: write

.github/workflows/update_docker_ci.yml

@@ -18,30 +18,19 @@ jobs:
     name: Build and push docker image
     runs-on: [self-hosted, heavy]
     steps:
-      - name: Login to DockerHub
-        if: ${{ github.event_name != 'pull_request' }}
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PW }}
-
       - uses: actions/checkout@v4
-      - uses: docker/setup-qemu-action@v3
-      - uses: docker/setup-buildx-action@v3
-      - uses: docker/metadata-action@v5
-        id: meta
+      - uses: ./.github/actions/build_docker_image
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          images: rippleci/clio_ci
+          image_name: rippleci/clio_ci
+          push_image: ${{ github.event_name != 'pull_request' }}
+          directory: docker/ci
           tags: |
             type=raw,value=latest
             type=raw,value=gcc_12_clang_16
-            type=raw,value=${{ env.GITHUB_SHA }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: ${{ github.workspace }}/docker/ci
+            type=raw,value=${{ github.sha }}
           platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-
+          description: CI image for XRPLF/clio.

.github/workflows/upload_coverage_report.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Upload coverage report
         if: ${{ hashFiles('build/coverage_report.xml') != '' }}
-        uses: wandalen/wretry.action@v1.4.10
+        uses: wandalen/wretry.action@v3.7.2
         with:
           action: codecov/codecov-action@v4
           with: |

.gitignore

@@ -8,4 +8,4 @@
 .DS_Store
 CMakeUserPresets.json
 config.json
-src/main/impl/Build.cpp
+src/util/build/Build.cpp

CONTRIBUTING.md

@@ -21,7 +21,7 @@ git config --local core.hooksPath .githooks
 ```
 
 ## Git hooks dependencies
-The pre-commit hook requires `clang-format >= 18.0.0` and `cmake-format` to be installed on your machine.
+The pre-commit hook requires `clang-format >= 19.0.0` and `cmake-format` to be installed on your machine.
 `clang-format` can be installed using `brew` on macOS and default package manager on Linux.
 `cmake-format` can be installed using `pip`.
 The hook will also attempt to automatically use `doxygen` to verify that everything public in the codebase is covered by doc comments. If `doxygen` is not installed, the hook will raise a warning suggesting to install `doxygen` for future commits.
@@ -72,6 +72,9 @@ git push --force
 Clio uses `ccache` to speed up compilation. If you want to use it, please make sure it is installed on your machine.
 CMake will automatically detect it and use it if it is available.
 
+## Opening a pull request
+When a pull request is open CI will perform checks on the new code.
+Title of the pull request and squashed commit should follow [conventional commits specification](https://www.conventionalcommits.org/en/v1.0.0/).
 
 ## Fixing issues found during code review
 While your code is in review, it's possible that some changes will be requested by reviewer(s).
@@ -102,7 +105,7 @@ The button for that is near the bottom of the PR's page on GitHub.
 This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent.
 
 ## Formatting
-Code must conform to `clang-format` version 18, unless the result would be unreasonably difficult to read or maintain.
+Code must conform to `clang-format` version 19, unless the result would be unreasonably difficult to read or maintain.
 In most cases the pre-commit hook will take care of formatting and will fix any issues automatically.
 To manually format your code, use `clang-format -i <your changed files>` for C++ files and `cmake-format -i <your changed files>` for CMake files.
 

README.md

@@ -36,6 +36,7 @@ Below are some useful docs to learn more about Clio.
 - [How to configure Clio and rippled](./docs/configure-clio.md)
 - [How to run Clio](./docs/run-clio.md)
 - [Logging](./docs/logging.md)
+- [Troubleshooting guide](./docs/trouble_shooting.md)
 
 **General reference material:**
 

benchmarks/CMakeLists.txt

@@ -12,5 +12,5 @@ target_sources(
 include(deps/gbench)
 
 target_include_directories(clio_benchmark PRIVATE .)
-target_link_libraries(clio_benchmark PUBLIC clio benchmark::benchmark_main)
+target_link_libraries(clio_benchmark PUBLIC clio_etl benchmark::benchmark_main)
 set_target_properties(clio_benchmark PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

benchmarks/util/async/ExecutionContextBenchmarks.cpp

@@ -31,7 +31,6 @@
 #include <cstdint>
 #include <latch>
 #include <optional>
-#include <stdexcept>
 #include <thread>
 #include <vector>
 

cmake/Build.cpp.in

@@ -17,11 +17,12 @@
 */
 //==============================================================================
 
-#include "main/Build.hpp"
+#include "util/build/Build.hpp"
 
 #include <string>
 
-namespace Build {
+namespace util::build {
+
 static constexpr char versionString[] = "@CLIO_VERSION@";
 
 std::string const&
@@ -38,4 +39,4 @@ getClioFullVersionString()
     return value;
 }
 
-}  // namespace Build
+}  // namespace util::build

cmake/ClangTidy.cmake

@@ -8,7 +8,7 @@ if (lint)
     endif ()
     message(STATUS "Using clang-tidy from CLIO_CLANG_TIDY_BIN")
   else ()
-    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-18" "clang-tidy" REQUIRED)
+    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-19" "clang-tidy" REQUIRED)
   endif ()
 
   if (NOT _CLANG_TIDY_BIN)

cmake/ClioVersion.cmake

@@ -45,4 +45,4 @@ endif ()
 
 message(STATUS "Build version: ${CLIO_VERSION}")
 
-configure_file(${CMAKE_CURRENT_LIST_DIR}/Build.cpp.in ${CMAKE_CURRENT_LIST_DIR}/../src/main/impl/Build.cpp)
+configure_file(${CMAKE_CURRENT_LIST_DIR}/Build.cpp.in ${CMAKE_CURRENT_LIST_DIR}/../src/util/build/Build.cpp)

conanfile.py

@@ -28,7 +28,7 @@ class Clio(ConanFile):
         'protobuf/3.21.9',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.2.0-rc3',
+        'xrpl/2.3.0-rc1',
         'libbacktrace/cci.20210118'
     ]
 

docker/ci/README.md

@@ -0,0 +1,15 @@
+# CI image for XRPLF/clio
+
+This image contains an environment to build [Clio](https://github.com/XRPLF/clio), check code and documentation.
+It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but can also be used to compile Clio locally.
+
+The image is based on Ubuntu 20.04 and contains:
+- clang 16
+- gcc 12.3
+- doxygen 1.10
+- gh 2.40
+- ccache 4.8.3
+- conan
+- and some other useful tools
+
+Conan is set up to build Clio without any additional steps. There are two preset conan profiles: `clang` and `gcc` to use corresponding compiler.

docker/ci/dockerfile

@@ -6,10 +6,10 @@ SHELL ["/bin/bash", "-c"]
 USER root
 WORKDIR /root
 
-ENV CCACHE_VERSION=4.8.3 \
-    LLVM_TOOLS_VERSION=18 \
+ENV CCACHE_VERSION=4.10.2 \
+    LLVM_TOOLS_VERSION=19 \
     GH_VERSION=2.40.0 \
-    DOXYGEN_VERSION=1.10.0
+    DOXYGEN_VERSION=1.12.0
  
 # Add repositories
 RUN apt-get -qq update \

docker/clio/README.md

@@ -0,0 +1,23 @@
+# Clio official docker image
+
+[Clio](https://github.com/XRPLF/clio) is an XRP Ledger API server optimized for RPC calls over WebSocket or JSON-RPC.
+It stores validated historical ledger and transaction data in a space efficient format.
+
+This image contains `clio_server` binary allowing users to run Clio easily.
+
+## Clio configuration file
+
+Please note that while Clio requires a configuration file, this image doesn't include any default config.
+Your configuration file should be mounted under the path `/opt/clio/etc/config.json`.
+Clio repository provides an [example](https://github.com/XRPLF/clio/blob/develop/docs/examples/config/example-config.json) of the configuration file.
+
+Config file recommendations:
+- Set `log_to_console` to `false` if you want to avoid logs being written to `stdout`.
+- Set `log_directory` to `/opt/clio/log` to store logs in a volume.
+
+## Usage
+
+The following command can be used to run Clio in docker (assuming server's port is `51233` in your config):
+```bash
+docker run -d -v <path to your config.json>:/opt/clio/etc/config.json -v <path to store logs>:/opt/clio/log -p 51233:51233 rippleci/clio
+```

docker/clio/dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:22.04
+
+COPY ./clio_server /opt/clio/bin/clio_server
+
+RUN ln -s /opt/clio/bin/clio_server /usr/local/bin/clio_server && \
+    mkdir -p /opt/clio/etc/ && \
+    mkdir -p /opt/clio/log/ && \
+    groupadd -g 10001 clio && \
+    useradd -u 10000 -g 10001 -s /bin/bash clio && \
+    chown clio:clio /opt/clio/log && \
+    apt update && \
+    apt install -y libatomic1
+
+USER clio
+ENTRYPOINT ["/opt/clio/bin/clio_server"]
+CMD ["--conf", "/opt/clio/etc/config.json"]

docker/develop/compose.yaml

@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   clio_develop:
     image: rippleci/clio_ci:latest

docs/build-clio.md

@@ -36,6 +36,8 @@ compiler.version=15
 compiler.libcxx=libc++
 build_type=Release
 compiler.cppstd=20
+[conf]
+tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]
 ```
 
 > Linux example:
@@ -91,6 +93,9 @@ If successful, `conan install` will find the required packages and `cmake` will
 > [!TIP]
 > To generate a Code Coverage report, include `-o coverage=True` in the `conan install` command above, along with `-o tests=True` to enable tests. After running the `cmake` commands, execute `make clio_tests-ccov`. The coverage report will be found at `clio_tests-llvm-cov/index.html`.
 
+> [!NOTE]
+> If you've built Clio before and the build is now failing, it's likely due to updated dependencies. Try deleting the build folder and then rerunning the Conan and CMake commands mentioned above.
+
 ### Generating API docs for Clio
 
 The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen on your system.
@@ -136,3 +141,43 @@ If you wish to develop against a `rippled` instance running in standalone mode t
 
 1. Advance the `rippled` ledger to at least ledger 256.
 2. Wait 10 minutes before first starting Clio against this standalone node.
+
+## Building with a Custom `libxrpl`
+
+Sometimes, during development, you need to build against a custom version of `libxrpl`. (For example, you may be developing compatibility for a proposed amendment that is not yet merged to the main `rippled` codebase.) To build Clio with compatibility for a custom fork or branch of `rippled`, follow these steps:
+
+1. First, pull/clone the appropriate `rippled` fork and switch to the branch you want to build. For example, the following example uses an in-development build with [XLS-33d Multi-Purpose Tokens](https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens):
+
+    ```sh
+    git clone https://github.com/shawnxie999/rippled/
+    cd rippled
+    git switch mpt-1.1
+    ```
+
+2. Export a custom package to your local Conan store using a user/channel:
+
+    ```sh
+    conan export . my/feature
+    ```
+
+3. Patch your local Clio build to use the right package.
+
+    Edit `conanfile.py` (from the Clio repository root). Replace the `xrpl` requirement with the custom package version from the previous step. This must also include the current version number from your `rippled` branch. For example:
+
+    ```py
+    # ... (excerpt from conanfile.py)
+        requires = [
+        'boost/1.82.0',
+        'cassandra-cpp-driver/2.17.0',
+        'fmt/10.1.1',
+        'protobuf/3.21.9',
+        'grpc/1.50.1',
+        'openssl/1.1.1u',
+        'xrpl/2.3.0-b1@my/feature', # Update this line
+        'libbacktrace/cci.20210118'
+    ]
+    ```
+
+4. Build Clio as you would have before.
+
+    See [Building Clio](#building-clio) for details.

docs/configure-clio.md

@@ -27,7 +27,7 @@ If you're running Clio and `rippled` on separate machines, in addition to uncomm
 
 2. Open a public, unencrypted WebSocket port on your `rippled` server.
 
-3. In the `rippled` config, change the IP specified for `secure_gateway`, under the `port_grpc` section, to the IP of your Clio server. This entry can take the form of a comma-separated list if you are running multiple Clio nodes.
+3. In the `rippled` config, change the IP specified for `secure_gateway`, under the `port_grpc` and websocket server sections, to the IP of your Clio server. This entry can take the form of a comma-separated list if you are running multiple Clio nodes.
 
 ## Ledger sequence
 

docs/examples/config/example-config.json

@@ -31,11 +31,17 @@
     "etl_sources": [
         {
             "ip": "127.0.0.1",
-            "ws_port": "6006",
+            "ws_port": "6005",
             "grpc_port": "50051"
         }
     ],
-    "forwarding_cache_timeout": 0.250, // in seconds, could be 0, which means no cache
+    "forwarding": {
+        "cache_timeout": 0.250, // in seconds, could be 0, which means no cache
+        "request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds)
+    },
+    "rpc": {
+        "cache_timeout": 0.5 // in seconds, could be 0, which means no cache for rpc
+    },
     "dos_guard": {
         // Comma-separated list of IPs to exclude from rate limiting
         "whitelist": [
@@ -64,10 +70,17 @@
         "admin_password": "xrp",
         // If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests
         // It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time
-        "local_amdin": false
+        "local_admin": false,
+        "processing_policy": "parallel", // Could be "sequent" or "parallel".
+        // For sequent policy request from one client connection will be processed one by one and the next one will not be read before
+        // the previous one is processed. For parallel policy Clio will take all requests and process them in parallel and
+        // send a reply for each request whenever it is ready.
+        "parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified.
+        // Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection.
+        "ws_max_sending_queue_size": 1500
     },
     // Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet.
-    "graceful_period": 10.,
+    "graceful_period": 10.0,
     // Overrides log level on a per logging channel.
     // Defaults to global "log_level" for each unspecified channel.
     "log_channels": [

docs/metrics-and-static-analysis.md

@@ -14,7 +14,7 @@ You can find an example docker-compose file, with Prometheus and Grafana configs
 
 ## Using `clang-tidy` for static analysis
 
-The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 17.0.
+The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 19.0.
 
 Clang-tidy can be run by Cmake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command:
 
@@ -26,5 +26,5 @@ By default Cmake will try to find `clang-tidy` automatically in your system.
 To force Cmake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example:
 
 ```sh
-export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@17/bin/clang-tidy
+export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@19/bin/clang-tidy
 ```

docs/trouble_shooting.md

@@ -0,0 +1,47 @@
+# Troubleshooting Guide
+This guide will help you troubleshoot common issues of Clio.
+
+## Can't connect to DB
+If you see the error log message `Could not connect to Cassandra: No hosts available`, this means that Clio can't connect to the database. Check the following:
+- Make sure the database is running at the specified address and port.
+- Make sure the database is accessible from the machine where Clio is running.
+You can use [cqlsh](https://pypi.org/project/cqlsh/) to check the connection to the database.
+If you would like to run a local ScyllaDB, you can call:
+```sh
+docker run --rm -p 9042:9042 --name clio-scylla -d scylladb/scylla 
+```
+
+## Check the server status of Clio
+To check if Clio is syncing with rippled:
+```sh
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep seq
+```
+If Clio is syncing with rippled, the `seq` value will be increasing.
+
+## Clio fails to start
+If you see the error log message `Failed to fetch ETL state from...`, this means the configured rippled node is not reachable. Check the following:
+- Make sure the rippled node is running at the specified address and port.
+- Make sure the rippled node is accessible from the machine where Clio is running.
+
+If you would like to run Clio without an avaliable rippled node, you can add below setting to Clio's configuration file:
+```
+"allow_no_etl": true
+```
+
+## Clio is not added to secure_gateway in rippled's config
+If you see the warning message `AsyncCallData is_unlimited is false.`, this means that Clio is not added to the `secure_gateway` of `port_grpc` session in the rippled configuration file. It will slow down the sync process. Please add Clio's IP to the `secure_gateway` in the rippled configuration file for both grpc and ws port.
+
+## Clio is slow
+To speed up the response time, Clio has a cache inside. However, cache can take time to warm up. If you see slow response time, you can firstly check if cache is still loading.
+You can check the cache status by calling:
+```sh
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_full
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_enabled
+```
+If `is_full` is false, it means the cache is still loading. Normally, the Clio can respond quicker if cache finishs loading. If `is_enabled` is false, it means the cache is disabled in the configuration file or there is data corruption in the database. 
+
+## Receive error message `Too many requests`
+If client sees the error message `Too many requests`, this means that the client is blocked by Clio's DosGuard protection. You may want to add the client's IP to the whitelist in the configuration file, Or update other your DosGuard settings.
+
+
+

src/CMakeLists.txt

@@ -1,7 +1,9 @@
 add_subdirectory(util)
 add_subdirectory(data)
 add_subdirectory(etl)
+add_subdirectory(etlng)
 add_subdirectory(feed)
 add_subdirectory(rpc)
 add_subdirectory(web)
+add_subdirectory(app)
 add_subdirectory(main)

src/app/CMakeLists.txt

@@ -0,0 +1,4 @@
+add_library(clio_app)
+target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp)
+
+target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc)

src/app/CliArgs.cpp

@@ -0,0 +1,70 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "app/CliArgs.hpp"
+
+#include "util/build/Build.hpp"
+
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/program_options/positional_options.hpp>
+#include <boost/program_options/value_semantic.hpp>
+#include <boost/program_options/variables_map.hpp>
+
+#include <cstdlib>
+#include <iostream>
+#include <string>
+#include <utility>
+
+namespace app {
+
+CliArgs::Action
+CliArgs::parse(int argc, char const* argv[])
+{
+    namespace po = boost::program_options;
+    // clang-format off
+    po::options_description description("Options");
+    description.add_options()
+        ("help,h", "print help message and exit")
+        ("version,v", "print version and exit")
+        ("conf,c", po::value<std::string>()->default_value(defaultConfigPath), "configuration file")
+    ;
+    // clang-format on
+    po::positional_options_description positional;
+    positional.add("conf", 1);
+
+    po::variables_map parsed;
+    po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
+    po::notify(parsed);
+
+    if (parsed.count("help") != 0u) {
+        std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n" << description;
+        return Action{Action::Exit{EXIT_SUCCESS}};
+    }
+
+    if (parsed.count("version") != 0u) {
+        std::cout << util::build::getClioFullVersionString() << '\n';
+        return Action{Action::Exit{EXIT_SUCCESS}};
+    }
+
+    auto configPath = parsed["conf"].as<std::string>();
+    return Action{Action::Run{std::move(configPath)}};
+}
+
+}  // namespace app

src/app/CliArgs.hpp

@@ -0,0 +1,96 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/OverloadSet.hpp"
+
+#include <string>
+#include <variant>
+
+namespace app {
+
+/**
+ * @brief Parsed command line arguments representation.
+ */
+class CliArgs {
+public:
+    /**
+     * @brief Default configuration path.
+     */
+    static constexpr char defaultConfigPath[] = "/etc/opt/clio/config.json";
+
+    /**
+     * @brief An action parsed from the command line.
+     */
+    class Action {
+    public:
+        /** @brief Run action. */
+        struct Run {
+            /** @brief Configuration file path. */
+            std::string configPath;
+        };
+
+        /** @brief Exit action. */
+        struct Exit {
+            /** @brief Exit code. */
+            int exitCode;
+        };
+
+        /**
+         * @brief Construct an action from a Run.
+         *
+         * @param action Run action.
+         */
+        template <typename ActionType>
+            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit>
+        explicit Action(ActionType&& action) : action_(std::forward<ActionType>(action))
+        {
+        }
+
+        /**
+         * @brief Apply a function to the action.
+         *
+         * @tparam Processors Action processors types. Must be callable with the action type and return int.
+         * @param processors Action processors.
+         * @return Exit code.
+         */
+        template <typename... Processors>
+        int
+        apply(Processors&&... processors) const
+        {
+            return std::visit(util::OverloadSet{std::forward<Processors>(processors)...}, action_);
+        }
+
+    private:
+        std::variant<Run, Exit> action_;
+    };
+
+    /**
+     * @brief Parse command line arguments.
+     *
+     * @param argc Number of arguments.
+     * @param argv Array of arguments.
+     * @return Parsed command line arguments.
+     */
+    static Action
+    parse(int argc, char const* argv[]);
+};
+
+}  // namespace app

src/app/ClioApplication.cpp

@@ -0,0 +1,142 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "app/ClioApplication.hpp"
+
+#include "data/AmendmentCenter.hpp"
+#include "data/BackendFactory.hpp"
+#include "etl/ETLService.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/NetworkValidatedLedgers.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "rpc/Counters.hpp"
+#include "rpc/RPCEngine.hpp"
+#include "rpc/WorkQueue.hpp"
+#include "rpc/common/impl/HandlerProvider.hpp"
+#include "util/build/Build.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
+#include "util/prometheus/Prometheus.hpp"
+#include "web/RPCServerHandler.hpp"
+#include "web/Server.hpp"
+#include "web/dosguard/DOSGuard.hpp"
+#include "web/dosguard/IntervalSweepHandler.hpp"
+#include "web/dosguard/WhitelistHandler.hpp"
+
+#include <boost/asio/io_context.hpp>
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <thread>
+#include <vector>
+
+namespace app {
+
+namespace {
+
+/**
+ * @brief Start context threads
+ *
+ * @param ioc Context
+ * @param numThreads Number of worker threads to start
+ */
+void
+start(boost::asio::io_context& ioc, std::uint32_t numThreads)
+{
+    std::vector<std::thread> v;
+    v.reserve(numThreads - 1);
+    for (auto i = numThreads - 1; i > 0; --i)
+        v.emplace_back([&ioc] { ioc.run(); });
+
+    ioc.run();
+    for (auto& t : v)
+        t.join();
+}
+
+}  // namespace
+
+ClioApplication::ClioApplication(util::Config const& config) : config_(config), signalsHandler_{config_}
+{
+    LOG(util::LogService::info()) << "Clio version: " << util::build::getClioFullVersionString();
+    PrometheusService::init(config);
+}
+
+int
+ClioApplication::run()
+{
+    auto const threads = config_.valueOr("io_threads", 2);
+    if (threads <= 0) {
+        LOG(util::LogService::fatal()) << "io_threads is less than 1";
+        return EXIT_FAILURE;
+    }
+    LOG(util::LogService::info()) << "Number of io threads = " << threads;
+
+    // IO context to handle all incoming requests, as well as other things.
+    // This is not the only io context in the application.
+    boost::asio::io_context ioc{threads};
+
+    // Rate limiter, to prevent abuse
+    auto whitelistHandler = web::dosguard::WhitelistHandler{config_};
+    auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler};
+    auto sweepHandler = web::dosguard::IntervalSweepHandler{config_, ioc, dosGuard};
+
+    // Interface to the database
+    auto backend = data::make_Backend(config_);
+
+    // Manages clients subscribed to streams
+    auto subscriptions = feed::SubscriptionManager::make_SubscriptionManager(config_, backend);
+
+    // Tracks which ledgers have been validated by the network
+    auto ledgers = etl::NetworkValidatedLedgers::make_ValidatedLedgers();
+
+    // Handles the connection to one or more rippled nodes.
+    // ETL uses the balancer to extract data.
+    // The server uses the balancer to forward RPCs to a rippled node.
+    // The balancer itself publishes to streams (transactions_proposed and accounts_proposed)
+    auto balancer = etl::LoadBalancer::make_LoadBalancer(config_, ioc, backend, subscriptions, ledgers);
+
+    // ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes
+    auto etl = etl::ETLService::make_ETLService(config_, ioc, backend, subscriptions, balancer, ledgers);
+
+    auto workQueue = rpc::WorkQueue::make_WorkQueue(config_);
+    auto counters = rpc::Counters::make_Counters(workQueue);
+    auto const amendmentCenter = std::make_shared<data::AmendmentCenter const>(backend);
+    auto const handlerProvider = std::make_shared<rpc::impl::ProductionHandlerProvider const>(
+        config_, backend, subscriptions, balancer, etl, amendmentCenter, counters
+    );
+
+    using RPCEngineType = rpc::RPCEngine<etl::LoadBalancer, rpc::Counters>;
+    auto const rpcEngine =
+        RPCEngineType::make_RPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
+
+    // Init the web server
+    auto handler =
+        std::make_shared<web::RPCServerHandler<RPCEngineType, etl::ETLService>>(config_, backend, rpcEngine, etl);
+    auto const httpServer = web::make_HttpServer(config_, ioc, dosGuard, handler);
+
+    // Blocks until stopped.
+    // When stopped, shared_ptrs fall out of scope
+    // Calls destructors on all resources, and destructs in order
+    start(ioc, threads);
+
+    return EXIT_SUCCESS;
+}
+
+}  // namespace app

src/app/ClioApplication.hpp

@@ -0,0 +1,51 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/SignalsHandler.hpp"
+#include "util/config//Config.hpp"
+
+namespace app {
+
+/**
+ * @brief The main application class
+ */
+class ClioApplication {
+    util::Config const& config_;
+    util::SignalsHandler signalsHandler_;
+
+public:
+    /**
+     * @brief Construct a new ClioApplication object
+     *
+     * @param config The configuration of the application
+     */
+    ClioApplication(util::Config const& config);
+
+    /**
+     * @brief Run the application
+     *
+     * @return exit code
+     */
+    int
+    run();
+};
+
+}  // namespace app

src/data/AmendmentCenter.cpp

@@ -0,0 +1,203 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/AmendmentCenter.hpp"
+
+#include "data/BackendInterface.hpp"
+#include "data/Types.hpp"
+#include "util/Assert.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/Feature.h>
+#include <xrpl/protocol/Indexes.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/digest.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <iterator>
+#include <map>
+#include <memory>
+#include <optional>
+#include <ranges>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace {
+
+std::unordered_set<std::string>&
+SUPPORTED_AMENDMENTS()
+{
+    static std::unordered_set<std::string> amendments = {};
+    return amendments;
+}
+
+bool
+lookupAmendment(auto const& allAmendments, std::vector<ripple::uint256> const& ledgerAmendments, std::string_view name)
+{
+    namespace rg = std::ranges;
+    if (auto const am = rg::find(allAmendments, name, &data::Amendment::name); am != rg::end(allAmendments))
+        return rg::find(ledgerAmendments, am->feature) != rg::end(ledgerAmendments);
+    return false;
+}
+
+}  // namespace
+
+namespace data {
+namespace impl {
+
+WritingAmendmentKey::WritingAmendmentKey(std::string amendmentName) : AmendmentKey{std::move(amendmentName)}
+{
+    ASSERT(not SUPPORTED_AMENDMENTS().contains(name), "Attempt to register the same amendment twice");
+    SUPPORTED_AMENDMENTS().insert(name);
+}
+
+}  // namespace impl
+
+AmendmentKey::operator std::string const&() const
+{
+    return name;
+}
+
+AmendmentKey::operator std::string_view() const
+{
+    return name;
+}
+
+AmendmentKey::operator ripple::uint256() const
+{
+    return Amendment::GetAmendmentId(name);
+}
+
+AmendmentCenter::AmendmentCenter(std::shared_ptr<data::BackendInterface> const& backend) : backend_{backend}
+{
+    namespace rg = std::ranges;
+    namespace vs = std::views;
+
+    rg::copy(
+        ripple::allAmendments() | vs::transform([&](auto const& p) {
+            auto const& [name, support] = p;
+            return Amendment{
+                .name = name,
+                .feature = Amendment::GetAmendmentId(name),
+                .isSupportedByXRPL = support != ripple::AmendmentSupport::Unsupported,
+                .isSupportedByClio = rg::find(SUPPORTED_AMENDMENTS(), name) != rg::end(SUPPORTED_AMENDMENTS()),
+                .isRetired = support == ripple::AmendmentSupport::Retired
+            };
+        }),
+        std::back_inserter(all_)
+    );
+
+    for (auto const& am : all_ | vs::filter([](auto const& am) { return am.isSupportedByClio; }))
+        supported_.insert_or_assign(am.name, am);
+}
+
+bool
+AmendmentCenter::isSupported(AmendmentKey const& key) const
+{
+    return supported_.contains(key);
+}
+
+std::map<std::string, Amendment> const&
+AmendmentCenter::getSupported() const
+{
+    return supported_;
+}
+
+std::vector<Amendment> const&
+AmendmentCenter::getAll() const
+{
+    return all_;
+}
+
+bool
+AmendmentCenter::isEnabled(AmendmentKey const& key, uint32_t seq) const
+{
+    return data::synchronous([this, &key, seq](auto yield) { return isEnabled(yield, key, seq); });
+}
+
+bool
+AmendmentCenter::isEnabled(boost::asio::yield_context yield, AmendmentKey const& key, uint32_t seq) const
+{
+    if (auto const listAmendments = fetchAmendmentsList(yield, seq); listAmendments)
+        return lookupAmendment(all_, *listAmendments, key);
+
+    return false;
+}
+
+std::vector<bool>
+AmendmentCenter::isEnabled(boost::asio::yield_context yield, std::vector<AmendmentKey> const& keys, uint32_t seq) const
+{
+    namespace rg = std::ranges;
+
+    if (auto const listAmendments = fetchAmendmentsList(yield, seq); listAmendments) {
+        std::vector<bool> out;
+        rg::transform(keys, std::back_inserter(out), [this, &listAmendments](auto const& key) {
+            return lookupAmendment(all_, *listAmendments, key);
+        });
+
+        return out;
+    }
+
+    return std::vector<bool>(keys.size(), false);
+}
+
+Amendment const&
+AmendmentCenter::getAmendment(AmendmentKey const& key) const
+{
+    ASSERT(supported_.contains(key), "The amendment '{}' must be present in supported amendments list", key.name);
+    return supported_.at(key);
+}
+
+Amendment const&
+AmendmentCenter::operator[](AmendmentKey const& key) const
+{
+    return getAmendment(key);
+}
+
+ripple::uint256
+Amendment::GetAmendmentId(std::string_view name)
+{
+    return ripple::sha512Half(ripple::Slice(name.data(), name.size()));
+}
+
+std::optional<std::vector<ripple::uint256>>
+AmendmentCenter::fetchAmendmentsList(boost::asio::yield_context yield, uint32_t seq) const
+{
+    // the amendments should always be present on the ledger
+    auto const amendments = backend_->fetchLedgerObject(ripple::keylet::amendments().key, seq, yield);
+    if (not amendments.has_value())
+        throw std::runtime_error("Amendments ledger object must be present in the database");
+
+    ripple::SLE const amendmentsSLE{
+        ripple::SerialIter{amendments->data(), amendments->size()}, ripple::keylet::amendments().key
+    };
+
+    return amendmentsSLE[~ripple::sfAmendments];
+}
+
+}  // namespace data

src/data/AmendmentCenter.hpp

@@ -0,0 +1,259 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/AmendmentCenterInterface.hpp"
+#include "data/BackendInterface.hpp"
+#include "data/Types.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/preprocessor.hpp>
+#include <boost/preprocessor/seq/for_each.hpp>
+#include <boost/preprocessor/stringize.hpp>
+#include <boost/preprocessor/variadic/to_seq.hpp>
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/Feature.h>
+#include <xrpl/protocol/Indexes.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/digest.h>
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#define REGISTER(name)                                   \
+    inline static impl::WritingAmendmentKey const name = \
+        impl::WritingAmendmentKey(std::string(BOOST_PP_STRINGIZE(name)))
+
+namespace data {
+namespace impl {
+
+struct WritingAmendmentKey : AmendmentKey {
+    explicit WritingAmendmentKey(std::string amendmentName);
+};
+
+}  // namespace impl
+
+/**
+ * @brief List of supported amendments
+ */
+struct Amendments {
+    // NOTE: if Clio wants to report it supports an Amendment it should be listed here.
+    // Whether an amendment is obsolete and/or supported by libxrpl is extracted directly from libxrpl.
+    // If an amendment is in the list below it just means Clio did whatever changes needed to support it.
+    // Most of the time it's going to be no changes at all.
+
+    /** @cond */
+    REGISTER(OwnerPaysFee);
+    REGISTER(Flow);
+    REGISTER(FlowCross);
+    REGISTER(fix1513);
+    REGISTER(DepositAuth);
+    REGISTER(Checks);
+    REGISTER(fix1571);
+    REGISTER(fix1543);
+    REGISTER(fix1623);
+    REGISTER(DepositPreauth);
+    REGISTER(fix1515);
+    REGISTER(fix1578);
+    REGISTER(MultiSignReserve);
+    REGISTER(fixTakerDryOfferRemoval);
+    REGISTER(fixMasterKeyAsRegularKey);
+    REGISTER(fixCheckThreading);
+    REGISTER(fixPayChanRecipientOwnerDir);
+    REGISTER(DeletableAccounts);
+    REGISTER(fixQualityUpperBound);
+    REGISTER(RequireFullyCanonicalSig);
+    REGISTER(fix1781);
+    REGISTER(HardenedValidations);
+    REGISTER(fixAmendmentMajorityCalc);
+    REGISTER(NegativeUNL);
+    REGISTER(TicketBatch);
+    REGISTER(FlowSortStrands);
+    REGISTER(fixSTAmountCanonicalize);
+    REGISTER(fixRmSmallIncreasedQOffers);
+    REGISTER(CheckCashMakesTrustLine);
+    REGISTER(ExpandedSignerList);
+    REGISTER(NonFungibleTokensV1_1);
+    REGISTER(fixTrustLinesToSelf);
+    REGISTER(fixRemoveNFTokenAutoTrustLine);
+    REGISTER(ImmediateOfferKilled);
+    REGISTER(DisallowIncoming);
+    REGISTER(XRPFees);
+    REGISTER(fixUniversalNumber);
+    REGISTER(fixNonFungibleTokensV1_2);
+    REGISTER(fixNFTokenRemint);
+    REGISTER(fixReducedOffersV1);
+    REGISTER(Clawback);
+    REGISTER(AMM);
+    REGISTER(XChainBridge);
+    REGISTER(fixDisallowIncomingV1);
+    REGISTER(DID);
+    REGISTER(fixFillOrKill);
+    REGISTER(fixNFTokenReserve);
+    REGISTER(fixInnerObjTemplate);
+    REGISTER(fixAMMOverflowOffer);
+    REGISTER(PriceOracle);
+    REGISTER(fixEmptyDID);
+    REGISTER(fixXChainRewardRounding);
+    REGISTER(fixPreviousTxnID);
+    REGISTER(fixAMMv1_1);
+    REGISTER(NFTokenMintOffer);
+    REGISTER(fixReducedOffersV2);
+    REGISTER(fixEnforceNFTokenTrustline);
+    REGISTER(fixInnerObjTemplate2);
+    REGISTER(fixNFTokenPageLinks);
+    REGISTER(InvariantsV1_1);
+    REGISTER(MPTokensV1);
+    REGISTER(fixAMMv1_2);
+    REGISTER(AMMClawback);
+    REGISTER(Credentials);
+
+    // Obsolete but supported by libxrpl
+    REGISTER(CryptoConditionsSuite);
+    REGISTER(NonFungibleTokensV1);
+    REGISTER(fixNFTokenDirV1);
+    REGISTER(fixNFTokenNegOffer);
+
+    // Retired amendments
+    REGISTER(MultiSign);
+    REGISTER(TrustSetAuth);
+    REGISTER(FeeEscalation);
+    REGISTER(PayChan);
+    REGISTER(fix1368);
+    REGISTER(CryptoConditions);
+    REGISTER(Escrow);
+    REGISTER(TickSize);
+    REGISTER(fix1373);
+    REGISTER(EnforceInvariants);
+    REGISTER(SortedDirectories);
+    REGISTER(fix1201);
+    REGISTER(fix1512);
+    REGISTER(fix1523);
+    REGISTER(fix1528);
+    /** @endcond */
+};
+
+#undef REGISTER
+
+/**
+ * @brief Knowledge center for amendments within XRPL
+ */
+class AmendmentCenter : public AmendmentCenterInterface {
+    std::shared_ptr<data::BackendInterface> backend_;
+
+    std::map<std::string, Amendment> supported_;
+    std::vector<Amendment> all_;
+
+public:
+    /**
+     * @brief Construct a new AmendmentCenter instance
+     *
+     * @param backend The backend
+     */
+    explicit AmendmentCenter(std::shared_ptr<data::BackendInterface> const& backend);
+
+    /**
+     * @brief Check whether an amendment is supported by Clio
+     *
+     * @param key The key of the amendment to check
+     * @return true if supported; false otherwise
+     */
+    [[nodiscard]] bool
+    isSupported(AmendmentKey const& key) const final;
+
+    /**
+     * @brief Get all supported amendments as a map
+     *
+     * @return The amendments supported by Clio
+     */
+    [[nodiscard]] std::map<std::string, Amendment> const&
+    getSupported() const final;
+
+    /**
+     * @brief Get all known amendments
+     *
+     * @return All known amendments as a vector
+     */
+    [[nodiscard]] std::vector<Amendment> const&
+    getAll() const final;
+
+    /**
+     * @brief Check whether an amendment was/is enabled for a given sequence
+     *
+     * @param key The key of the amendment to check
+     * @param seq The sequence to check for
+     * @return true if enabled; false otherwise
+     */
+    [[nodiscard]] bool
+    isEnabled(AmendmentKey const& key, uint32_t seq) const final;
+
+    /**
+     * @brief Check whether an amendment was/is enabled for a given sequence
+     *
+     * @param yield The coroutine context to use
+     * @param key The key of the amendment to check
+     * @param seq The sequence to check for
+     * @return true if enabled; false otherwise
+     */
+    [[nodiscard]] bool
+    isEnabled(boost::asio::yield_context yield, AmendmentKey const& key, uint32_t seq) const final;
+
+    /**
+     * @brief Check whether an amendment was/is enabled for a given sequence
+     *
+     * @param yield The coroutine context to use
+     * @param keys The keys of the amendments to check
+     * @param seq The sequence to check for
+     * @return A vector of bools representing enabled state for each of the given keys
+     */
+    [[nodiscard]] std::vector<bool>
+    isEnabled(boost::asio::yield_context yield, std::vector<AmendmentKey> const& keys, uint32_t seq) const final;
+
+    /**
+     * @brief Get an amendment
+     *
+     * @param key The key of the amendment to get
+     * @return The amendment as a const ref; asserts if the amendment is unknown
+     */
+    [[nodiscard]] Amendment const&
+    getAmendment(AmendmentKey const& key) const final;
+
+    /**
+     * @brief Get an amendment by its key
+
+     * @param key The amendment key from @see Amendments
+     * @return The amendment as a const ref; asserts if the amendment is unknown
+     */
+    [[nodiscard]] Amendment const&
+    operator[](AmendmentKey const& key) const final;
+
+private:
+    [[nodiscard]] std::optional<std::vector<ripple::uint256>>
+    fetchAmendmentsList(boost::asio::yield_context yield, uint32_t seq) const;
+};
+
+}  // namespace data

src/data/AmendmentCenterInterface.hpp

@@ -0,0 +1,116 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/Types.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <cstdint>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace data {
+
+/**
+ * @brief The interface of an amendment center
+ */
+class AmendmentCenterInterface {
+public:
+    virtual ~AmendmentCenterInterface() = default;
+
+    /**
+     * @brief Check whether an amendment is supported by Clio
+     *
+     * @param key The key of the amendment to check
+     * @return true if supported; false otherwise
+     */
+    [[nodiscard]] virtual bool
+    isSupported(AmendmentKey const& key) const = 0;
+
+    /**
+     * @brief Get all supported amendments as a map
+     *
+     * @return The amendments supported by Clio
+     */
+    [[nodiscard]] virtual std::map<std::string, Amendment> const&
+    getSupported() const = 0;
+
+    /**
+     * @brief Get all known amendments
+     *
+     * @return All known amendments as a vector
+     */
+    [[nodiscard]] virtual std::vector<Amendment> const&
+    getAll() const = 0;
+
+    /**
+     * @brief Check whether an amendment was/is enabled for a given sequence
+     *
+     * @param key The key of the amendment to check
+     * @param seq The sequence to check for
+     * @return true if enabled; false otherwise
+     */
+    [[nodiscard]] virtual bool
+    isEnabled(AmendmentKey const& key, uint32_t seq) const = 0;
+
+    /**
+     * @brief Check whether an amendment was/is enabled for a given sequence
+     *
+     * @param yield The coroutine context to use
+     * @param key The key of the amendment to check
+     * @param seq The sequence to check for
+     * @return true if enabled; false otherwise
+     */
+    [[nodiscard]] virtual bool
+    isEnabled(boost::asio::yield_context yield, AmendmentKey const& key, uint32_t seq) const = 0;
+
+    /**
+     * @brief Check whether an amendment was/is enabled for a given sequence
+     *
+     * @param yield The coroutine context to use
+     * @param keys The keys of the amendments to check
+     * @param seq The sequence to check for
+     * @return A vector of bools representing enabled state for each of the given keys
+     */
+    [[nodiscard]] virtual std::vector<bool>
+    isEnabled(boost::asio::yield_context yield, std::vector<AmendmentKey> const& keys, uint32_t seq) const = 0;
+
+    /**
+     * @brief Get an amendment
+     *
+     * @param key The key of the amendment to get
+     * @return The amendment as a const ref; asserts if the amendment is unknown
+     */
+    [[nodiscard]] virtual Amendment const&
+    getAmendment(AmendmentKey const& key) const = 0;
+
+    /**
+     * @brief Get an amendment by its key
+     *
+     * @param key The amendment key from @see Amendments
+     * @return The amendment as a const ref; asserts if the amendment is unknown
+     */
+    [[nodiscard]] virtual Amendment const&
+    operator[](AmendmentKey const& key) const = 0;
+};
+
+}  // namespace data

src/data/BackendFactory.hpp

@@ -51,8 +51,7 @@ make_Backend(util::Config const& config)
     auto const type = config.value<std::string>("database.type");
     std::shared_ptr<BackendInterface> backend = nullptr;
 
-    // TODO: retire `cassandra-new` by next release after 2.0
-    if (boost::iequals(type, "cassandra") or boost::iequals(type, "cassandra-new")) {
+    if (boost::iequals(type, "cassandra")) {
         auto cfg = config.section("database." + type);
         backend = std::make_shared<data::cassandra::CassandraBackend>(data::cassandra::SettingsProvider{cfg}, readOnly);
     }

src/data/BackendInterface.cpp

@@ -24,13 +24,13 @@
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/spawn.hpp>
-#include <ripple/basics/base_uint.h>
-#include <ripple/basics/strHex.h>
-#include <ripple/protocol/Fees.h>
-#include <ripple/protocol/Indexes.h>
-#include <ripple/protocol/SField.h>
-#include <ripple/protocol/STLedgerEntry.h>
-#include <ripple/protocol/Serializer.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/protocol/Fees.h>
+#include <xrpl/protocol/Indexes.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/Serializer.h>
 
 #include <chrono>
 #include <cstddef>
@@ -93,7 +93,6 @@ BackendInterface::fetchLedgerObject(
         return obj;
     }
 
-    LOG(gLog.trace()) << "Cache miss - " << ripple::strHex(key);
     auto dbObj = doFetchLedgerObject(key, sequence, yield);
     if (!dbObj) {
         LOG(gLog.trace()) << "Missed cache and missed in db";
@@ -103,6 +102,19 @@ BackendInterface::fetchLedgerObject(
     return dbObj;
 }
 
+std::optional<std::uint32_t>
+BackendInterface::fetchLedgerObjectSeq(
+    ripple::uint256 const& key,
+    std::uint32_t const sequence,
+    boost::asio::yield_context yield
+) const
+{
+    auto seq = doFetchLedgerObjectSeq(key, sequence, yield);
+    if (!seq)
+        LOG(gLog.trace()) << "Missed in db";
+    return seq;
+}
+
 std::vector<Blob>
 BackendInterface::fetchLedgerObjects(
     std::vector<ripple::uint256> const& keys,

src/data/BackendInterface.hpp

@@ -31,10 +31,10 @@
 #include <boost/json.hpp>
 #include <boost/json/object.hpp>
 #include <boost/utility/result_of.hpp>
-#include <ripple/basics/base_uint.h>
-#include <ripple/protocol/AccountID.h>
-#include <ripple/protocol/Fees.h>
-#include <ripple/protocol/LedgerHeader.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/AccountID.h>
+#include <xrpl/protocol/Fees.h>
+#include <xrpl/protocol/LedgerHeader.h>
 
 #include <chrono>
 #include <cstddef>
@@ -364,6 +364,25 @@ public:
         boost::asio::yield_context yield
     ) const = 0;
 
+    /**
+     * @brief Fetches all holders' balances for a MPTIssuanceID
+     *
+     * @param mptID MPTIssuanceID you wish you query.
+     * @param limit Paging limit.
+     * @param cursorIn Optional cursor to allow us to pick up from where we last left off.
+     * @param ledgerSequence The ledger sequence to fetch for
+     * @param yield Currently executing coroutine.
+     * @return std::vector<Blob> of MPToken balances and an optional marker
+     */
+    virtual MPTHoldersAndCursor
+    fetchMPTHolders(
+        ripple::uint192 const& mptID,
+        std::uint32_t const limit,
+        std::optional<ripple::AccountID> const& cursorIn,
+        std::uint32_t const ledgerSequence,
+        boost::asio::yield_context yield
+    ) const = 0;
+
     /**
      * @brief Fetches a specific ledger object.
      *
@@ -378,6 +397,19 @@ public:
     std::optional<Blob>
     fetchLedgerObject(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield) const;
 
+    /**
+     * @brief Fetches a specific ledger object sequence.
+     *
+     * Currently the real fetch happens in doFetchLedgerObjectSeq
+     *
+     * @param key The key of the object
+     * @param sequence The ledger sequence to fetch for
+     * @param yield The coroutine context
+     * @return The sequence in unit32_t on success; nullopt otherwise
+     */
+    std::optional<std::uint32_t>
+    fetchLedgerObjectSeq(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield) const;
+
     /**
      * @brief Fetches all ledger objects by their keys.
      *
@@ -407,6 +439,18 @@ public:
     virtual std::optional<Blob>
     doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield) const = 0;
 
+    /**
+     * @brief The database-specific implementation for fetching a ledger object sequence.
+     *
+     * @param key The key to fetch for
+     * @param sequence The ledger sequence to fetch for
+     * @param yield The coroutine context
+     * @return The sequence in unit32_t on success; nullopt otherwise
+     */
+    virtual std::optional<std::uint32_t>
+    doFetchLedgerObjectSeq(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield)
+        const = 0;
+
     /**
      * @brief The database-specific implementation for fetching ledger objects.
      *
@@ -592,6 +636,14 @@ public:
     virtual void
     writeNFTTransactions(std::vector<NFTTransactionsData> const& data) = 0;
 
+    /**
+     * @brief Write accounts that started holding onto a MPT.
+     *
+     * @param data A vector of MPT ID and account pairs
+     */
+    virtual void
+    writeMPTHolders(std::vector<MPTHolderData> const& data) = 0;
+
     /**
      * @brief Write a new successor.
      *

src/data/CMakeLists.txt

@@ -1,7 +1,8 @@
 add_library(clio_data)
 target_sources(
   clio_data
-  PRIVATE BackendCounters.cpp
+  PRIVATE AmendmentCenter.cpp
+          BackendCounters.cpp
           BackendInterface.cpp
           LedgerCache.cpp
           cassandra/impl/Future.cpp

src/data/CassandraBackend.hpp

@@ -22,7 +22,6 @@
 #include "data/BackendInterface.hpp"
 #include "data/DBHelpers.hpp"
 #include "data/Types.hpp"
-#include "data/cassandra/Concepts.hpp"
 #include "data/cassandra/Handle.hpp"
 #include "data/cassandra/Schema.hpp"
 #include "data/cassandra/SettingsProvider.hpp"
@@ -36,13 +35,13 @@
 #include <boost/asio/spawn.hpp>
 #include <boost/json/object.hpp>
 #include <cassandra.h>
-#include <ripple/basics/Blob.h>
-#include <ripple/basics/base_uint.h>
-#include <ripple/basics/strHex.h>
-#include <ripple/protocol/AccountID.h>
-#include <ripple/protocol/Indexes.h>
-#include <ripple/protocol/LedgerHeader.h>
-#include <ripple/protocol/nft.h>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/protocol/AccountID.h>
+#include <xrpl/protocol/Indexes.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/nft.h>
 
 #include <atomic>
 #include <chrono>
@@ -94,7 +93,7 @@ public:
         , executor_{settingsProvider_.getSettings(), handle_}
     {
         if (auto const res = handle_.connect(); not res)
-            throw std::runtime_error("Could not connect to Cassandra: " + res.error());
+            throw std::runtime_error("Could not connect to databse: " + res.error());
 
         if (not readOnly) {
             if (auto const res = handle_.execute(schema_.createKeyspace); not res) {
@@ -206,13 +205,13 @@ public:
     }
 
     void
-    writeLedger(ripple::LedgerHeader const& ledgerInfo, std::string&& blob) override
+    writeLedger(ripple::LedgerHeader const& ledgerHeader, std::string&& blob) override
     {
-        executor_.write(schema_->insertLedgerHeader, ledgerInfo.seq, std::move(blob));
+        executor_.write(schema_->insertLedgerHeader, ledgerHeader.seq, std::move(blob));
 
-        executor_.write(schema_->insertLedgerHash, ledgerInfo.hash, ledgerInfo.seq);
+        executor_.write(schema_->insertLedgerHash, ledgerHeader.hash, ledgerHeader.seq);
 
-        ledgerSequence_ = ledgerInfo.seq;
+        ledgerSequence_ = ledgerHeader.seq;
     }
 
     std::optional<std::uint32_t>
@@ -336,8 +335,8 @@ public:
 
         auto const& result = res.value();
         if (not result.hasRows()) {
-            LOG(log_.error()) << "Could not fetch all transaction hashes - no rows; ledger = "
-                              << std::to_string(ledgerSequence);
+            LOG(log_.warn()) << "Could not fetch all transaction hashes - no rows; ledger = "
+                             << std::to_string(ledgerSequence);
             return {};
         }
 
@@ -346,7 +345,7 @@ public:
             hashes.push_back(std::move(hash));
 
         auto end = std::chrono::system_clock::now();
-        LOG(log_.debug()) << "Fetched " << hashes.size() << " transaction hashes from Cassandra in "
+        LOG(log_.debug()) << "Fetched " << hashes.size() << " transaction hashes from database in "
                           << std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count()
                           << " milliseconds";
 
@@ -548,6 +547,45 @@ public:
         return ret;
     }
 
+    MPTHoldersAndCursor
+    fetchMPTHolders(
+        ripple::uint192 const& mptID,
+        std::uint32_t const limit,
+        std::optional<ripple::AccountID> const& cursorIn,
+        std::uint32_t const ledgerSequence,
+        boost::asio::yield_context yield
+    ) const override
+    {
+        auto const holderEntries = executor_.read(
+            yield, schema_->selectMPTHolders, mptID, cursorIn.value_or(ripple::AccountID(0)), Limit{limit}
+        );
+
+        auto const& holderResults = holderEntries.value();
+        if (not holderResults.hasRows()) {
+            LOG(log_.debug()) << "No rows returned";
+            return {};
+        }
+
+        std::vector<ripple::uint256> mptKeys;
+        std::optional<ripple::AccountID> cursor;
+        for (auto const [holder] : extract<ripple::AccountID>(holderResults)) {
+            mptKeys.push_back(ripple::keylet::mptoken(mptID, holder).key);
+            cursor = holder;
+        }
+
+        auto mptObjects = doFetchLedgerObjects(mptKeys, ledgerSequence, yield);
+
+        auto it = std::remove_if(mptObjects.begin(), mptObjects.end(), [](Blob const& mpt) { return mpt.size() == 0; });
+
+        mptObjects.erase(it, mptObjects.end());
+
+        ASSERT(mptKeys.size() <= limit, "Number of keys can't exceed the limit");
+        if (mptKeys.size() == limit)
+            return {mptObjects, cursor};
+
+        return {mptObjects, {}};
+    }
+
     std::optional<Blob>
     doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield)
         const override
@@ -567,6 +605,25 @@ public:
         return std::nullopt;
     }
 
+    std::optional<std::uint32_t>
+    doFetchLedgerObjectSeq(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield)
+        const override
+    {
+        LOG(log_.debug()) << "Fetching ledger object for seq " << sequence << ", key = " << ripple::to_string(key);
+        if (auto const res = executor_.read(yield, schema_->selectObject, key, sequence); res) {
+            if (auto const result = res->template get<Blob, std::uint32_t>(); result) {
+                auto [_, seq] = result.value();
+                return seq;
+            }
+            LOG(log_.debug()) << "Could not fetch ledger object sequence - no rows";
+
+        } else {
+            LOG(log_.error()) << "Could not fetch ledger object sequence: " << res.error();
+        }
+
+        return std::nullopt;
+    }
+
     std::optional<TransactionAndMetadata>
     fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const override
     {
@@ -640,7 +697,7 @@ public:
         });
 
         ASSERT(numHashes == results.size(), "Number of hashes and results must match");
-        LOG(log_.debug()) << "Fetched " << numHashes << " transactions from Cassandra in " << timeDiff
+        LOG(log_.debug()) << "Fetched " << numHashes << " transactions from database in " << timeDiff
                           << " milliseconds";
         return results;
     }
@@ -760,7 +817,7 @@ public:
         if (keys.empty())
             return {};
 
-        LOG(log_.debug()) << "Fetched " << keys.size() << " diff hashes from Cassandra in " << timeDiff
+        LOG(log_.debug()) << "Fetched " << keys.size() << " diff hashes from database in " << timeDiff
                           << " milliseconds";
 
         auto const objs = fetchLedgerObjects(keys, ledgerSequence, yield);
@@ -848,7 +905,7 @@ public:
         std::string&& metadata
     ) override
     {
-        LOG(log_.trace()) << "Writing txn to cassandra";
+        LOG(log_.trace()) << "Writing txn to database";
 
         executor_.write(schema_->insertLedgerTransaction, seq, hash);
         executor_.write(
@@ -887,6 +944,16 @@ public:
         executor_.write(std::move(statements));
     }
 
+    void
+    writeMPTHolders(std::vector<MPTHolderData> const& data) override
+    {
+        std::vector<Statement> statements;
+        for (auto [mptId, holder] : data)
+            statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
+
+        executor_.write(std::move(statements));
+    }
+
     void
     startWrites() const override
     {

src/data/DBHelpers.hpp

@@ -23,16 +23,16 @@
 #include "util/Assert.hpp"
 
 #include <boost/container/flat_set.hpp>
-#include <ripple/basics/Blob.h>
-#include <ripple/basics/Log.h>
-#include <ripple/basics/StringUtilities.h>
-#include <ripple/basics/base_uint.h>
-#include <ripple/protocol/AccountID.h>
-#include <ripple/protocol/SField.h>
-#include <ripple/protocol/STAccount.h>
-#include <ripple/protocol/STLedgerEntry.h>
-#include <ripple/protocol/Serializer.h>
-#include <ripple/protocol/TxMeta.h>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/Log.h>
+#include <xrpl/basics/StringUtilities.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/AccountID.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STAccount.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
 
 #include <cstddef>
 #include <cstdint>
@@ -172,6 +172,14 @@ struct NFTsData {
     }
 };
 
+/**
+ * @brief Represents an MPT and holder pair
+ */
+struct MPTHolderData {
+    ripple::uint192 mptID;
+    ripple::AccountID holder;
+};
+
 /**
  * @brief Check whether the supplied object is an offer.
  *

src/data/LedgerCache.cpp

@@ -22,7 +22,7 @@
 #include "data/Types.hpp"
 #include "util/Assert.hpp"
 
-#include <ripple/basics/base_uint.h>
+#include <xrpl/basics/base_uint.h>
 
 #include <cstddef>
 #include <cstdint>

src/data/LedgerCache.hpp

@@ -24,8 +24,8 @@
 #include "util/prometheus/Label.hpp"
 #include "util/prometheus/Prometheus.hpp"
 
-#include <ripple/basics/base_uint.h>
-#include <ripple/basics/hardened_hash.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/hardened_hash.h>
 
 #include <atomic>
 #include <condition_variable>

src/data/Types.hpp

@@ -19,11 +19,13 @@
 
 #pragma once
 
-#include <ripple/basics/base_uint.h>
-#include <ripple/protocol/AccountID.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/AccountID.h>
 
+#include <concepts>
 #include <cstdint>
 #include <optional>
+#include <string>
 #include <string_view>
 #include <tuple>
 #include <utility>
@@ -231,6 +233,14 @@ struct NFTsAndCursor {
     std::optional<ripple::uint256> cursor;
 };
 
+/**
+ * @brief Represents an array of MPTokens
+ */
+struct MPTHoldersAndCursor {
+    std::vector<Blob> mptokens;
+    std::optional<ripple::AccountID> cursor;
+};
+
 /**
  * @brief Stores a range of sequences as a min and max pair.
  */
@@ -239,6 +249,69 @@ struct LedgerRange {
     std::uint32_t maxSequence = 0;
 };
 
+/**
+ * @brief Represents an amendment in the XRPL
+ */
+struct Amendment {
+    std::string name;
+    ripple::uint256 feature;
+    bool isSupportedByXRPL = false;
+    bool isSupportedByClio = false;
+    bool isRetired = false;
+
+    /**
+     * @brief Get the amendment Id from its name
+     *
+     * @param name The name of the amendment
+     * @return The amendment Id as uint256
+     */
+    static ripple::uint256
+    GetAmendmentId(std::string_view const name);
+
+    /**
+     * @brief Equality comparison operator
+     * @param other The object to compare to
+     * @return Whether the objects are equal
+     */
+    bool
+    operator==(Amendment const& other) const
+    {
+        return name == other.name;
+    }
+};
+
+/**
+ * @brief A helper for amendment name to feature conversions
+ */
+struct AmendmentKey {
+    std::string name;
+
+    /**
+     * @brief Construct a new AmendmentKey
+     * @param val Anything convertible to a string
+     */
+    AmendmentKey(std::convertible_to<std::string> auto&& val) : name{std::forward<decltype(val)>(val)}
+    {
+    }
+
+    /** @brief Conversion to string */
+    operator std::string const&() const;
+
+    /** @brief Conversion to string_view */
+    operator std::string_view() const;
+
+    /** @brief Conversion to uint256 */
+    operator ripple::uint256() const;
+
+    /**
+     * @brief Comparison operators
+     * @param other The object to compare to
+     * @return Whether the objects are equal, greater or less
+     */
+    auto
+    operator<=>(AmendmentKey const& other) const = default;
+};
+
 constexpr ripple::uint256 firstKey{"0000000000000000000000000000000000000000000000000000000000000000"};
 constexpr ripple::uint256 lastKey{"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"};
 constexpr ripple::uint256 hi192{"0000000000000000000000000000000000000000000000001111111111111111"};

src/data/cassandra/Schema.hpp

@@ -257,6 +257,19 @@ public:
             qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")
         ));
 
+        statements.emplace_back(fmt::format(
+            R"(
+           CREATE TABLE IF NOT EXISTS {}
+                  ( 
+                    mpt_id blob,
+                    holder blob,
+                   PRIMARY KEY (mpt_id, holder)
+                  ) 
+             WITH CLUSTERING ORDER BY (holder ASC)
+            )",
+            qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
+        ));
+
         return statements;
     }();
 
@@ -393,6 +406,17 @@ public:
             ));
         }();
 
+        PreparedStatement insertMPTHolder = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                INSERT INTO {} 
+                       (mpt_id, holder)
+                VALUES (?, ?)
+                )",
+                qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
+            ));
+        }();
+
         PreparedStatement insertLedgerHeader = [this]() {
             return handle_.get().prepare(fmt::format(
                 R"(
@@ -687,6 +711,20 @@ public:
             ));
         }();
 
+        PreparedStatement selectMPTHolders = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                SELECT holder
+                  FROM {}    
+                 WHERE mpt_id = ?
+                   AND holder > ?
+              ORDER BY holder ASC
+                 LIMIT ?
+                )",
+                qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
+            ));
+        }();
+
         PreparedStatement selectLedgerByHash = [this]() {
             return handle_.get().prepare(fmt::format(
                 R"(

src/data/cassandra/impl/Cluster.cpp

@@ -21,6 +21,7 @@
 
 #include "data/cassandra/impl/ManagedObject.hpp"
 #include "data/cassandra/impl/SslContext.hpp"
+#include "util/OverloadSet.hpp"
 #include "util/log/Logger.hpp"
 
 #include <cassandra.h>
@@ -31,16 +32,9 @@
 #include <variant>
 
 namespace {
+
 constexpr auto clusterDeleter = [](CassCluster* ptr) { cass_cluster_free(ptr); };
 
-template <typename... Ts>
-struct overloadSet : Ts... {
-    using Ts::operator()...;
-};
-
-// explicit deduction guide (not needed as of C++20, but clang be clang)
-template <typename... Ts>
-overloadSet(Ts...) -> overloadSet<Ts...>;
 };  // namespace
 
 namespace data::cassandra::impl {
@@ -90,7 +84,7 @@ void
 Cluster::setupConnection(Settings const& settings)
 {
     std::visit(
-        overloadSet{
+        util::OverloadSet{
             [this](Settings::ContactPoints const& points) { setupContactPoints(points); },
             [this](Settings::SecureConnectionBundle const& bundle) { setupSecureBundle(bundle); }
         },

src/data/cassandra/impl/Collection.hpp

@@ -22,7 +22,7 @@
 #include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
-#include <ripple/basics/base_uint.h>
+#include <xrpl/basics/base_uint.h>
 
 #include <cstdint>
 #include <stdexcept>

src/data/cassandra/impl/Result.hpp

@@ -21,10 +21,11 @@
 
 #include "data/cassandra/impl/ManagedObject.hpp"
 #include "data/cassandra/impl/Tuple.hpp"
+#include "util/UnsupportedType.hpp"
 
 #include <cassandra.h>
-#include <ripple/basics/base_uint.h>
-#include <ripple/protocol/AccountID.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/AccountID.h>
 
 #include <cstddef>
 #include <cstdint>
@@ -41,9 +42,6 @@
 
 namespace data::cassandra::impl {
 
-template <typename>
-static constexpr bool unsupported_v = false;
-
 template <typename Type>
 inline Type
 extractColumn(CassRow const* row, std::size_t idx)
@@ -103,7 +101,7 @@ extractColumn(CassRow const* row, std::size_t idx)
         output = static_cast<DecayedType>(out);
     } else {
         // type not supported for extraction
-        static_assert(unsupported_v<DecayedType>);
+        static_assert(util::Unsupported<DecayedType>);
     }
 
     return output;

src/data/cassandra/impl/Statement.hpp

@@ -23,12 +23,13 @@
 #include "data/cassandra/impl/Collection.hpp"
 #include "data/cassandra/impl/ManagedObject.hpp"
 #include "data/cassandra/impl/Tuple.hpp"
+#include "util/UnsupportedType.hpp"
 
 #include <cassandra.h>
 #include <fmt/core.h>
-#include <ripple/basics/base_uint.h>
-#include <ripple/protocol/AccountID.h>
-#include <ripple/protocol/STAccount.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/AccountID.h>
+#include <xrpl/protocol/STAccount.h>
 
 #include <cstddef>
 #include <cstdint>
@@ -44,9 +45,6 @@ namespace data::cassandra::impl {
 class Statement : public ManagedObject<CassStatement> {
     static constexpr auto deleter = [](CassStatement* ptr) { cass_statement_free(ptr); };
 
-    template <typename>
-    static constexpr bool unsupported_v = false;
-
 public:
     /**
      * @brief Construct a new statement with optionally provided arguments.
@@ -108,9 +106,9 @@ public:
         using UintByteTupleType = std::tuple<uint32_t, ripple::uint256>;
         using ByteVectorType = std::vector<ripple::uint256>;
 
-        if constexpr (std::is_same_v<DecayedType, ripple::uint256>) {
+        if constexpr (std::is_same_v<DecayedType, ripple::uint256> || std::is_same_v<DecayedType, ripple::uint192>) {
             auto const rc = bindBytes(value.data(), value.size());
-            throwErrorIfNeeded(rc, "Bind ripple::uint256");
+            throwErrorIfNeeded(rc, "Bind ripple::base_uint");
         } else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>) {
             auto const rc = bindBytes(value.data(), value.size());
             throwErrorIfNeeded(rc, "Bind ripple::AccountID");
@@ -141,7 +139,7 @@ public:
             throwErrorIfNeeded(rc, "Bind int64");
         } else {
             // type not supported for binding
-            static_assert(unsupported_v<DecayedType>);
+            static_assert(util::Unsupported<DecayedType>);
         }
     }
 };

src/data/cassandra/impl/Tuple.hpp

@@ -20,9 +20,10 @@
 #pragma once
 
 #include "data/cassandra/impl/ManagedObject.hpp"
+#include "util/UnsupportedType.hpp"
 
 #include <cassandra.h>
-#include <ripple/basics/base_uint.h>
+#include <xrpl/basics/base_uint.h>
 
 #include <cstddef>
 #include <cstdint>
@@ -38,9 +39,6 @@ namespace data::cassandra::impl {
 class Tuple : public ManagedObject<CassTuple> {
     static constexpr auto deleter = [](CassTuple* ptr) { cass_tuple_free(ptr); };
 
-    template <typename>
-    static constexpr bool unsupported_v = false;
-
 public:
     /* implicit */ Tuple(CassTuple* ptr);
 
@@ -91,15 +89,12 @@ public:
             throwErrorIfNeeded(rc, "Bind ripple::uint256");
         } else {
             // type not supported for binding
-            static_assert(unsupported_v<DecayedType>);
+            static_assert(util::Unsupported<DecayedType>);
         }
     }
 };
 
 class TupleIterator : public ManagedObject<CassIterator> {
-    template <typename>
-    static constexpr bool unsupported_v = false;
-
 public:
     /* implicit */ TupleIterator(CassIterator* ptr);
 
@@ -141,7 +136,7 @@ private:
             output = static_cast<DecayedType>(out);
         } else {
             // type not supported for extraction
-            static_assert(unsupported_v<DecayedType>);
+            static_assert(util::Unsupported<DecayedType>);
         }
 
         return output;

src/etl/CMakeLists.txt

@@ -10,7 +10,8 @@ target_sources(
           NetworkValidatedLedgers.cpp
           NFTHelpers.cpp
           Source.cpp
-          impl/ForwardingCache.cpp
+          MPTHelpers.cpp
+          impl/AmendmentBlockHandler.cpp
           impl/ForwardingSource.cpp
           impl/GrpcSource.cpp
           impl/SubscriptionSource.cpp

src/etl/ETLHelpers.cpp

@@ -21,7 +21,7 @@
 
 #include "util/Assert.hpp"
 
-#include <ripple/basics/base_uint.h>
+#include <xrpl/basics/base_uint.h>
 
 #include <cstddef>
 #include <vector>

src/etl/ETLHelpers.hpp

@@ -20,7 +20,7 @@
 /** @file */
 #pragma once
 
-#include <ripple/basics/base_uint.h>
+#include <xrpl/basics/base_uint.h>
 
 #include <condition_variable>
 #include <cstddef>

src/etl/ETLService.cpp

@@ -30,8 +30,8 @@
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
-#include <ripple/beast/core/CurrentThreadName.h>
-#include <ripple/protocol/LedgerHeader.h>
+#include <xrpl/beast/core/CurrentThreadName.h>
+#include <xrpl/protocol/LedgerHeader.h>
 
 #include <chrono>
 #include <cstddef>

src/etl/ETLService.hpp

@@ -22,11 +22,11 @@
 #include "data/BackendInterface.hpp"
 #include "data/LedgerCache.hpp"
 #include "etl/CacheLoader.hpp"
-#include "etl/ETLHelpers.hpp"
 #include "etl/ETLState.hpp"
 #include "etl/LoadBalancer.hpp"
+#include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/SystemState.hpp"
-#include "etl/impl/AmendmentBlock.hpp"
+#include "etl/impl/AmendmentBlockHandler.hpp"
 #include "etl/impl/ExtractionDataPipe.hpp"
 #include "etl/impl/Extractor.hpp"
 #include "etl/impl/LedgerFetcher.hpp"
@@ -37,11 +37,10 @@
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
-#include <boost/asio/steady_timer.hpp>
 #include <boost/json/object.hpp>
 #include <grpcpp/grpcpp.h>
 #include <org/xrpl/rpc/v1/get_ledger.pb.h>
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
 #include <cstddef>
 #include <cstdint>
@@ -53,9 +52,6 @@
 struct AccountTransactionsData;
 struct NFTTransactionsData;
 struct NFTsData;
-namespace feed {
-class SubscriptionManager;
-}  // namespace feed
 
 /**
  * @brief This namespace contains everything to do with the ETL and ETL sources.
@@ -85,7 +81,7 @@ class ETLService {
     using ExtractorType = etl::impl::Extractor<DataPipeType, LedgerFetcherType>;
     using LedgerLoaderType = etl::impl::LedgerLoader<LoadBalancerType, LedgerFetcherType>;
     using LedgerPublisherType = etl::impl::LedgerPublisher<CacheType>;
-    using AmendmentBlockHandlerType = etl::impl::AmendmentBlockHandler<>;
+    using AmendmentBlockHandlerType = etl::impl::AmendmentBlockHandler;
     using TransformerType =
         etl::impl::Transformer<DataPipeType, LedgerLoaderType, LedgerPublisherType, AmendmentBlockHandlerType>;
 

src/etl/ETLState.cpp

@@ -24,7 +24,7 @@
 #include <boost/json/conversion.hpp>
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
-#include <ripple/protocol/jss.h>
+#include <xrpl/protocol/jss.h>
 
 #include <cstdint>
 #include <optional>

src/etl/ETLState.hpp

@@ -23,6 +23,7 @@
 
 #include <boost/json.hpp>
 #include <boost/json/conversion.hpp>
+#include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
 
@@ -46,8 +47,11 @@ struct ETLState {
     static std::optional<ETLState>
     fetchETLStateFromSource(Forward& source) noexcept
     {
-        auto const serverInfoRippled = data::synchronous([&source](auto yield) {
-            return source.forwardToRippled({{"command", "server_info"}}, std::nullopt, yield);
+        auto const serverInfoRippled = data::synchronous([&source](auto yield) -> std::optional<boost::json::object> {
+            if (auto result = source.forwardToRippled({{"command", "server_info"}}, std::nullopt, {}, yield)) {
+                return std::move(result).value();
+            }
+            return std::nullopt;
         });
 
         if (serverInfoRippled)

src/etl/LoadBalancer.cpp

@@ -24,9 +24,10 @@
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/Source.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
+#include "rpc/Errors.hpp"
 #include "util/Assert.hpp"
-#include "util/Constants.hpp"
 #include "util/Random.hpp"
+#include "util/ResponseExpirationCache.hpp"
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
@@ -34,13 +35,14 @@
 #include <boost/json/array.hpp>
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
+#include <boost/json/value_to.hpp>
 #include <fmt/core.h>
 
 #include <algorithm>
 #include <chrono>
-#include <cmath>
 #include <cstddef>
 #include <cstdint>
+#include <expected>
 #include <memory>
 #include <optional>
 #include <stdexcept>
@@ -77,11 +79,12 @@ LoadBalancer::LoadBalancer(
     SourceFactory sourceFactory
 )
 {
-    auto const forwardingCacheTimeout = config.valueOr<float>("forwarding_cache_timeout", 0.f);
+    auto const forwardingCacheTimeout = config.valueOr<float>("forwarding.cache_timeout", 0.f);
     if (forwardingCacheTimeout > 0.f) {
-        forwardingCache_ = impl::ForwardingCache{std::chrono::milliseconds{
-            std::lroundf(forwardingCacheTimeout * static_cast<float>(util::MILLISECONDS_PER_SECOND))
-        }};
+        forwardingCache_ = util::ResponseExpirationCache{
+            Config::toMilliseconds(forwardingCacheTimeout),
+            {"server_info", "server_state", "server_definitions", "fee", "ledger_closed"}
+        };
     }
 
     static constexpr std::uint32_t MAX_DOWNLOAD = 256;
@@ -103,6 +106,7 @@ LoadBalancer::LoadBalancer(
         }
     };
 
+    auto const forwardingTimeout = Config::toMilliseconds(config.valueOr<float>("forwarding.request_timeout", 10.));
     for (auto const& entry : config.array("etl_sources")) {
         auto source = sourceFactory(
             entry,
@@ -110,22 +114,27 @@ LoadBalancer::LoadBalancer(
             backend,
             subscriptions,
             validatedLedgers,
+            forwardingTimeout,
             [this]() {
-                if (not hasForwardingSource_)
+                if (not hasForwardingSource_.lock().get())
                     chooseForwardingSource();
             },
-            [this]() { chooseForwardingSource(); },
-            [this]() { forwardingCache_->invalidate(); }
+            [this](bool wasForwarding) {
+                if (wasForwarding)
+                    chooseForwardingSource();
+            },
+            [this]() {
+                if (forwardingCache_.has_value())
+                    forwardingCache_->invalidate();
+            }
         );
 
         // checking etl node validity
         auto const stateOpt = ETLState::fetchETLStateFromSource(*source);
 
         if (!stateOpt) {
-            checkOnETLFailure(fmt::format(
-                "Failed to fetch ETL state from source = {} Please check the configuration and network",
-                source->toString()
-            ));
+            LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString()
+                             << " Please check the configuration and network";
         } else if (etlState_ && etlState_->networkID && stateOpt->networkID &&
                    etlState_->networkID != stateOpt->networkID) {
             checkOnETLFailure(fmt::format(
@@ -141,6 +150,9 @@ LoadBalancer::LoadBalancer(
         LOG(log_.info()) << "Added etl source - " << sources_.back()->toString();
     }
 
+    if (!etlState_)
+        checkOnETLFailure("Failed to fetch ETL state from any source. Please check the configuration and network");
+
     if (sources_.empty())
         checkOnETLFailure("No ETL sources configured. Please check the configuration");
 
@@ -209,16 +221,21 @@ LoadBalancer::fetchLedger(
     return response;
 }
 
-std::optional<boost::json::object>
+std::expected<boost::json::object, rpc::ClioError>
 LoadBalancer::forwardToRippled(
     boost::json::object const& request,
     std::optional<std::string> const& clientIp,
+    bool isAdmin,
     boost::asio::yield_context yield
 )
 {
+    if (not request.contains("command"))
+        return std::unexpected{rpc::ClioError::rpcCOMMAND_IS_MISSING};
+
+    auto const cmd = boost::json::value_to<std::string>(request.at("command"));
     if (forwardingCache_) {
-        if (auto cachedResponse = forwardingCache_->get(request); cachedResponse) {
-            return cachedResponse;
+        if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
+            return std::move(cachedResponse).value();
         }
     }
 
@@ -227,21 +244,29 @@ LoadBalancer::forwardToRippled(
 
     auto numAttempts = 0u;
 
+    auto xUserValue = isAdmin ? ADMIN_FORWARDING_X_USER_VALUE : USER_FORWARDING_X_USER_VALUE;
+
     std::optional<boost::json::object> response;
+    rpc::ClioError error = rpc::ClioError::etlCONNECTION_ERROR;
     while (numAttempts < sources_.size()) {
-        if (auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, yield)) {
-            response = std::move(res);
+        auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield);
+        if (res) {
+            response = std::move(res).value();
             break;
         }
+        error = std::max(error, res.error());  // Choose the best result between all sources
 
         sourceIdx = (sourceIdx + 1) % sources_.size();
         ++numAttempts;
     }
 
-    if (response and forwardingCache_ and not response->contains("error"))
-        forwardingCache_->put(request, *response);
+    if (response) {
+        if (forwardingCache_ and not response->contains("error"))
+            forwardingCache_->put(cmd, *response);
+        return std::move(response).value();
+    }
 
-    return response;
+    return std::unexpected{error};
 }
 
 boost::json::value
@@ -309,11 +334,13 @@ LoadBalancer::getETLState() noexcept
 void
 LoadBalancer::chooseForwardingSource()
 {
-    hasForwardingSource_ = false;
+    LOG(log_.info()) << "Choosing a new source to forward subscriptions";
+    auto hasForwardingSourceLock = hasForwardingSource_.lock();
+    hasForwardingSourceLock.get() = false;
     for (auto& source : sources_) {
-        if (not hasForwardingSource_ and source->isConnected()) {
+        if (not hasForwardingSourceLock.get() and source->isConnected()) {
             source->setForwarding(true);
-            hasForwardingSource_ = true;
+            hasForwardingSourceLock.get() = true;
         } else {
             source->setForwarding(false);
         }

src/etl/LoadBalancer.hpp

@@ -20,11 +20,12 @@
 #pragma once
 
 #include "data/BackendInterface.hpp"
-#include "etl/ETLHelpers.hpp"
 #include "etl/ETLState.hpp"
+#include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/Source.hpp"
-#include "etl/impl/ForwardingCache.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
+#include "util/Mutex.hpp"
+#include "util/ResponseExpirationCache.hpp"
 #include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
 
@@ -36,15 +37,15 @@
 #include <grpcpp/grpcpp.h>
 #include <org/xrpl/rpc/v1/get_ledger.pb.h>
 #include <org/xrpl/rpc/v1/ledger.pb.h>
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
-#include <atomic>
 #include <chrono>
 #include <cstdint>
+#include <expected>
 #include <memory>
 #include <optional>
 #include <string>
-#include <utility>
+#include <string_view>
 #include <vector>
 
 namespace etl {
@@ -67,14 +68,29 @@ private:
 
     util::Logger log_{"ETL"};
     // Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache
-    std::optional<impl::ForwardingCache> forwardingCache_;
+    std::optional<util::ResponseExpirationCache> forwardingCache_;
+    std::optional<std::string> forwardingXUserValue_;
+
     std::vector<SourcePtr> sources_;
     std::optional<ETLState> etlState_;
     std::uint32_t downloadRanges_ =
         DEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
-    std::atomic_bool hasForwardingSource_{false};
+
+    // Using mutext instead of atomic_bool because choosing a new source to
+    // forward messages should be done with a mutual exclusion otherwise there will be a race condition
+    util::Mutex<bool> hasForwardingSource_{false};
 
 public:
+    /**
+     * @brief Value for the X-User header when forwarding admin requests
+     */
+    static constexpr std::string_view ADMIN_FORWARDING_X_USER_VALUE = "clio_admin";
+
+    /**
+     * @brief Value for the X-User header when forwarding user requests
+     */
+    static constexpr std::string_view USER_FORWARDING_X_USER_VALUE = "clio_user";
+
     /**
      * @brief Create an instance of the load balancer.
      *
@@ -167,13 +183,15 @@ public:
      *
      * @param request JSON-RPC request to forward
      * @param clientIp The IP address of the peer, if known
+     * @param isAdmin Whether the request is from an admin
      * @param yield The coroutine context
-     * @return Response received from rippled node as JSON object on success; nullopt on failure
+     * @return Response received from rippled node as JSON object on success or error on failure
      */
-    std::optional<boost::json::object>
+    std::expected<boost::json::object, rpc::ClioError>
     forwardToRippled(
         boost::json::object const& request,
         std::optional<std::string> const& clientIp,
+        bool isAdmin,
         boost::asio::yield_context yield
     );
 

src/etl/MPTHelpers.cpp

@@ -0,0 +1,78 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+
+#include <fmt/core.h>
+#include <ripple/protocol/STBase.h>
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/TxMeta.h>
+
+#include <vector>
+
+namespace etl {
+
+/**
+ * @brief Get the MPToken created from a transaction
+ *
+ * @param txMeta Transaction metadata
+ * @return MPT and holder account pair
+ */
+static std::optional<MPTHolderData>
+getMPTokenAuthorize(ripple::TxMeta const& txMeta)
+{
+    for (ripple::STObject const& node : txMeta.getNodes()) {
+        if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltMPTOKEN)
+            continue;
+
+        if (node.getFName() == ripple::sfCreatedNode) {
+            auto const& newMPT = node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>();
+            return MPTHolderData{newMPT[ripple::sfMPTokenIssuanceID], newMPT.getAccountID(ripple::sfAccount)};
+        }
+    }
+    return {};
+}
+
+std::optional<MPTHolderData>
+getMPTHolderFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
+{
+    if (txMeta.getResultTER() != ripple::tesSUCCESS || sttx.getTxnType() != ripple::TxType::ttMPTOKEN_AUTHORIZE)
+        return {};
+
+    return getMPTokenAuthorize(txMeta);
+}
+
+std::optional<MPTHolderData>
+getMPTHolderFromObj(std::string const& key, std::string const& blob)
+{
+    ripple::STLedgerEntry const sle =
+        ripple::STLedgerEntry(ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data()));
+
+    if (sle.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltMPTOKEN)
+        return {};
+
+    auto const mptIssuanceID = sle[ripple::sfMPTokenIssuanceID];
+    auto const holder = sle.getAccountID(ripple::sfAccount);
+
+    return MPTHolderData{mptIssuanceID, holder};
+}
+
+}  // namespace etl

src/etl/MPTHelpers.hpp

@@ -0,0 +1,50 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+/** @file */
+#pragma once
+
+#include "data/DBHelpers.hpp"
+
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/TxMeta.h>
+
+namespace etl {
+
+/**
+ * @brief Pull MPT data from TX via ETLService.
+ *
+ * @param txMeta Transaction metadata
+ * @param sttx The transaction
+ * @return The MPTIssuanceID and holder pair as a optional
+ */
+std::optional<MPTHolderData>
+getMPTHolderFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx);
+
+/**
+ * @brief Pull MPT data from ledger object via loadInitialLedger.
+ *
+ * @param key The owner key
+ * @param blob Object data as blob
+ * @return The MPTIssuanceID and holder pair as a optional
+ */
+std::optional<MPTHolderData>
+getMPTHolderFromObj(std::string const& key, std::string const& blob);
+
+}  // namespace etl

src/etl/NFTHelpers.cpp

@@ -20,20 +20,20 @@
 #include "data/DBHelpers.hpp"
 
 #include <fmt/core.h>
-#include <ripple/basics/base_uint.h>
-#include <ripple/basics/strHex.h>
-#include <ripple/protocol/AccountID.h>
-#include <ripple/protocol/LedgerFormats.h>
-#include <ripple/protocol/SField.h>
-#include <ripple/protocol/STArray.h>
-#include <ripple/protocol/STBase.h>
-#include <ripple/protocol/STLedgerEntry.h>
-#include <ripple/protocol/STObject.h>
-#include <ripple/protocol/STTx.h>
-#include <ripple/protocol/Serializer.h>
-#include <ripple/protocol/TER.h>
-#include <ripple/protocol/TxFormats.h>
-#include <ripple/protocol/TxMeta.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/protocol/AccountID.h>
+#include <xrpl/protocol/LedgerFormats.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STArray.h>
+#include <xrpl/protocol/STBase.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TER.h>
+#include <xrpl/protocol/TxFormats.h>
+#include <xrpl/protocol/TxMeta.h>
 
 #include <algorithm>
 #include <cstdint>
@@ -116,8 +116,8 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
         }
     }
 
-    std::sort(finalIDs.begin(), finalIDs.end());
-    std::sort(prevIDs.begin(), prevIDs.end());
+    std::ranges::sort(finalIDs);
+    std::ranges::sort(prevIDs);
 
     // Find the first NFT ID that doesn't match.  We're looking for an
     // added NFT, so the one we want will be the mismatch in finalIDs.
@@ -294,14 +294,12 @@ getNFTokenCancelOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
         txs.emplace_back(tokenID, txMeta, sttx.getTransactionID());
     }
 
-    // Deduplicate any transactions based on tokenID/txIdx combo. Can't just
-    // use txIdx because in this case one tx can cancel offers for several
-    // NFTs.
-    std::sort(txs.begin(), txs.end(), [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
-        return a.tokenID < b.tokenID && a.transactionIndex < b.transactionIndex;
+    // Deduplicate any transactions based on tokenID
+    std::ranges::sort(txs, [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
+        return a.tokenID < b.tokenID;
     });
     auto last = std::unique(txs.begin(), txs.end(), [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
-        return a.tokenID == b.tokenID && a.transactionIndex == b.transactionIndex;
+        return a.tokenID == b.tokenID;
     });
     txs.erase(last, txs.end());
     return {txs, {}};
@@ -358,4 +356,21 @@ getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string c
 
     return nfts;
 }
+
+std::vector<NFTsData>
+getUniqueNFTsDatas(std::vector<NFTsData> const& nfts)
+{
+    std::vector<NFTsData> results = nfts;
+
+    std::ranges::sort(results, [](NFTsData const& a, NFTsData const& b) {
+        return a.tokenID == b.tokenID ? a.transactionIndex > b.transactionIndex : a.tokenID > b.tokenID;
+    });
+
+    auto const last = std::unique(results.begin(), results.end(), [](NFTsData const& a, NFTsData const& b) {
+        return a.tokenID == b.tokenID;
+    });
+    results.erase(last, results.end());
+    return results;
+}
+
 }  // namespace etl

src/etl/NFTHelpers.hpp

@@ -22,8 +22,8 @@
 
 #include "data/DBHelpers.hpp"
 
-#include <ripple/protocol/STTx.h>
-#include <ripple/protocol/TxMeta.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/TxMeta.h>
 
 #include <cstdint>
 #include <optional>
@@ -104,4 +104,14 @@ getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx);
 std::vector<NFTsData>
 getNFTDataFromObj(std::uint32_t seq, std::string const& key, std::string const& blob);
 
+/**
+ * @brief Get the unique NFTs data from a vector of NFTsData happening in the same ledger. For example, if a NFT has
+ * both accept offer and burn happening in the same ledger,we only keep the final state of the NFT.
+
+ * @param nfts The NFTs data to filter, happening in the same ledger
+ * @return The unique NFTs data
+ */
+std::vector<NFTsData>
+getUniqueNFTsDatas(std::vector<NFTsData> const& nfts);
+
 }  // namespace etl

src/etl/Source.cpp

@@ -30,6 +30,7 @@
 
 #include <boost/asio/io_context.hpp>
 
+#include <chrono>
 #include <memory>
 #include <string>
 #include <utility>
@@ -43,6 +44,7 @@ make_Source(
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
     std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
+    std::chrono::steady_clock::duration forwardingTimeout,
     SourceBase::OnConnectHook onConnect,
     SourceBase::OnDisconnectHook onDisconnect,
     SourceBase::OnLedgerClosedHook onLedgerClosed
@@ -52,7 +54,7 @@ make_Source(
     auto const wsPort = config.valueOr<std::string>("ws_port", {});
     auto const grpcPort = config.valueOr<std::string>("grpc_port", {});
 
-    impl::ForwardingSource forwardingSource{ip, wsPort};
+    impl::ForwardingSource forwardingSource{ip, wsPort, forwardingTimeout};
     impl::GrpcSource grpcSource{ip, grpcPort, std::move(backend)};
     auto subscriptionSource = std::make_unique<impl::SubscriptionSource>(
         ioc,

src/etl/Source.hpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
+#include "rpc/Errors.hpp"
 #include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
 
@@ -32,11 +33,14 @@
 #include <grpcpp/support/status.h>
 #include <org/xrpl/rpc/v1/get_ledger.pb.h>
 
+#include <chrono>
 #include <cstdint>
+#include <expected>
 #include <functional>
 #include <memory>
 #include <optional>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
 
@@ -49,7 +53,7 @@ namespace etl {
 class SourceBase {
 public:
     using OnConnectHook = std::function<void()>;
-    using OnDisconnectHook = std::function<void()>;
+    using OnDisconnectHook = std::function<void(bool)>;
     using OnLedgerClosedHook = std::function<void()>;
 
     virtual ~SourceBase() = default;
@@ -127,13 +131,15 @@ public:
      *
      * @param request The request to forward
      * @param forwardToRippledClientIp IP of the client forwarding this request if known
+     * @param xUserValue Value of the X-User header
      * @param yield The coroutine context
-     * @return Response wrapped in an optional on success; nullopt otherwise
+     * @return Response on success or error on failure
      */
-    virtual std::optional<boost::json::object>
+    virtual std::expected<boost::json::object, rpc::ClioError>
     forwardToRippled(
         boost::json::object const& request,
         std::optional<std::string> const& forwardToRippledClientIp,
+        std::string_view xUserValue,
         boost::asio::yield_context yield
     ) const = 0;
 };
@@ -146,6 +152,7 @@ using SourceFactory = std::function<SourcePtr(
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
     std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
+    std::chrono::steady_clock::duration forwardingTimeout,
     SourceBase::OnConnectHook onConnect,
     SourceBase::OnDisconnectHook onDisconnect,
     SourceBase::OnLedgerClosedHook onLedgerClosed
@@ -159,6 +166,7 @@ using SourceFactory = std::function<SourcePtr(
  * @param backend BackendInterface implementation
  * @param subscriptions Subscription manager
  * @param validatedLedgers The network validated ledgers data structure
+ * @param forwardingTimeout The timeout for forwarding to rippled
  * @param onConnect The hook to call on connect
  * @param onDisconnect The hook to call on disconnect
  * @param onLedgerClosed The hook to call on ledger closed. This is called when a ledger is closed and the source is set
@@ -172,6 +180,7 @@ make_Source(
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
     std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
+    std::chrono::steady_clock::duration forwardingTimeout,
     SourceBase::OnConnectHook onConnect,
     SourceBase::OnDisconnectHook onDisconnect,
     SourceBase::OnLedgerClosedHook onLedgerClosed

src/etl/impl/AmendmentBlock.hpp

@@ -1,95 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    This file is part of clio: https://github.com/XRPLF/clio
-    Copyright (c) 2023, the clio developers.
-
-    Permission to use, copy, modify, and distribute this software for any
-    purpose with or without fee is hereby granted, provided that the above
-    copyright notice and this permission notice appear in all copies.
-
-    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
-    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
-    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-*/
-//==============================================================================
-
-#pragma once
-
-#include "etl/SystemState.hpp"
-#include "util/log/Logger.hpp"
-
-#include <boost/asio/io_context.hpp>
-#include <boost/asio/post.hpp>
-#include <boost/asio/steady_timer.hpp>
-
-#include <chrono>
-#include <functional>
-
-namespace etl::impl {
-
-struct AmendmentBlockAction {
-    void
-    operator()()
-    {
-        static util::Logger const log{"ETL"};
-        LOG(log.fatal()) << "Can't process new ledgers: The current ETL source is not compatible with the version of "
-                         << "the libxrpl Clio is currently using. Please upgrade Clio to a newer version.";
-    }
-};
-
-template <typename ActionCallableType = AmendmentBlockAction>
-class AmendmentBlockHandler {
-    std::reference_wrapper<boost::asio::io_context> ctx_;
-    std::reference_wrapper<SystemState> state_;
-    boost::asio::steady_timer timer_;
-    std::chrono::milliseconds interval_;
-
-    ActionCallableType action_;
-
-public:
-    template <typename DurationType = std::chrono::seconds>
-    AmendmentBlockHandler(
-        boost::asio::io_context& ioc,
-        SystemState& state,
-        DurationType interval = DurationType{1},
-        ActionCallableType&& action = ActionCallableType()
-    )
-        : ctx_{std::ref(ioc)}
-        , state_{std::ref(state)}
-        , timer_{ioc}
-        , interval_{std::chrono::duration_cast<std::chrono::milliseconds>(interval)}
-        , action_{std::move(action)}
-    {
-    }
-
-    ~AmendmentBlockHandler()
-    {
-        boost::asio::post(ctx_.get(), [this]() { timer_.cancel(); });
-    }
-
-    void
-    onAmendmentBlock()
-    {
-        state_.get().isAmendmentBlocked = true;
-        startReportingTimer();
-    }
-
-private:
-    void
-    startReportingTimer()
-    {
-        action_();
-
-        timer_.expires_after(interval_);
-        timer_.async_wait([this](auto ec) {
-            if (!ec)
-                boost::asio::post(ctx_.get(), [this] { startReportingTimer(); });
-        });
-    }
-};
-
-}  // namespace etl::impl

src/etl/impl/AmendmentBlockHandler.cpp

@@ -0,0 +1,56 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etl/impl/AmendmentBlockHandler.hpp"
+
+#include "etl/SystemState.hpp"
+#include "util/log/Logger.hpp"
+
+#include <boost/asio/io_context.hpp>
+
+#include <chrono>
+#include <functional>
+#include <utility>
+
+namespace etl::impl {
+
+AmendmentBlockHandler::ActionType const AmendmentBlockHandler::defaultAmendmentBlockAction = []() {
+    static util::Logger const log{"ETL"};
+    LOG(log.fatal()) << "Can't process new ledgers: The current ETL source is not compatible with the version of "
+                     << "the libxrpl Clio is currently using. Please upgrade Clio to a newer version.";
+};
+
+AmendmentBlockHandler::AmendmentBlockHandler(
+    boost::asio::io_context& ioc,
+    SystemState& state,
+    std::chrono::steady_clock::duration interval,
+    ActionType action
+)
+    : state_{std::ref(state)}, repeat_{ioc}, interval_{interval}, action_{std::move(action)}
+{
+}
+
+void
+AmendmentBlockHandler::onAmendmentBlock()
+{
+    state_.get().isAmendmentBlocked = true;
+    repeat_.start(interval_, action_);
+}
+
+}  // namespace etl::impl

src/etl/impl/AmendmentBlockHandler.hpp

@@ -0,0 +1,59 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/SystemState.hpp"
+#include "util/Repeat.hpp"
+
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/post.hpp>
+#include <boost/asio/steady_timer.hpp>
+
+#include <chrono>
+#include <functional>
+
+namespace etl::impl {
+
+class AmendmentBlockHandler {
+public:
+    using ActionType = std::function<void()>;
+
+private:
+    std::reference_wrapper<SystemState> state_;
+    util::Repeat repeat_;
+    std::chrono::steady_clock::duration interval_;
+
+    ActionType action_;
+
+public:
+    static ActionType const defaultAmendmentBlockAction;
+
+    AmendmentBlockHandler(
+        boost::asio::io_context& ioc,
+        SystemState& state,
+        std::chrono::steady_clock::duration interval = std::chrono::seconds{1},
+        ActionType action = defaultAmendmentBlockAction
+    );
+
+    void
+    onAmendmentBlock();
+};
+
+}  // namespace etl::impl

src/etl/impl/AsyncData.hpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "data/Types.hpp"
 #include "etl/ETLHelpers.hpp"
+#include "etl/MPTHelpers.hpp"
 #include "etl/NFTHelpers.hpp"
 #include "util/Assert.hpp"
 #include "util/log/Logger.hpp"
@@ -30,9 +31,9 @@
 #include <grpcpp/grpcpp.h>
 #include <grpcpp/support/status.h>
 #include <org/xrpl/rpc/v1/get_ledger_data.pb.h>
-#include <ripple/basics/base_uint.h>
-#include <ripple/basics/strHex.h>
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
 #include <cstddef>
 #include <cstdint>
@@ -154,6 +155,11 @@ public:
                     backend.writeSuccessor(std::move(lastKey_), request_.ledger().sequence(), std::string{obj.key()});
                 lastKey_ = obj.key();
                 backend.writeNFTs(getNFTDataFromObj(request_.ledger().sequence(), obj.key(), obj.data()));
+
+                auto const maybeMPTHolder = getMPTHolderFromObj(obj.key(), obj.data());
+                if (maybeMPTHolder)
+                    backend.writeMPTHolders({*maybeMPTHolder});
+
                 backend.writeLedgerObject(
                     std::move(*obj.mutable_key()), request_.ledger().sequence(), std::move(*obj.mutable_data())
                 );

src/etl/impl/BaseCursorProvider.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include <ripple/basics/base_uint.h>
+#include <xrpl/basics/base_uint.h>
 
 #include <cstdint>
 #include <vector>

src/etl/impl/CacheLoader.hpp

@@ -28,9 +28,9 @@
 
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/context/detail/config.hpp>
-#include <ripple/basics/Blob.h>
-#include <ripple/basics/base_uint.h>
-#include <ripple/basics/strHex.h>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
 
 #include <algorithm>
 #include <atomic>

src/etl/impl/CursorFromAccountProvider.hpp

@@ -23,7 +23,7 @@
 #include "data/Types.hpp"
 #include "etl/impl/BaseCursorProvider.hpp"
 
-#include <ripple/basics/base_uint.h>
+#include <xrpl/basics/base_uint.h>
 
 #include <algorithm>
 #include <cstddef>

src/etl/impl/CursorFromDiffProvider.hpp

@@ -24,7 +24,7 @@
 #include "etl/impl/BaseCursorProvider.hpp"
 #include "util/Assert.hpp"
 
-#include <ripple/basics/base_uint.h>
+#include <xrpl/basics/base_uint.h>
 
 #include <algorithm>
 #include <cstddef>

src/etl/impl/CursorFromFixDiffNumProvider.hpp

@@ -25,8 +25,8 @@
 
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/asio/io_context.hpp>
-#include <ripple/basics/base_uint.h>
-#include <ripple/basics/strHex.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
 
 #include <algorithm>
 #include <cstddef>

src/etl/impl/Extractor.hpp

@@ -25,7 +25,7 @@
 #include "util/Profiler.hpp"
 #include "util/log/Logger.hpp"
 
-#include <ripple/beast/core/CurrentThreadName.h>
+#include <xrpl/beast/core/CurrentThreadName.h>
 
 #include <chrono>
 #include <cstdint>

src/etl/impl/ForwardingSource.cpp

@@ -19,6 +19,7 @@
 
 #include "etl/impl/ForwardingSource.hpp"
 
+#include "rpc/Errors.hpp"
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/spawn.hpp>
@@ -34,6 +35,7 @@
 #include <optional>
 #include <stdexcept>
 #include <string>
+#include <string_view>
 #include <utility>
 
 namespace etl::impl {
@@ -41,9 +43,12 @@ namespace etl::impl {
 ForwardingSource::ForwardingSource(
     std::string ip,
     std::string wsPort,
+    std::chrono::steady_clock::duration forwardingTimeout,
     std::chrono::steady_clock::duration connectionTimeout
 )
-    : log_(fmt::format("ForwardingSource[{}:{}]", ip, wsPort)), connectionBuilder_(std::move(ip), std::move(wsPort))
+    : log_(fmt::format("ForwardingSource[{}:{}]", ip, wsPort))
+    , connectionBuilder_(std::move(ip), std::move(wsPort))
+    , forwardingTimeout_{forwardingTimeout}
 {
     connectionBuilder_.setConnectionTimeout(connectionTimeout)
         .addHeader(
@@ -51,10 +56,11 @@ ForwardingSource::ForwardingSource(
         );
 }
 
-std::optional<boost::json::object>
+std::expected<boost::json::object, rpc::ClioError>
 ForwardingSource::forwardToRippled(
     boost::json::object const& request,
     std::optional<std::string> const& forwardToRippledClientIp,
+    std::string_view xUserValue,
     boost::asio::yield_context yield
 ) const
 {
@@ -64,20 +70,31 @@ ForwardingSource::forwardToRippled(
             {boost::beast::http::field::forwarded, fmt::format("for={}", *forwardToRippledClientIp)}
         );
     }
+
+    connectionBuilder.addHeader({"X-User", std::string{xUserValue}});
+
     auto expectedConnection = connectionBuilder.connect(yield);
     if (not expectedConnection) {
-        return std::nullopt;
+        LOG(log_.debug()) << "Couldn't connect to rippled to forward request.";
+        return std::unexpected{rpc::ClioError::etlCONNECTION_ERROR};
     }
     auto& connection = expectedConnection.value();
 
-    auto writeError = connection->write(boost::json::serialize(request), yield);
+    auto writeError = connection->write(boost::json::serialize(request), yield, forwardingTimeout_);
     if (writeError) {
-        return std::nullopt;
+        LOG(log_.debug()) << "Error sending request to rippled to forward request.";
+        return std::unexpected{rpc::ClioError::etlREQUEST_ERROR};
     }
 
-    auto response = connection->read(yield);
+    auto response = connection->read(yield, forwardingTimeout_);
     if (not response) {
-        return std::nullopt;
+        if (auto errorCode = response.error().errorCode();
+            errorCode.has_value() and errorCode->value() == boost::system::errc::timed_out) {
+            LOG(log_.debug()) << "Request to rippled timed out";
+            return std::unexpected{rpc::ClioError::etlREQUEST_TIMEOUT};
+        }
+        LOG(log_.debug()) << "Error sending request to rippled to forward request.";
+        return std::unexpected{rpc::ClioError::etlREQUEST_ERROR};
     }
 
     boost::json::value parsedResponse;
@@ -86,8 +103,8 @@ ForwardingSource::forwardToRippled(
         if (not parsedResponse.is_object())
             throw std::runtime_error("response is not an object");
     } catch (std::exception const& e) {
-        LOG(log_.error()) << "Error parsing response from rippled: " << e.what() << ". Response: " << *response;
-        return std::nullopt;
+        LOG(log_.debug()) << "Error parsing response from rippled: " << e.what() << ". Response: " << *response;
+        return std::unexpected{rpc::ClioError::etlINVALID_RESPONSE};
     }
 
     auto responseObject = parsedResponse.as_object();

src/etl/impl/ForwardingSource.hpp

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include "rpc/Errors.hpp"
 #include "util/log/Logger.hpp"
 #include "util/requests/WsConnection.hpp"
 
@@ -26,21 +27,25 @@
 #include <boost/json/object.hpp>
 
 #include <chrono>
+#include <expected>
 #include <optional>
 #include <string>
+#include <string_view>
 
 namespace etl::impl {
 
 class ForwardingSource {
     util::Logger log_;
     util::requests::WsConnectionBuilder connectionBuilder_;
+    std::chrono::steady_clock::duration forwardingTimeout_;
 
     static constexpr std::chrono::seconds CONNECTION_TIMEOUT{3};
 
 public:
     ForwardingSource(
-        std::string ip_,
-        std::string wsPort_,
+        std::string ip,
+        std::string wsPort,
+        std::chrono::steady_clock::duration forwardingTimeout,
         std::chrono::steady_clock::duration connectionTimeout = CONNECTION_TIMEOUT
     );
 
@@ -49,13 +54,15 @@ public:
      *
      * @param request The request to forward
      * @param forwardToRippledClientIp IP of the client forwarding this request if known
+     * @param xUserValue Optional value for X-User header
      * @param yield The coroutine context
-     * @return Response wrapped in an optional on success; nullopt otherwise
+     * @return Response on success or error on failure
      */
-    std::optional<boost::json::object>
+    std::expected<boost::json::object, rpc::ClioError>
     forwardToRippled(
         boost::json::object const& request,
         std::optional<std::string> const& forwardToRippledClientIp,
+        std::string_view xUserValue,
         boost::asio::yield_context yield
     ) const;
 };

src/etl/impl/GrpcSource.cpp

@@ -24,7 +24,7 @@
 #include "util/Assert.hpp"
 #include "util/log/Logger.hpp"
 
-#include <boost/asio/ip/address.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/ip/tcp.hpp>
 #include <fmt/core.h>
 #include <grpcpp/client_context.h>
@@ -39,6 +39,7 @@
 #include <exception>
 #include <memory>
 #include <sstream>
+#include <stdexcept>
 #include <string>
 #include <utility>
 #include <vector>
@@ -46,12 +47,17 @@
 namespace etl::impl {
 
 GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort, std::shared_ptr<BackendInterface> backend)
-    : log_(fmt::format("ETL_Grpc[{}:{}]", ip, grpcPort)), backend_(std::move(backend))
+    : log_(fmt::format("GrpcSource[{}:{}]", ip, grpcPort)), backend_(std::move(backend))
 {
     try {
-        boost::asio::ip::tcp::endpoint const endpoint{boost::asio::ip::make_address(ip), std::stoi(grpcPort)};
+        boost::asio::io_context ctx;
+        boost::asio::ip::tcp::resolver resolver{ctx};
+        auto const resolverResult = resolver.resolve(ip, grpcPort);
+        if (resolverResult.empty()) {
+            throw std::runtime_error("Failed to resolve " + ip + ":" + grpcPort);
+        }
         std::stringstream ss;
-        ss << endpoint;
+        ss << resolverResult.begin()->endpoint();
         grpc::ChannelArguments chArgs;
         chArgs.SetMaxReceiveMessageSize(-1);
         stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub(

src/etl/impl/GrpcSource.hpp

@@ -24,7 +24,7 @@
 
 #include <grpcpp/support/status.h>
 #include <org/xrpl/rpc/v1/get_ledger.pb.h>
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
 #include <cstdint>
 #include <memory>

src/etl/impl/LedgerFetcher.hpp

@@ -23,7 +23,7 @@
 #include "util/log/Logger.hpp"
 
 #include <grpcpp/grpcpp.h>
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
 #include <cstdint>
 #include <memory>

src/etl/impl/LedgerLoader.hpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "data/DBHelpers.hpp"
 #include "data/Types.hpp"
+#include "etl/MPTHelpers.hpp"
 #include "etl/NFTHelpers.hpp"
 #include "etl/SystemState.hpp"
 #include "etl/impl/LedgerFetcher.hpp"
@@ -30,13 +31,13 @@
 #include "util/Profiler.hpp"
 #include "util/log/Logger.hpp"
 
-#include <ripple/basics/base_uint.h>
-#include <ripple/basics/strHex.h>
-#include <ripple/beast/core/CurrentThreadName.h>
-#include <ripple/protocol/LedgerHeader.h>
-#include <ripple/protocol/STTx.h>
-#include <ripple/protocol/Serializer.h>
-#include <ripple/protocol/TxMeta.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/beast/core/CurrentThreadName.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
 
 #include <chrono>
 #include <cstddef>
@@ -55,6 +56,7 @@ struct FormattedTransactionsData {
     std::vector<AccountTransactionsData> accountTxData;
     std::vector<NFTTransactionsData> nfTokenTxData;
     std::vector<NFTsData> nfTokensData;
+    std::vector<MPTHolderData> mptHoldersData;
 };
 
 namespace etl::impl {
@@ -124,6 +126,10 @@ public:
             if (maybeNFT)
                 result.nfTokensData.push_back(*maybeNFT);
 
+            auto const maybeMPTHolder = getMPTHolderFromTx(txMeta, sttx);
+            if (maybeMPTHolder)
+                result.mptHoldersData.push_back(*maybeMPTHolder);
+
             result.accountTxData.emplace_back(txMeta, sttx.getTransactionID());
             static constexpr std::size_t KEY_SIZE = 32;
             std::string keyStr{reinterpret_cast<char const*>(sttx.getTransactionID().data()), KEY_SIZE};
@@ -136,20 +142,7 @@ public:
             );
         }
 
-        // Remove all but the last NFTsData for each id. unique removes all but the first of a group, so we want to
-        // reverse sort by transaction index
-        std::sort(result.nfTokensData.begin(), result.nfTokensData.end(), [](NFTsData const& a, NFTsData const& b) {
-            return a.tokenID > b.tokenID && a.transactionIndex > b.transactionIndex;
-        });
-
-        // Now we can unique the NFTs by tokenID.
-        auto last = std::unique(
-            result.nfTokensData.begin(),
-            result.nfTokensData.end(),
-            [](NFTsData const& a, NFTsData const& b) { return a.tokenID == b.tokenID; }
-        );
-        result.nfTokensData.erase(last, result.nfTokensData.end());
-
+        result.nfTokensData = getUniqueNFTsDatas(result.nfTokensData);
         return result;
     }
 
@@ -253,6 +246,7 @@ public:
                 backend_->writeAccountTransactions(std::move(insertTxResult.accountTxData));
                 backend_->writeNFTs(insertTxResult.nfTokensData);
                 backend_->writeNFTTransactions(insertTxResult.nfTokenTxData);
+                backend_->writeMPTHolders(insertTxResult.mptHoldersData);
             }
 
             backend_->finishWrites(sequence);

src/etl/impl/LedgerPublisher.hpp

@@ -31,12 +31,12 @@
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/strand.hpp>
-#include <ripple/basics/chrono.h>
-#include <ripple/protocol/Fees.h>
-#include <ripple/protocol/LedgerHeader.h>
-#include <ripple/protocol/SField.h>
-#include <ripple/protocol/STObject.h>
-#include <ripple/protocol/Serializer.h>
+#include <xrpl/basics/chrono.h>
+#include <xrpl/protocol/Fees.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STObject.h>
+#include <xrpl/protocol/Serializer.h>
 
 #include <chrono>
 #include <cstddef>
@@ -166,13 +166,16 @@ public:
             LOG(log_.info()) << "Publishing ledger " << std::to_string(lgrInfo.seq);
 
             if (!state_.get().isWriting) {
-                LOG(log_.info()) << "Updating cache";
+                LOG(log_.info()) << "Updating ledger range for read node.";
 
-                std::vector<data::LedgerObject> const diff = data::synchronousAndRetryOnTimeout([&](auto yield) {
-                    return backend_->fetchLedgerDiff(lgrInfo.seq, yield);
-                });
+                if (!cache_.get().isDisabled()) {
+                    std::vector<data::LedgerObject> const diff = data::synchronousAndRetryOnTimeout([&](auto yield) {
+                        return backend_->fetchLedgerDiff(lgrInfo.seq, yield);
+                    });
+
+                    cache_.get().update(diff, lgrInfo.seq);
+                }
 
-                cache_.get().update(diff, lgrInfo.seq);
                 backend_->updateRange(lgrInfo.seq);
             }
 

src/etl/impl/SourceImpl.hpp

@@ -23,6 +23,7 @@
 #include "etl/impl/ForwardingSource.hpp"
 #include "etl/impl/GrpcSource.hpp"
 #include "etl/impl/SubscriptionSource.hpp"
+#include "rpc/Errors.hpp"
 
 #include <boost/asio/spawn.hpp>
 #include <boost/json/object.hpp>
@@ -31,11 +32,14 @@
 
 #include <chrono>
 #include <cstdint>
+#include <expected>
 #include <memory>
 #include <optional>
 #include <string>
+#include <string_view>
 #include <utility>
 #include <vector>
+
 namespace etl::impl {
 
 /**
@@ -202,17 +206,19 @@ public:
      *
      * @param request The request to forward
      * @param forwardToRippledClientIp IP of the client forwarding this request if known
+     * @param xUserValue Optional value of the X-User header
      * @param yield The coroutine context
-     * @return Response wrapped in an optional on success; nullopt otherwise
+     * @return Response or ClioError
      */
-    std::optional<boost::json::object>
+    std::expected<boost::json::object, rpc::ClioError>
     forwardToRippled(
         boost::json::object const& request,
         std::optional<std::string> const& forwardToRippledClientIp,
+        std::string_view xUserValue,
         boost::asio::yield_context yield
     ) const final
     {
-        return forwardingSource_.forwardToRippled(request, forwardToRippledClientIp, yield);
+        return forwardingSource_.forwardToRippled(request, forwardToRippledClientIp, xUserValue, yield);
     }
 };
 

src/etl/impl/SubscriptionSource.cpp

@@ -24,6 +24,8 @@
 #include "rpc/JS.hpp"
 #include "util/Retry.hpp"
 #include "util/log/Logger.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
 #include "util/requests/Types.hpp"
 
 #include <boost/algorithm/string/classification.hpp>
@@ -40,7 +42,7 @@
 #include <boost/json/serialize.hpp>
 #include <boost/json/value_to.hpp>
 #include <fmt/core.h>
-#include <ripple/protocol/jss.h>
+#include <xrpl/protocol/jss.h>
 
 #include <algorithm>
 #include <chrono>
@@ -66,22 +68,28 @@ SubscriptionSource::SubscriptionSource(
     OnConnectHook onConnect,
     OnDisconnectHook onDisconnect,
     OnLedgerClosedHook onLedgerClosed,
-    std::chrono::steady_clock::duration const connectionTimeout,
+    std::chrono::steady_clock::duration const wsTimeout,
     std::chrono::steady_clock::duration const retryDelay
 )
-    : log_(fmt::format("GrpcSource[{}:{}]", ip, wsPort))
+    : log_(fmt::format("SubscriptionSource[{}:{}]", ip, wsPort))
     , wsConnectionBuilder_(ip, wsPort)
     , validatedLedgers_(std::move(validatedLedgers))
     , subscriptions_(std::move(subscriptions))
     , strand_(boost::asio::make_strand(ioContext))
+    , wsTimeout_(wsTimeout)
     , retry_(util::makeRetryExponentialBackoff(retryDelay, RETRY_MAX_DELAY, strand_))
     , onConnect_(std::move(onConnect))
     , onDisconnect_(std::move(onDisconnect))
     , onLedgerClosed_(std::move(onLedgerClosed))
+    , lastMessageTimeSecondsSinceEpoch_(PrometheusService::gaugeInt(
+          "subscription_source_last_message_time",
+          util::prometheus::Labels({{"source", fmt::format("{}:{}", ip, wsPort)}}),
+          "Seconds since epoch of the last message received from rippled subscription streams"
+      ))
 {
     wsConnectionBuilder_.addHeader({boost::beast::http::field::user_agent, "clio-client"})
         .addHeader({"X-User", "clio-client"})
-        .setConnectionTimeout(connectionTimeout);
+        .setConnectionTimeout(wsTimeout_);
 }
 
 SubscriptionSource::~SubscriptionSource()
@@ -133,6 +141,7 @@ void
 SubscriptionSource::setForwarding(bool isForwarding)
 {
     isForwarding_ = isForwarding;
+    LOG(log_.info()) << "Forwarding set to " << isForwarding_;
 }
 
 std::chrono::steady_clock::time_point
@@ -166,20 +175,22 @@ SubscriptionSource::subscribe()
             }
 
             wsConnection_ = std::move(connection).value();
-            isConnected_ = true;
-            onConnect_();
 
             auto const& subscribeCommand = getSubscribeCommandJson();
-            auto const writeErrorOpt = wsConnection_->write(subscribeCommand, yield);
+            auto const writeErrorOpt = wsConnection_->write(subscribeCommand, yield, wsTimeout_);
             if (writeErrorOpt) {
                 handleError(writeErrorOpt.value(), yield);
                 return;
             }
 
+            isConnected_ = true;
+            LOG(log_.info()) << "Connected";
+            onConnect_();
+
             retry_.reset();
 
             while (!stop_) {
-                auto const message = wsConnection_->read(yield);
+                auto const message = wsConnection_->read(yield, wsTimeout_);
                 if (not message) {
                     handleError(message.error(), yield);
                     return;
@@ -224,10 +235,11 @@ SubscriptionSource::handleMessage(std::string const& message)
                 auto validatedLedgers = boost::json::value_to<std::string>(result.at(JS(validated_ledgers)));
                 setValidatedRange(std::move(validatedLedgers));
             }
-            LOG(log_.info()) << "Received a message on ledger subscription stream. Message : " << object;
+            LOG(log_.debug()) << "Received a message on ledger subscription stream. Message: " << object;
 
         } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_LedgerClosed) {
-            LOG(log_.info()) << "Received a message on ledger subscription stream. Message : " << object;
+            LOG(log_.debug()) << "Received a message of type 'ledgerClosed' on ledger subscription stream. Message: "
+                              << object;
             if (object.contains(JS(ledger_index))) {
                 ledgerIndex = object.at(JS(ledger_index)).as_int64();
             }
@@ -245,10 +257,13 @@ SubscriptionSource::handleMessage(std::string const& message)
                 // 2 - Validated transaction
                 // Only forward proposed transaction, validated transactions are sent by Clio itself
                 if (object.contains(JS(transaction)) and !object.contains(JS(meta))) {
+                    LOG(log_.debug()) << "Forwarding proposed transaction: " << object;
                     subscriptions_->forwardProposedTransaction(object);
                 } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ValidationReceived) {
+                    LOG(log_.debug()) << "Forwarding validation: " << object;
                     subscriptions_->forwardValidation(object);
                 } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ManifestReceived) {
+                    LOG(log_.debug()) << "Forwarding manifest: " << object;
                     subscriptions_->forwardManifest(object);
                 }
             }
@@ -261,7 +276,7 @@ SubscriptionSource::handleMessage(std::string const& message)
 
         return std::nullopt;
     } catch (std::exception const& e) {
-        LOG(log_.error()) << "Exception in handleMessage : " << e.what();
+        LOG(log_.error()) << "Exception in handleMessage: " << e.what();
         return util::requests::RequestError{fmt::format("Error handling message: {}", e.what())};
     }
 }
@@ -270,16 +285,14 @@ void
 SubscriptionSource::handleError(util::requests::RequestError const& error, boost::asio::yield_context yield)
 {
     isConnected_ = false;
-    isForwarding_ = false;
+    bool const wasForwarding = isForwarding_.exchange(false);
     if (not stop_) {
-        onDisconnect_();
+        LOG(log_.info()) << "Disconnected";
+        onDisconnect_(wasForwarding);
     }
 
     if (wsConnection_ != nullptr) {
-        auto const err = wsConnection_->close(yield);
-        if (err) {
-            LOG(log_.error()) << "Error closing websocket connection: " << err->message();
-        }
+        wsConnection_->close(yield);
         wsConnection_.reset();
     }
 
@@ -306,7 +319,11 @@ SubscriptionSource::logError(util::requests::RequestError const& error) const
 void
 SubscriptionSource::setLastMessageTime()
 {
-    lastMessageTime_.lock().get() = std::chrono::steady_clock::now();
+    lastMessageTimeSecondsSinceEpoch_.get().set(
+        std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count()
+    );
+    auto lock = lastMessageTime_.lock();
+    lock.get() = std::chrono::steady_clock::now();
 }
 
 void

src/etl/impl/SubscriptionSource.hpp

@@ -19,12 +19,13 @@
 
 #pragma once
 
-#include "etl/ETLHelpers.hpp"
+#include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/Source.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "util/Mutex.hpp"
 #include "util/Retry.hpp"
 #include "util/log/Logger.hpp"
+#include "util/prometheus/Gauge.hpp"
 #include "util/requests/Types.hpp"
 #include "util/requests/WsConnection.hpp"
 
@@ -37,6 +38,7 @@
 #include <atomic>
 #include <chrono>
 #include <cstdint>
+#include <functional>
 #include <future>
 #include <memory>
 #include <optional>
@@ -71,6 +73,8 @@ private:
 
     boost::asio::strand<boost::asio::io_context::executor_type> strand_;
 
+    std::chrono::steady_clock::duration wsTimeout_;
+
     util::Retry retry_;
 
     OnConnectHook onConnect_;
@@ -83,9 +87,11 @@ private:
 
     util::Mutex<std::chrono::steady_clock::time_point> lastMessageTime_;
 
+    std::reference_wrapper<util::prometheus::GaugeInt> lastMessageTimeSecondsSinceEpoch_;
+
     std::future<void> runFuture_;
 
-    static constexpr std::chrono::seconds CONNECTION_TIMEOUT{30};
+    static constexpr std::chrono::seconds WS_TIMEOUT{30};
     static constexpr std::chrono::seconds RETRY_MAX_DELAY{30};
     static constexpr std::chrono::seconds RETRY_DELAY{1};
 
@@ -99,11 +105,11 @@ public:
      * @param wsPort The port of the source
      * @param validatedLedgers The network validated ledgers object
      * @param subscriptions The subscription manager object
+     * @param onConnect The onConnect hook. Called when the connection is established
      * @param onDisconnect The onDisconnect hook. Called when the connection is lost
-     * @param onNewLedger The onNewLedger hook. Called when a new ledger is received
      * @param onLedgerClosed The onLedgerClosed hook. Called when the ledger is closed but only if the source is
      * forwarding
-     * @param connectionTimeout The connection timeout. Defaults to 30 seconds
+     * @param wsTimeout A timeout for websocket operations. Defaults to 30 seconds
      * @param retryDelay The retry delay. Defaults to 1 second
      */
     SubscriptionSource(
@@ -115,7 +121,7 @@ public:
         OnConnectHook onConnect,
         OnDisconnectHook onDisconnect,
         OnLedgerClosedHook onLedgerClosed,
-        std::chrono::steady_clock::duration const connectionTimeout = CONNECTION_TIMEOUT,
+        std::chrono::steady_clock::duration const wsTimeout = WS_TIMEOUT,
         std::chrono::steady_clock::duration const retryDelay = RETRY_DELAY
     );
 

src/etl/impl/Transformer.hpp

@@ -23,7 +23,7 @@
 #include "data/DBHelpers.hpp"
 #include "data/Types.hpp"
 #include "etl/SystemState.hpp"
-#include "etl/impl/AmendmentBlock.hpp"
+#include "etl/impl/AmendmentBlockHandler.hpp"
 #include "etl/impl/LedgerLoader.hpp"
 #include "util/Assert.hpp"
 #include "util/LedgerUtils.hpp"
@@ -31,11 +31,11 @@
 #include "util/log/Logger.hpp"
 
 #include <grpcpp/grpcpp.h>
-#include <ripple/basics/base_uint.h>
-#include <ripple/basics/strHex.h>
-#include <ripple/beast/core/CurrentThreadName.h>
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
-#include <ripple/protocol/LedgerHeader.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/beast/core/CurrentThreadName.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <xrpl/protocol/LedgerHeader.h>
 
 #include <chrono>
 #include <cstdint>
@@ -213,6 +213,7 @@ private:
         backend_->writeAccountTransactions(std::move(insertTxResultOp->accountTxData));
         backend_->writeNFTs(insertTxResultOp->nfTokensData);
         backend_->writeNFTTransactions(insertTxResultOp->nfTokenTxData);
+        backend_->writeMPTHolders(insertTxResultOp->mptHoldersData);
 
         auto [success, duration] =
             ::util::timed<std::chrono::duration<double>>([&]() { return backend_->finishWrites(lgrInfo.seq); });

src/etlng/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library(clio_etlng INTERFACE)
+
+# target_sources(clio_etlng PRIVATE )
+
+target_link_libraries(clio_etlng INTERFACE clio_data)