fix: Add more account check (#1543)

Make sure all char is alphanumeric for account

.clang-tidy

@@ -4,6 +4,9 @@ Checks: '-*,
   bugprone-assert-side-effect,
   bugprone-bad-signal-to-kill-thread,
   bugprone-bool-pointer-implicit-conversion,
+  bugprone-casting-through-void,
+  bugprone-chained-comparison,
+  bugprone-compare-pointer-to-member-virtual-function,
   bugprone-copy-constructor-init,
   bugprone-dangling-handle,
   bugprone-dynamic-static-initializers,
@@ -11,6 +14,8 @@ Checks: '-*,
   bugprone-fold-init-type,
   bugprone-forward-declaration-namespace,
   bugprone-inaccurate-erase,
+  bugprone-inc-dec-in-conditions,
+  bugprone-incorrect-enable-if,
   bugprone-incorrect-roundings,
   bugprone-infinite-loop,
   bugprone-integer-division,
@@ -21,15 +26,16 @@ Checks: '-*,
   bugprone-misplaced-pointer-arithmetic-in-alloc,
   bugprone-misplaced-widening-cast,
   bugprone-move-forwarding-reference,
+  bugprone-multi-level-implicit-pointer-conversion,
   bugprone-multiple-new-in-one-expression,
   bugprone-multiple-statement-macro,
   bugprone-no-escape,
   bugprone-non-zero-enum-to-bool-conversion,
+  bugprone-optional-value-conversion,
   bugprone-parent-virtual-call,
   bugprone-posix-return,
   bugprone-redundant-branch-condition,
   bugprone-reserved-identifier,
-  bugprone-unused-return-value,
   bugprone-shared-ptr-array-mismatch,
   bugprone-signal-handler,
   bugprone-signed-char-misuse,
@@ -60,16 +66,20 @@ Checks: '-*,
   bugprone-unhandled-self-assignment,
   bugprone-unique-ptr-array-mismatch,
   bugprone-unsafe-functions,
+  bugprone-unused-local-non-trivial-variable,
   bugprone-unused-raii,
+  bugprone-unused-return-value,
   bugprone-use-after-move,
   bugprone-virtual-near-miss,
   cppcoreguidelines-init-variables,
   cppcoreguidelines-misleading-capture-default-by-value,
+  cppcoreguidelines-no-suspend-with-lock,
   cppcoreguidelines-pro-type-member-init,
   cppcoreguidelines-pro-type-static-cast-downcast,
   cppcoreguidelines-rvalue-reference-param-not-moved,
   cppcoreguidelines-use-default-member-init,
   cppcoreguidelines-virtual-class-destructor,
+  hicpp-ignored-remove-result,
   llvm-namespace-comment,
   misc-const-correctness,
   misc-definitions-in-headers,
@@ -91,6 +101,8 @@ Checks: '-*,
   modernize-use-equals-default,
   modernize-use-equals-delete,
   modernize-use-override,
+  modernize-use-starts-ends-with,
+  modernize-use-std-numbers,
   modernize-use-using,
   performance-faster-string-find,
   performance-for-range-copy,
@@ -100,6 +112,8 @@ Checks: '-*,
   performance-move-constructor-init,
   performance-no-automatic-move,
   performance-trivially-destructible,
+  readability-avoid-nested-conditional-operator,
+  readability-avoid-return-with-void-value,
   readability-braces-around-statements,
   readability-const-return-type,
   readability-container-contains,
@@ -112,9 +126,12 @@ Checks: '-*,
   readability-make-member-function-const,
   readability-misleading-indentation,
   readability-non-const-parameter,
+  readability-redundant-casting,
   readability-redundant-declaration,
+  readability-redundant-inline-specifier,
   readability-redundant-member-init,
   readability-redundant-string-init,
+  readability-reference-to-constructed-temporary,
   readability-simplify-boolean-expr,
   readability-static-accessed-through-instance,
   readability-static-definition-in-anonymous-namespace,
@@ -124,9 +141,8 @@ Checks: '-*,
 CheckOptions:
   readability-braces-around-statements.ShortStatementLines: 2
   bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true
-  bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc;::std::expected
-  misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*'
+  bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc
+  misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*'
 
-HeaderFilterRegex: '^.*/(src|unittests)/.*\.(h|hpp)$'
+HeaderFilterRegex: '^.*/(src|tests)/.*\.(h|hpp)$'
 WarningsAsErrors: '*'
-

.clangd

@@ -1,5 +1,10 @@
+CompileFlags:
+  Add: [-D__cpp_concepts=202002]
+
 Diagnostics:
   UnusedIncludes: Strict
   MissingIncludes: Strict
   Includes:
-    IgnoreHeader: ".*/(detail|impl)/.*"
+    IgnoreHeader:
+      - ".*/(detail|impl)/.*"
+      - ".*expected.*"

.gitattributes

@@ -0,0 +1,2 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.svg filter=lfs diff=lfs merge=lfs -text

.githooks/check-format

@@ -8,16 +8,16 @@
 echo "+ Checking code format..."
 
 # paths to check and re-format
-sources="src unittests"
+sources="src tests"
 formatter="clang-format -i"
 version=$($formatter --version | grep -o '[0-9\.]*')
 
-if [[ "17.0.0" > "$version" ]]; then
+if [[ "18.0.0" > "$version" ]]; then
     cat <<EOF
 
                                     ERROR
 -----------------------------------------------------------------------------
-            A minimum of version 17 of `which clang-format` is required.
+            A minimum of version 18 of `which clang-format` is required.
             Your version is $version.
             Please fix paths and run again.
 -----------------------------------------------------------------------------
@@ -59,7 +59,9 @@ function grep_code {
     grep -l "${1}" ${sources} -r --include \*.hpp --include \*.cpp
 }
 
-if [[ "$OSTYPE" == "darwin"* ]]; then
+GNU_SED=$(sed --version 2>&1 | grep -q 'GNU' && echo true || echo false)
+
+if [[ "$GNU_SED" == "false" ]]; then # macOS sed
     # make all includes to be <...> style
     grep_code '#include ".*"' | xargs sed -i '' -E 's|#include "(.*)"|#include <\1>|g'
 
@@ -71,11 +73,11 @@ else
     grep_code '#include ".*"' | xargs sed -i -E 's|#include "(.*)"|#include <\1>|g'
 
     # make local includes to be "..." style
-    main_src_dirs=$(find ./src -type d -maxdepth 1 -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g')
+    main_src_dirs=$(find ./src -maxdepth 1 -type d  -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g')
     grep_code "#include <\($main_src_dirs\)/.*>" | xargs sed -i -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g"
 fi
 
-cmake_dirs=$(echo CMake $sources)
+cmake_dirs=$(echo cmake $sources)
 cmake_files=$(find $cmake_dirs -type f \( -name "CMakeLists.txt" -o -name "*.cmake" \))
 cmake_files=$(echo $cmake_files ./CMakeLists.txt)
 
@@ -83,9 +85,10 @@ first=$(git diff $sources $cmake_files)
 find $sources -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.ipp' \) -print0 | xargs -0 $formatter
 cmake-format -i $cmake_files
 second=$(git diff $sources $cmake_files)
-changes=$(diff <(echo "$first") <(echo "$second") | wc -l | sed -e 's/^[[:space:]]*//')
+changes=$(diff <(echo "$first") <(echo "$second"))
+changes_number=$(echo -n "$changes" | wc -l | sed -e 's/^[[:space:]]*//')
 
-if [ "$changes" != "0" ]; then
+if [ "$changes_number" != "0" ]; then
     cat <<\EOF
 
                                    WARNING
@@ -95,5 +98,8 @@ if [ "$changes" != "0" ]; then
 -----------------------------------------------------------------------------
 
 EOF
+    if [[ "$1" == "--diff" ]]; then
+        echo "$changes"
+    fi
     exit 1
 fi

.githooks/post-checkout

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-checkout' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-checkout "$@"

.githooks/post-commit

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-commit' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-commit "$@"

.githooks/post-merge

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-merge' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-merge "$@"

.githooks/pre-commit

@@ -3,4 +3,6 @@
 # This script is intended to be run from the root of the repository.
 
 source .githooks/check-format
-source .githooks/check-docs
+#source .githooks/check-docs
+
+# TODO: Fix Doxygen issue with reference links. See https://github.com/XRPLF/clio/issues/1431

.githooks/pre-push

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'pre-push' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs pre-push "$@"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,6 +8,7 @@ assignees: ''
 ---
 
 <!-- Please search existing issues to avoid creating duplicates. -->
+<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
 
 ## Issue Description
 <!-- Provide a summary for your issue/bug. -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -8,6 +8,7 @@ assignees: ''
 ---
 
 <!-- Please search existing issues to avoid creating duplicates. -->
+<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
 
 ## Summary
 <!-- Provide a summary to the feature request -->

.github/ISSUE_TEMPLATE/question.md

@@ -9,6 +9,7 @@ assignees: ''
 
 <!-- Please search existing issues to avoid creating duplicates. -->
 <!-- Consider starting a [discussion](https://github.com/XRPLF/clio/discussions) instead. -->
+<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
 
 ## Question
 <!-- Your question -->

.github/actions/code_coverage/action.yml

@@ -6,15 +6,19 @@ runs:
     - name: Run tests
       shell: bash
       run: |
-        build/clio_tests --backend_host=scylladb
+        build/clio_tests
 
     - name: Run gcovr
       shell: bash
       run: |
-        gcovr -e unittests --xml build/coverage_report.xml -j8 --exclude-throw-branches
+        gcovr -e tests \
+          -e src/data/cassandra \
+          -e src/data/CassandraBackend.hpp \
+          -e 'src/data/BackendFactory.*' \
+          --xml build/coverage_report.xml -j8 --exclude-throw-branches
 
     - name: Archive coverage report
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: coverage-report.xml
         path: build/coverage_report.xml

.github/actions/create_issue/action.yml

@@ -0,0 +1,35 @@
+name: Create an issue
+description: Create an issue
+inputs:
+  title:
+    description: Issue title
+    required: true
+  body:
+    description: Issue body
+    required: true
+  labels:
+    description: Comma-separated list of labels
+    required: true
+    default: 'bug'
+  assignees:
+    description: Comma-separated list of assignees
+    required: true
+    default: 'cindyyan317,godexsoft,kuznetsss'
+outputs:
+  created_issue_id:
+    description: Created issue id
+    value: ${{ steps.create_issue.outputs.created_issue }}
+runs:
+  using: composite
+  steps:
+      - name: Create an issue
+        id: create_issue
+        shell: bash
+        run: |
+          echo -e '${{ inputs.body }}' > issue.md
+          gh issue create --assignee '${{ inputs.assignees }}' --label '${{ inputs.labels }}' --title '${{ inputs.title }}' --body-file ./issue.md > create_issue.log
+          created_issue=$(cat create_issue.log | sed 's|.*/||')
+          echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
+          rm create_issue.log issue.md
+
+

.github/actions/generate/action.yml

@@ -16,6 +16,10 @@ inputs:
     description: Whether conan's coverage option should be on or not
     required: true
     default: 'false'
+  static:
+    description: Whether Clio is to be statically linked
+    required: true
+    default: 'false'
 runs:
   using: composite
   steps:
@@ -28,9 +32,10 @@ runs:
       env:
         BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '' }}"
         CODE_COVERAGE: "${{ inputs.code_coverage == 'true' && 'True' || 'False' }}"
+        STATIC_OPTION: "${{ inputs.static == 'true' && 'True' || 'False' }}"
       run: |
         cd build
-        conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:tests=True -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
+        conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:static="${STATIC_OPTION}" -o clio:tests=True -o clio:integration_tests=True -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
 
     - name: Run cmake
       shell: bash

.github/actions/prepare_runner/action.yml

@@ -12,9 +12,7 @@ runs:
       shell: bash
       run: |
           brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1
-          if ! command -v conan &> /dev/null; then
-            echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
-          fi
+          echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
 
     - name: Fix git permissions on Linux
       if: ${{ runner.os == 'Linux' }}

.github/actions/restore_cache/action.yml

@@ -4,6 +4,9 @@ inputs:
   conan_dir:
     description: Path to .conan directory
     required: true
+  conan_profile:
+    description: Conan profile name
+    required: true
   ccache_dir:
     description: Path to .ccache directory
     required: true
@@ -44,16 +47,16 @@ runs:
         echo "hash=$hash" >> $GITHUB_OUTPUT
 
     - name: Restore conan cache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       id: conan_cache
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ steps.conan_hash.outputs.hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ steps.conan_hash.outputs.hash }}
 
     - name: Restore ccache cache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       id: ccache_cache
       if: ${{ env.CCACHE_DISABLE != '1' }}
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-${{ inputs.conan_profile }}-develop-${{ steps.git_common_ancestor.outputs.commit }}

.github/actions/save_cache/action.yml

@@ -4,6 +4,9 @@ inputs:
   conan_dir:
     description: Path to .conan directory
     required: true
+  conan_profile:
+    description: Conan profile name
+    required: true
   conan_hash:
     description: Hash to use as a part of conan cache key
     required: true
@@ -41,16 +44,16 @@ runs:
 
     - name: Save conan cache
       if: ${{ inputs.conan_cache_hit != 'true' }}
-      uses: actions/cache/save@v3
+      uses: actions/cache/save@v4
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ inputs.conan_hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ inputs.conan_hash }}
 
     - name: Save ccache cache
       if: ${{ inputs.ccache_cache_hit != 'true' || inputs.ccache_cache_miss_rate == '100.0' }}
-      uses: actions/cache/save@v3
+      uses: actions/cache/save@v4
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-${{ inputs.conan_profile }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
 
 

.github/actions/setup_conan/action.yml

@@ -1,5 +1,9 @@
 name: Setup conan
 description: Setup conan profile and artifactory
+inputs:
+  conan_profile:
+    description: Conan profile name
+    required: true
 outputs:
   conan_profile:
     description: Created conan profile name
@@ -11,7 +15,7 @@ runs:
       if: ${{ runner.os == 'macOS' }}
       shell: bash
       env:
-        CONAN_PROFILE: clio_apple_clang_15
+        CONAN_PROFILE: apple_clang_15
       id: conan_setup_mac
       run: |
         echo "Creating $CONAN_PROFILE conan profile";
@@ -27,7 +31,7 @@ runs:
       shell: bash
       id: conan_setup_linux
       run: |
-        echo "created_conan_profile=default" >> $GITHUB_OUTPUT
+        echo "created_conan_profile=${{ inputs.conan_profile }}" >> $GITHUB_OUTPUT
 
     - name: Export output variable
       shell: bash

.github/scripts/update-libxrpl-version

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Note: This script is intended to be run from the root of the repository.
+#
+# This script modifies conanfile.py such that the specified version of libXRPL is used.
+
+if [[ -z "$1" ]]; then
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+            Version should be passed as first argument to the script.
+-----------------------------------------------------------------------------
+
+EOF
+    exit 1
+fi
+
+VERSION=$1
+GNU_SED=$(sed --version 2>&1 | grep -q 'GNU' && echo true || echo false)
+
+echo "+ Updating required libXRPL version to $VERSION"
+
+if [[ "$GNU_SED" == "false" ]]; then
+    sed -i '' -E "s|'xrpl/[a-zA-Z0-9\\.\\-]+'|'xrpl/$VERSION'|g" conanfile.py
+else
+    sed -i -E "s|'xrpl/[a-zA-Z0-9\\.\\-]+'|'xrpl/$VERSION'|g" conanfile.py
+fi

.github/workflows/build.yml

@@ -13,11 +13,15 @@ jobs:
     container:
       image: rippleci/clio_ci:latest
     steps:
+      - name: Fix git permissions on Linux
+        shell: bash
+        run: git config --global --add safe.directory $PWD
+
       - uses: actions/checkout@v4
       - name: Run formatters
         id: run_formatters
         run: |
-          ./.githooks/check-format
+          ./.githooks/check-format --diff
         shell: bash
   
   check_docs:
@@ -35,8 +39,8 @@ jobs:
 
   build:
     name: Build
-    needs: 
-      - check_format 
+    needs:
+      - check_format
       - check_docs
     strategy:
       fail-fast: false
@@ -46,28 +50,42 @@ jobs:
             container:
               image: rippleci/clio_ci:latest
             build_type: Release
+            conan_profile: gcc
             code_coverage: false
+            static: true
           - os: heavy
             container:
               image: rippleci/clio_ci:latest
             build_type: Debug
+            conan_profile: gcc
             code_coverage: true
+            static: true
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            build_type: Release
+            conan_profile: clang
+            code_coverage: false
+            static: true
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            build_type: Debug
+            conan_profile: clang
+            code_coverage: false
+            static: true
           - os: macos14
             build_type: Release
             code_coverage: false
+            static: false
     runs-on: [self-hosted, "${{ matrix.os }}"]
     container: ${{ matrix.container }}
 
-    services:
-      scylladb:
-        image: ${{ (matrix.code_coverage) && 'scylladb/scylla' || '' }}
-        options: >-
-          --health-cmd "cqlsh -e 'describe cluster'" 
-          --health-interval 10s 
-          --health-timeout 5s 
-          --health-retries 5
-
     steps:
+      - name: Clean workdir
+        if: ${{ runner.os == 'macOS' }}
+        uses: kuznetsss/workspace-cleanup@1.0
+
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -80,12 +98,15 @@ jobs:
       - name: Setup conan
         uses: ./.github/actions/setup_conan
         id: conan
+        with:
+          conan_profile: ${{ matrix.conan_profile }}
 
       - name: Restore cache
         uses: ./.github/actions/restore_cache
         id: restore_cache
         with:
           conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
           ccache_dir: ${{ env.CCACHE_DIR }}
           build_type: ${{ matrix.build_type }}
           code_coverage: ${{ matrix.code_coverage }}
@@ -97,6 +118,7 @@ jobs:
           conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
           build_type: ${{ matrix.build_type }}
           code_coverage: ${{ matrix.code_coverage }}
+          static: ${{ matrix.static }}
 
       - name: Build Clio
         uses: ./.github/actions/build_clio
@@ -112,20 +134,20 @@ jobs:
 
       - name: Strip tests
         if: ${{ !matrix.code_coverage }}
-        run: strip build/clio_tests
+        run: strip build/clio_tests && strip build/clio_integration_tests
 
       - name: Upload clio_server
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: clio_server_${{ runner.os }}_${{ matrix.build_type }}
+          name: clio_server_${{ runner.os }}_${{ matrix.build_type }}_${{ steps.conan.outputs.conan_profile }}
           path: build/clio_server
 
       - name: Upload clio_tests
         if: ${{ !matrix.code_coverage }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: clio_tests_${{ runner.os }}
-          path: build/clio_tests
+          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ steps.conan.outputs.conan_profile }}
+          path: build/clio_*tests
 
       - name: Save cache
         uses: ./.github/actions/save_cache
@@ -138,6 +160,7 @@ jobs:
           ccache_cache_miss_rate: ${{ steps.ccache_stats.outputs.miss_rate }}
           build_type: ${{ matrix.build_type }}
           code_coverage: ${{ matrix.code_coverage }}
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
 
       # TODO: This is not a part of build process but it is the easiest way to do it here.
       # It will be refactored in https://github.com/XRPLF/clio/issues/1075
@@ -162,15 +185,33 @@ jobs:
           - os: heavy
             container:
               image: rippleci/clio_ci:latest
+            conan_profile: gcc
+            build_type: Release
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            conan_profile: clang
+            build_type: Release
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            conan_profile: clang
+            build_type: Debug
           - os: macos14
+            conan_profile: apple_clang_15
+            build_type: Release
     runs-on: [self-hosted, "${{ matrix.os }}"]
     container: ${{ matrix.container }}
 
     steps:
-      - uses: actions/download-artifact@v3
+      - name: Clean workdir
+        if: ${{ runner.os == 'macOS' }}
+        uses: kuznetsss/workspace-cleanup@1.0
+
+      - uses: actions/download-artifact@v4
         with:
-          name: clio_tests_${{ runner.os }}
+          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests
-          ./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
+          ./clio_tests

.github/workflows/check_libxrpl.yml

@@ -0,0 +1,91 @@
+name: Check new libXRPL
+on:
+  repository_dispatch:
+    types: [check_libxrpl]
+
+jobs:
+  build:
+    name: Build Clio / `libXRPL ${{ github.event.client_payload.version }}`
+    runs-on: [self-hosted, heavy]
+    container:
+      image: rippleci/clio_ci:latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            fetch-depth: 0
+
+      - name: Update libXRPL version requirement
+        shell: bash
+        run: |
+          ./.github/scripts/update-libxrpl-version ${{ github.event.client_payload.version }}
+
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+            disable_ccache: true
+
+      - name: Setup conan
+        uses: ./.github/actions/setup_conan
+        id: conan
+        with:
+          conan_profile: gcc
+
+      - name: Run conan and cmake
+        uses: ./.github/actions/generate
+        with:
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
+          build_type: Release
+
+      - name: Build Clio
+        uses: ./.github/actions/build_clio
+
+      - name: Strip tests
+        run: strip build/clio_tests
+
+      - name: Upload clio_tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: clio_tests_libxrpl-${{ github.event.client_payload.version }}
+          path: build/clio_tests
+
+  run_tests:
+    name: Run tests
+    needs: build
+    runs-on: [self-hosted, heavy]
+    container:
+      image: rippleci/clio_ci:latest
+
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: clio_tests_libxrpl-${{ github.event.client_payload.version }}
+
+      - name: Run clio_tests
+        run: |
+          chmod +x ./clio_tests
+          ./clio_tests
+
+  create_issue_on_failure:
+    name: Create an issue on failure
+    needs: [build, run_tests]
+    if: ${{ always() && contains(needs.*.result, 'failure') }}
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create an issue
+        uses: ./.github/actions/create_issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          labels: 'compatibility,bug'
+          title: 'Proposed libXRPL check failed'
+          body: >
+            Clio build or tests failed against `libXRPL ${{ github.event.client_payload.version }}`.
+
+            Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/

.github/workflows/clang-tidy.yml

@@ -33,6 +33,8 @@ jobs:
       - name: Setup conan
         uses: ./.github/actions/setup_conan
         id: conan
+        with:
+          conan_profile: clang
 
       - name: Restore cache
         uses: ./.github/actions/restore_cache
@@ -40,6 +42,7 @@ jobs:
         with:
           conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
           ccache_dir: ${{ env.CCACHE_DIR }}
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
 
       - name: Run conan and cmake
         uses: ./.github/actions/generate
@@ -57,7 +60,7 @@ jobs:
         shell: bash
         id: run_clang_tidy
         run: |
-          run-clang-tidy-17 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
+          run-clang-tidy-18 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
 
       - name: Check format
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
@@ -76,16 +79,15 @@ jobs:
       - name: Create an issue
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
         id: create_issue
-        shell: bash
+        uses: ./.github/actions/create_issue
         env:
           GH_TOKEN: ${{ github.token }}
-        run: |
-          echo -e 'Clang-tidy found issues in the code:\n' > issue.md
-          echo -e "List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/" >> issue.md
-          gh issue create --assignee 'cindyyan317,godexsoft,kuznetsss' --label bug --title 'Clang-tidy found bugs in code🐛' --body-file ./issue.md > create_issue.log
-          created_issue=$(cat create_issue.log | sed 's|.*/||')
-          echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
-          rm create_issue.log issue.md
+        with:
+          title: 'Clang-tidy found bugs in code 🐛'
+          body: >
+            Clang-tidy found issues in the code:
+
+            List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/
 
       - uses: crazy-max/ghaction-import-gpg@v5
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
@@ -108,7 +110,7 @@ jobs:
           branch-suffix: timestamp
           delete-branch: true
           title: "[CI] clang-tidy auto fixes"
-          body: "Fixes #${{ steps.create_issue.outputs.created_issue }}. Please review and commit clang-tidy fixes."
+          body: "Fixes #${{ steps.create_issue.outputs.created_issue_id }}. Please review and commit clang-tidy fixes."
           reviewers: "cindyyan317,godexsoft,kuznetsss"
 
       - name: Fail the job

.github/workflows/docs.yml

@@ -1,7 +1,7 @@
 name: Documentation
 on:
   push:
-    branches: [release/*, develop]
+    branches: [develop]
   workflow_dispatch:
 
 permissions:
@@ -24,7 +24,9 @@ jobs:
       image: rippleci/clio_ci:latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with: 
+          lfs: true
 
       - name: Build docs
         run: |
@@ -32,13 +34,13 @@ jobs:
           cmake ../docs && cmake --build . --target docs
       
       - name: Setup Pages
-        uses: actions/configure-pages@v3
+        uses: actions/configure-pages@v4
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
           path: build_docs/html
-          name: docs-develop # TODO: use x.y.z for `release/x.y.z` branches and `develop` for latest dev docs
+          name: docs-develop
       
       - name: Deploy to GitHub Pages
         id: deployment

.github/workflows/nightly.yml

@@ -25,6 +25,10 @@ jobs:
     container: ${{ matrix.container }}
 
     steps:
+      - name: Clean workdir
+        if: ${{ runner.os == 'macOS' }}
+        uses: kuznetsss/workspace-cleanup@1.0
+
       - uses: actions/checkout@v4
         with:
             fetch-depth: 0
@@ -37,6 +41,8 @@ jobs:
       - name: Setup conan
         uses: ./.github/actions/setup_conan
         id: conan
+        with:
+          conan_profile: gcc
 
       - name: Run conan and cmake
         uses: ./.github/actions/generate
@@ -49,13 +55,13 @@ jobs:
         uses: ./.github/actions/build_clio
 
       - name: Strip tests
-        run: strip build/clio_tests
+        run: strip build/clio_tests && strip build/clio_integration_tests
 
       - name: Upload clio_tests
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
-          path: build/clio_tests
+          path: build/clio_*tests
 
       - name: Compress clio_server
         shell: bash
@@ -64,7 +70,7 @@ jobs:
             tar czf ./clio_server_${{ runner.os }}_${{ matrix.build_type }}.tar.gz ./clio_server
 
       - name: Upload clio_server
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: clio_server_${{ runner.os }}_${{ matrix.build_type }}
           path: build/clio_server_${{ runner.os }}_${{ matrix.build_type }}.tar.gz
@@ -78,21 +84,50 @@ jobs:
         include:
           - os: macos14
             build_type: Release
+            integration_tests: false
           - os: heavy
             build_type: Release
+            container:
+              image: rippleci/clio_ci:latest
+            integration_tests: true  
           - os: heavy
             build_type: Debug
+            container:
+              image: rippleci/clio_ci:latest
+            integration_tests: true  
     runs-on: [self-hosted, "${{ matrix.os }}"]
+    container: ${{ matrix.container }}
+
+    services:
+      scylladb:
+        image: ${{ (matrix.integration_tests) && 'scylladb/scylla' || '' }}
+        options: >-
+          --health-cmd "cqlsh -e 'describe cluster'" 
+          --health-interval 10s 
+          --health-timeout 5s 
+          --health-retries 5
 
     steps:
-      - uses: actions/download-artifact@v3
+      - name: Clean workdir
+        if: ${{ runner.os == 'macOS' }}
+        uses: kuznetsss/workspace-cleanup@1.0
+
+      - uses: actions/download-artifact@v4
         with:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
 
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests
-          ./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
+          ./clio_tests
+  
+  # To be enabled back once docker in mac runner arrives
+  # https://github.com/XRPLF/clio/issues/1400
+      - name: Run clio_integration_tests
+        if: matrix.integration_tests 
+        run: |
+          chmod +x ./clio_integration_tests
+          ./clio_integration_tests --backend_host=scylladb
 
   nightly_release:
     needs: run_tests
@@ -105,7 +140,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           path: nightly_release
 
@@ -114,7 +149,7 @@ jobs:
         run: |
           cp ${{ github.workspace }}/.github/workflows/nightly_notes.md "${RUNNER_TEMP}/nightly_notes.md"
           cd nightly_release
-          rm -r clio_tests*
+          rm -r clio_*tests*
           for d in $(ls); do
             archive_name=$(ls $d)
             mv ${d}/${archive_name} ./
@@ -136,3 +171,24 @@ jobs:
           gh release create nightly --prerelease --title "Clio development (nightly) build" \
             --target $GITHUB_SHA --notes-file "${RUNNER_TEMP}/nightly_notes.md" \
             ./nightly_release/clio_server*
+
+  create_issue_on_failure:
+    needs: [build, run_tests, nightly_release]
+    if: ${{ always() && contains(needs.*.result, 'failure') }}
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create an issue
+        uses: ./.github/actions/create_issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          title: 'Nightly release failed 🌙'
+          body: >
+            Nightly release failed:
+
+            Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/

.github/workflows/update_docker_ci.yml

@@ -1,18 +1,25 @@
 name: Update CI docker image
 on:
+  pull_request:
+    paths:
+      - 'docker/ci/**'
+      - 'docker/compilers/**'
+      - .github/workflows/update_docker_ci.yml
   push:
     branches: [develop]
     paths:
-      - 'docker/ci/**'
+      - 'docker/ci/**'        # CI image must update when either its dockerfile changes
+      - 'docker/compilers/**' # or any compilers changed and were pushed by hand
       - .github/workflows/update_docker_ci.yml
   workflow_dispatch:
 
 jobs:
   build_and_push:
     name: Build and push docker image
-    runs-on: ubuntu-20.04
+    runs-on: [self-hosted, heavy]
     steps:
       - name: Login to DockerHub
+        if: ${{ github.event_name != 'pull_request' }}
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
@@ -27,7 +34,7 @@ jobs:
           images: rippleci/clio_ci
           tags: |
             type=raw,value=latest
-            type=raw,value=gcc_11
+            type=raw,value=gcc_12_clang_16
             type=raw,value=${{ env.GITHUB_SHA }}
 
       - name: Build and push
@@ -35,6 +42,6 @@ jobs:
         with:
           context: ${{ github.workspace }}/docker/ci
           platforms: linux/amd64,linux/arm64
-          push: true
+          push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
 

.github/workflows/upload_coverage_report.yml

@@ -16,16 +16,16 @@ jobs:
           fetch-depth: 0
 
       - name: Download report artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: coverage-report.xml
           path: build
 
       - name: Upload coverage report
         if: ${{ hashFiles('build/coverage_report.xml') != '' }}
-        uses: wandalen/wretry.action@v1.3.0
+        uses: wandalen/wretry.action@v1.4.10
         with:
-          action: codecov/codecov-action@v3
+          action: codecov/codecov-action@v4
           with: |
             files: build/coverage_report.xml 
             fail_ci_if_error: false

CMake/deps/Boost.cmake

@@ -1,4 +0,0 @@
-set(Boost_USE_STATIC_LIBS ON)
-set(Boost_USE_STATIC_RUNTIME ON)
-
-find_package(Boost 1.82 REQUIRED COMPONENTS program_options coroutine system log log_setup)

CMake/deps/cassandra.cmake

@@ -1 +0,0 @@
-find_package(cassandra-cpp-driver REQUIRED)

CMake/deps/gbench.cmake

@@ -1 +0,0 @@
-find_package(benchmark REQUIRED)

CMake/deps/libbacktrace.cmake

@@ -1,3 +0,0 @@
-target_compile_definitions(clio PUBLIC BOOST_STACKTRACE_LINK)
-target_compile_definitions(clio PUBLIC BOOST_STACKTRACE_USE_BACKTRACE)
-find_package(libbacktrace REQUIRED)

CMake/deps/libfmt.cmake

@@ -1 +0,0 @@
-find_package(fmt REQUIRED)

CMake/deps/libxrpl.cmake

@@ -1 +0,0 @@
-find_package(xrpl REQUIRED)

CMakeLists.txt

@@ -1,30 +1,37 @@
-cmake_minimum_required(VERSION 3.16.3)
-project(clio)
+cmake_minimum_required(VERSION 3.20)
+
+set(CMAKE_PROJECT_INCLUDE_BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ClioVersion.cmake)
+
+project(clio VERSION ${CLIO_VERSION} HOMEPAGE_URL "https://github.com/XRPLF/clio"
+        DESCRIPTION "An XRP Ledger API Server"
+)
 
 # =========================== Options ====================================== #
 option(verbose "Verbose build" FALSE)
-option(tests "Build tests" FALSE)
+option(tests "Build unit tests" FALSE)
+option(integration_tests "Build integration tests" FALSE)
 option(benchmark "Build benchmarks" FALSE)
 option(docs "Generate doxygen docs" FALSE)
 option(coverage "Build test coverage report" FALSE)
 option(packaging "Create distribution packages" FALSE)
 option(lint "Run clang-tidy checks during compilation" FALSE)
+option(static "Statically linked Clio" FALSE)
 # ========================================================================== #
 set(san "" CACHE STRING "Add sanitizer instrumentation")
 set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
 set_property(CACHE san PROPERTY STRINGS ";undefined;memory;address;thread")
 # ========================================================================== #
 
-# Include required modules
-include(CMake/Ccache.cmake)
-include(CheckCXXCompilerFlag)
-include(CMake/ClangTidy.cmake)
+set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
 
-# Set coverage build options
-if (tests AND coverage)
-  include(CMake/CodeCoverage.cmake)
-  append_coverage_compiler_flags()
-endif ()
+# Include required modules
+include(Ccache)
+include(CheckCXXCompilerFlag)
+include(ClangTidy)
+
+add_library(clio_options INTERFACE)
+target_compile_features(clio_options INTERFACE cxx_std_23) # Clio needs c++23 but deps can remain c++20 for now
+target_include_directories(clio_options INTERFACE ${CMAKE_SOURCE_DIR}/src)
 
 if (verbose)
   set(CMAKE_VERBOSE_MAKEFILE TRUE)
@@ -32,414 +39,28 @@ endif ()
 
 if (packaging)
   add_definitions(-DPKG=1)
+  target_compile_definitions(clio_options INTERFACE PKG=1)
 endif ()
 
-add_library(clio)
-
 # Clio tweaks and checks
-include(CMake/CheckCompiler.cmake)
-include(CMake/Settings.cmake)
-include(CMake/ClioVersion.cmake)
-include(CMake/SourceLocation.cmake)
+include(CheckCompiler)
+include(Settings)
+include(SourceLocation)
 
 # Clio deps
-include(CMake/deps/libxrpl.cmake)
-include(CMake/deps/Boost.cmake)
-include(CMake/deps/OpenSSL.cmake)
-include(CMake/deps/Threads.cmake)
-include(CMake/deps/libfmt.cmake)
-include(CMake/deps/cassandra.cmake)
-include(CMake/deps/libbacktrace.cmake)
+include(deps/libxrpl)
+include(deps/Boost)
+include(deps/OpenSSL)
+include(deps/Threads)
+include(deps/libfmt)
+include(deps/cassandra)
+include(deps/libbacktrace)
 
-# TODO: Include directory will be wrong when installed.
-target_include_directories(clio PUBLIC src)
-target_compile_features(clio PUBLIC cxx_std_20)
+add_subdirectory(src)
+add_subdirectory(tests)
 
-target_link_libraries(
-  clio
-  PUBLIC Boost::boost
-  PUBLIC Boost::coroutine
-  PUBLIC Boost::program_options
-  PUBLIC Boost::system
-  PUBLIC Boost::log
-  PUBLIC Boost::log_setup
-  PUBLIC Boost::stacktrace_backtrace
-  PUBLIC cassandra-cpp-driver::cassandra-cpp-driver
-  PUBLIC fmt::fmt
-  PUBLIC OpenSSL::Crypto
-  PUBLIC OpenSSL::SSL
-  PUBLIC xrpl::libxrpl
-  PUBLIC dl
-  PUBLIC libbacktrace::libbacktrace
-  INTERFACE Threads::Threads
-)
-
-if (is_gcc)
-  # FIXME: needed on gcc for now
-  target_compile_definitions(clio PUBLIC BOOST_ASIO_DISABLE_CONCEPTS)
-endif ()
-
-target_sources(
-  clio
-  PRIVATE # Main
-          src/main/impl/Build.cpp
-          # Backend
-          src/data/BackendCounters.cpp
-          src/data/BackendInterface.cpp
-          src/data/LedgerCache.cpp
-          src/data/cassandra/impl/Future.cpp
-          src/data/cassandra/impl/Cluster.cpp
-          src/data/cassandra/impl/Batch.cpp
-          src/data/cassandra/impl/Result.cpp
-          src/data/cassandra/impl/Tuple.cpp
-          src/data/cassandra/impl/SslContext.cpp
-          src/data/cassandra/Handle.cpp
-          src/data/cassandra/SettingsProvider.cpp
-          # ETL
-          src/etl/NFTHelpers.cpp
-          src/etl/ETLService.cpp
-          src/etl/ETLState.cpp
-          src/etl/LoadBalancer.cpp
-          src/etl/CacheLoaderSettings.cpp
-          src/etl/Source.cpp
-          src/etl/impl/ForwardingCache.cpp
-          src/etl/impl/ForwardingSource.cpp
-          src/etl/impl/GrpcSource.cpp
-          src/etl/impl/SubscriptionSource.cpp
-          # Feed
-          src/feed/SubscriptionManager.cpp
-          src/feed/impl/TransactionFeed.cpp
-          src/feed/impl/LedgerFeed.cpp
-          src/feed/impl/ProposedTransactionFeed.cpp
-          src/feed/impl/SingleFeedBase.cpp
-          # Web
-          src/web/impl/AdminVerificationStrategy.cpp
-          src/web/IntervalSweepHandler.cpp
-          src/web/Resolver.cpp
-          # RPC
-          src/rpc/Errors.cpp
-          src/rpc/Factories.cpp
-          src/rpc/AMMHelpers.cpp
-          src/rpc/RPCHelpers.cpp
-          src/rpc/Counters.cpp
-          src/rpc/WorkQueue.cpp
-          src/rpc/common/Specs.cpp
-          src/rpc/common/Validators.cpp
-          src/rpc/common/MetaProcessors.cpp
-          src/rpc/common/impl/APIVersionParser.cpp
-          src/rpc/common/impl/HandlerProvider.cpp
-          # RPC handlers
-          src/rpc/handlers/AccountChannels.cpp
-          src/rpc/handlers/AccountCurrencies.cpp
-          src/rpc/handlers/AccountInfo.cpp
-          src/rpc/handlers/AccountLines.cpp
-          src/rpc/handlers/AccountNFTs.cpp
-          src/rpc/handlers/AccountObjects.cpp
-          src/rpc/handlers/AccountOffers.cpp
-          src/rpc/handlers/AccountTx.cpp
-          src/rpc/handlers/AMMInfo.cpp
-          src/rpc/handlers/BookChanges.cpp
-          src/rpc/handlers/BookOffers.cpp
-          src/rpc/handlers/DepositAuthorized.cpp
-          src/rpc/handlers/GatewayBalances.cpp
-          src/rpc/handlers/Ledger.cpp
-          src/rpc/handlers/LedgerData.cpp
-          src/rpc/handlers/LedgerEntry.cpp
-          src/rpc/handlers/LedgerRange.cpp
-          src/rpc/handlers/NFTsByIssuer.cpp
-          src/rpc/handlers/NFTBuyOffers.cpp
-          src/rpc/handlers/NFTHistory.cpp
-          src/rpc/handlers/NFTInfo.cpp
-          src/rpc/handlers/NFTOffersCommon.cpp
-          src/rpc/handlers/NFTSellOffers.cpp
-          src/rpc/handlers/NoRippleCheck.cpp
-          src/rpc/handlers/Random.cpp
-          src/rpc/handlers/TransactionEntry.cpp
-          # Util
-          src/util/config/Config.cpp
-          src/util/log/Logger.cpp
-          src/util/prometheus/Http.cpp
-          src/util/prometheus/Label.cpp
-          src/util/prometheus/MetricBase.cpp
-          src/util/prometheus/MetricBuilder.cpp
-          src/util/prometheus/MetricsFamily.cpp
-          src/util/prometheus/OStream.cpp
-          src/util/prometheus/Prometheus.cpp
-          src/util/Random.cpp
-          src/util/Retry.cpp
-          src/util/requests/RequestBuilder.cpp
-          src/util/requests/Types.cpp
-          src/util/requests/WsConnection.cpp
-          src/util/requests/impl/SslContext.cpp
-          src/util/Taggable.cpp
-          src/util/TerminationHandler.cpp
-          src/util/TxUtils.cpp
-          src/util/LedgerUtils.cpp
-)
-
-# Clio server
-add_executable(clio_server src/main/Main.cpp)
-target_link_libraries(clio_server PRIVATE clio)
-target_link_options(
-  clio_server PRIVATE $<$<AND:$<NOT:$<BOOL:${APPLE}>>,$<NOT:$<BOOL:${san}>>>:-static-libstdc++ -static-libgcc>
-)
-
-# Unittesting
-if (tests)
-  set(TEST_TARGET clio_tests)
-  add_executable(
-    ${TEST_TARGET}
-    # Common
-    unittests/ConfigTests.cpp
-    unittests/data/BackendCountersTests.cpp
-    unittests/data/BackendCountersTests.cpp
-    unittests/data/BackendFactoryTests.cpp
-    unittests/data/BackendFactoryTests.cpp
-    unittests/data/cassandra/AsyncExecutorTests.cpp
-    # Webserver
-    unittests/data/cassandra/AsyncExecutorTests.cpp
-    # Webserver
-    unittests/data/cassandra/BackendTests.cpp
-    unittests/data/cassandra/BackendTests.cpp
-    unittests/data/cassandra/BaseTests.cpp
-    unittests/data/cassandra/BaseTests.cpp
-    unittests/data/cassandra/ExecutionStrategyTests.cpp
-    unittests/data/cassandra/ExecutionStrategyTests.cpp
-    unittests/data/cassandra/RetryPolicyTests.cpp
-    unittests/data/cassandra/RetryPolicyTests.cpp
-    unittests/data/cassandra/SettingsProviderTests.cpp
-    unittests/data/cassandra/SettingsProviderTests.cpp
-    unittests/DOSGuardTests.cpp
-    unittests/etl/AmendmentBlockHandlerTests.cpp
-    unittests/etl/AmendmentBlockHandlerTests.cpp
-    unittests/etl/CacheLoaderSettingsTests.cpp
-    unittests/etl/CacheLoaderTests.cpp
-    unittests/etl/CacheLoaderTests.cpp
-    unittests/etl/CursorProviderTests.cpp
-    unittests/etl/ETLStateTests.cpp
-    unittests/etl/ETLStateTests.cpp
-    unittests/etl/ExtractionDataPipeTests.cpp
-    unittests/etl/ExtractionDataPipeTests.cpp
-    unittests/etl/ExtractorTests.cpp
-    unittests/etl/ExtractorTests.cpp
-    unittests/etl/ForwardingCacheTests.cpp
-    unittests/etl/ForwardingSourceTests.cpp
-    unittests/etl/ForwardingSourceTests.cpp
-    unittests/etl/GrpcSourceTests.cpp
-    unittests/etl/GrpcSourceTests.cpp
-    unittests/etl/LedgerPublisherTests.cpp
-    unittests/etl/LedgerPublisherTests.cpp
-    unittests/etl/SourceTests.cpp
-    unittests/etl/SourceTests.cpp
-    unittests/etl/SubscriptionSourceDependenciesTests.cpp
-    unittests/etl/SubscriptionSourceDependenciesTests.cpp
-    unittests/etl/SubscriptionSourceTests.cpp
-    unittests/etl/SubscriptionSourceTests.cpp
-    unittests/etl/TransformerTests.cpp
-    # RPC
-    unittests/etl/TransformerTests.cpp
-    # RPC
-    unittests/feed/BookChangesFeedTests.cpp
-    unittests/feed/ForwardFeedTests.cpp
-    unittests/feed/LedgerFeedTests.cpp
-    unittests/feed/ProposedTransactionFeedTests.cpp
-    unittests/feed/SingleFeedBaseTests.cpp
-    unittests/feed/SubscriptionManagerTests.cpp
-    unittests/feed/TrackableSignalTests.cpp
-    unittests/feed/TransactionFeedTests.cpp
-    unittests/JsonUtilTests.cpp
-    unittests/LoggerTests.cpp
-    unittests/Main.cpp
-    unittests/Playground.cpp
-    unittests/ProfilerTests.cpp
-    unittests/rpc/AmendmentsTests.cpp
-    unittests/rpc/AmendmentsTests.cpp
-    unittests/rpc/APIVersionTests.cpp
-    unittests/rpc/APIVersionTests.cpp
-    unittests/rpc/BaseTests.cpp
-    unittests/rpc/BaseTests.cpp
-    unittests/rpc/CountersTests.cpp
-    unittests/rpc/CountersTests.cpp
-    unittests/rpc/ErrorTests.cpp
-    unittests/rpc/ErrorTests.cpp
-    unittests/rpc/ForwardingProxyTests.cpp
-    unittests/rpc/ForwardingProxyTests.cpp
-    unittests/rpc/handlers/AccountChannelsTests.cpp
-    unittests/rpc/handlers/AccountChannelsTests.cpp
-    unittests/rpc/handlers/AccountCurrenciesTests.cpp
-    unittests/rpc/handlers/AccountCurrenciesTests.cpp
-    unittests/rpc/handlers/AccountInfoTests.cpp
-    unittests/rpc/handlers/AccountInfoTests.cpp
-    unittests/rpc/handlers/AccountLinesTests.cpp
-    unittests/rpc/handlers/AccountLinesTests.cpp
-    unittests/rpc/handlers/AccountNFTsTests.cpp
-    unittests/rpc/handlers/AccountNFTsTests.cpp
-    unittests/rpc/handlers/AccountObjectsTests.cpp
-    unittests/rpc/handlers/AccountObjectsTests.cpp
-    unittests/rpc/handlers/AccountOffersTests.cpp
-    unittests/rpc/handlers/AccountOffersTests.cpp
-    unittests/rpc/handlers/AccountTxTests.cpp
-    unittests/rpc/handlers/AccountTxTests.cpp
-    unittests/rpc/handlers/AMMInfoTests.cpp
-    # Backend
-    unittests/rpc/handlers/AMMInfoTests.cpp
-    # Backend
-    unittests/rpc/handlers/BookChangesTests.cpp
-    unittests/rpc/handlers/BookChangesTests.cpp
-    unittests/rpc/handlers/BookOffersTests.cpp
-    unittests/rpc/handlers/BookOffersTests.cpp
-    unittests/rpc/handlers/DefaultProcessorTests.cpp
-    unittests/rpc/handlers/DefaultProcessorTests.cpp
-    unittests/rpc/handlers/DepositAuthorizedTests.cpp
-    unittests/rpc/handlers/DepositAuthorizedTests.cpp
-    unittests/rpc/handlers/GatewayBalancesTests.cpp
-    unittests/rpc/handlers/GatewayBalancesTests.cpp
-    unittests/rpc/handlers/LedgerDataTests.cpp
-    unittests/rpc/handlers/LedgerDataTests.cpp
-    unittests/rpc/handlers/LedgerEntryTests.cpp
-    unittests/rpc/handlers/LedgerEntryTests.cpp
-    unittests/rpc/handlers/LedgerRangeTests.cpp
-    unittests/rpc/handlers/LedgerRangeTests.cpp
-    unittests/rpc/handlers/LedgerTests.cpp
-    unittests/rpc/handlers/LedgerTests.cpp
-    unittests/rpc/handlers/NFTBuyOffersTests.cpp
-    unittests/rpc/handlers/NFTBuyOffersTests.cpp
-    unittests/rpc/handlers/NFTHistoryTests.cpp
-    unittests/rpc/handlers/NFTHistoryTests.cpp
-    unittests/rpc/handlers/NFTInfoTests.cpp
-    unittests/rpc/handlers/NFTInfoTests.cpp
-    unittests/rpc/handlers/NFTsByIssuerTest.cpp
-    unittests/rpc/handlers/NFTsByIssuerTest.cpp
-    unittests/rpc/handlers/NFTSellOffersTests.cpp
-    unittests/rpc/handlers/NFTSellOffersTests.cpp
-    unittests/rpc/handlers/NoRippleCheckTests.cpp
-    unittests/rpc/handlers/NoRippleCheckTests.cpp
-    unittests/rpc/handlers/PingTests.cpp
-    unittests/rpc/handlers/PingTests.cpp
-    unittests/rpc/handlers/RandomTests.cpp
-    unittests/rpc/handlers/RandomTests.cpp
-    unittests/rpc/handlers/ServerInfoTests.cpp
-    unittests/rpc/handlers/ServerInfoTests.cpp
-    unittests/rpc/handlers/SubscribeTests.cpp
-    unittests/rpc/handlers/SubscribeTests.cpp
-    unittests/rpc/handlers/TestHandlerTests.cpp
-    unittests/rpc/handlers/TestHandlerTests.cpp
-    unittests/rpc/handlers/TransactionEntryTests.cpp
-    unittests/rpc/handlers/TransactionEntryTests.cpp
-    unittests/rpc/handlers/TxTests.cpp
-    unittests/rpc/handlers/TxTests.cpp
-    unittests/rpc/handlers/UnsubscribeTests.cpp
-    unittests/rpc/handlers/UnsubscribeTests.cpp
-    unittests/rpc/handlers/VersionHandlerTests.cpp
-    unittests/rpc/handlers/VersionHandlerTests.cpp
-    unittests/rpc/JsonBoolTests.cpp
-    # RPC handlers
-    unittests/rpc/JsonBoolTests.cpp
-    # RPC handlers
-    unittests/rpc/RPCHelpersTests.cpp
-    unittests/rpc/RPCHelpersTests.cpp
-    unittests/rpc/WorkQueueTests.cpp
-    unittests/rpc/WorkQueueTests.cpp
-    unittests/util/AssertTests.cpp
-    unittests/util/async/AnyExecutionContextTests.cpp
-    unittests/util/async/AnyOperationTests.cpp
-    unittests/util/async/AnyStopTokenTests.cpp
-    unittests/util/async/AnyStrandTests.cpp
-    unittests/util/async/AsyncExecutionContextTests.cpp
-    # Requests framework
-    unittests/util/BatchingTests.cpp
-    unittests/util/LedgerUtilsTests.cpp
-    # Prometheus support
-    unittests/util/prometheus/CounterTests.cpp
-    unittests/util/prometheus/GaugeTests.cpp
-    unittests/util/prometheus/HistogramTests.cpp
-    unittests/util/prometheus/HttpTests.cpp
-    unittests/util/prometheus/LabelTests.cpp
-    unittests/util/prometheus/MetricBuilderTests.cpp
-    unittests/util/prometheus/MetricsFamilyTests.cpp
-    unittests/util/prometheus/OStreamTests.cpp
-    unittests/util/requests/RequestBuilderTests.cpp
-    unittests/util/requests/SslContextTests.cpp
-    unittests/util/requests/WsConnectionTests.cpp
-    # ETL
-    unittests/util/RetryTests.cpp
-    # Async framework
-    unittests/util/StringUtils.cpp
-    unittests/util/TestGlobals.cpp
-    unittests/util/TestHttpServer.cpp
-    unittests/util/TestObject.cpp
-    unittests/util/TestWsServer.cpp
-    unittests/util/TxUtilTests.cpp
-    unittests/web/AdminVerificationTests.cpp
-    unittests/web/RPCServerHandlerTests.cpp
-    unittests/web/ServerTests.cpp
-    unittests/web/SweepHandlerTests.cpp
-    # Feed
-    unittests/web/WhitelistHandlerTests.cpp
-  )
-
-  include(CMake/deps/gtest.cmake)
-
-  # See https://github.com/google/googletest/issues/3475
-  gtest_discover_tests(clio_tests DISCOVERY_TIMEOUT 90)
-
-  # Fix for dwarf5 bug on ci
-  target_compile_options(clio PUBLIC -gdwarf-4)
-
-  target_compile_definitions(${TEST_TARGET} PUBLIC UNITTEST_BUILD)
-  target_include_directories(${TEST_TARGET} PRIVATE unittests)
-  target_link_libraries(${TEST_TARGET} PUBLIC clio gtest::gtest)
-
-  # Generate `coverage_report` target if coverage is enabled
-  if (coverage)
-    if (DEFINED CODE_COVERAGE_REPORT_FORMAT)
-      set(CODE_COVERAGE_FORMAT ${CODE_COVERAGE_REPORT_FORMAT})
-    else ()
-      set(CODE_COVERAGE_FORMAT html-details)
-    endif ()
-
-    if (DEFINED CODE_COVERAGE_TESTS_ARGS)
-      set(TESTS_ADDITIONAL_ARGS ${CODE_COVERAGE_TESTS_ARGS})
-      separate_arguments(TESTS_ADDITIONAL_ARGS)
-    else ()
-      set(TESTS_ADDITIONAL_ARGS "")
-    endif ()
-
-    set(GCOVR_ADDITIONAL_ARGS --exclude-throw-branches -s)
-
-    setup_target_for_coverage_gcovr(
-      NAME
-      coverage_report
-      FORMAT
-      ${CODE_COVERAGE_FORMAT}
-      EXECUTABLE
-      clio_tests
-      EXECUTABLE_ARGS
-      --gtest_brief=1
-      ${TESTS_ADDITIONAL_ARGS}
-      EXCLUDE
-      "unittests"
-      DEPENDENCIES
-      clio_tests
-    )
-  endif ()
-endif ()
-
-# Benchmarks
 if (benchmark)
-  set(BENCH_TARGET clio_benchmarks)
-  add_executable(
-    ${BENCH_TARGET}
-    # Common
-    benchmarks/Main.cpp benchmarks/Playground.cpp
-    # ExecutionContext
-    benchmarks/util/async/ExecutionContextBenchmarks.cpp
-  )
-
-  include(CMake/deps/gbench.cmake)
-
-  target_include_directories(${BENCH_TARGET} PRIVATE benchmarks)
-  target_link_libraries(${BENCH_TARGET} PUBLIC clio benchmark::benchmark_main)
+  add_subdirectory(benchmarks)
 endif ()
 
 # Enable selected sanitizer if enabled via `san`
@@ -460,7 +81,7 @@ if (docs)
   add_subdirectory(docs)
 endif ()
 
-include(CMake/install/install.cmake)
+include(install/install)
 if (packaging)
-  include(CMake/packaging.cmake) # This file exists only in build runner
+  include(cmake/packaging.cmake) # This file exists only in build runner
 endif ()

CONTRIBUTING.md

@@ -6,7 +6,7 @@ To contribute, please:
 2. Create a new branch on which to commit/push your changes.
 3. Write and test your code.
 4. Ensure that your code compiles with the provided build engine and update the provided build engine as part of your PR where needed and where appropriate.
-5. Where applicable, write test cases for your code and include those in `unittests`.
+5. Where applicable, write test cases for your code and include those in the relevant subfolder under `tests`.
 6. Ensure your code passes automated checks (e.g. clang-format)
 7. Squash your commits (i.e. rebase) into as few commits as is reasonable to describe your changes at a high level (typically a single commit for a small change). See below for more details.
 8. Open a PR to the main repository onto the _develop_ branch, and follow the provided template.
@@ -21,7 +21,7 @@ git config --local core.hooksPath .githooks
 ```
 
 ## Git hooks dependencies
-The pre-commit hook requires `clang-format >= 17.0.0` and `cmake-format` to be installed on your machine.
+The pre-commit hook requires `clang-format >= 18.0.0` and `cmake-format` to be installed on your machine.
 `clang-format` can be installed using `brew` on macOS and default package manager on Linux.
 `cmake-format` can be installed using `pip`.
 The hook will also attempt to automatically use `doxygen` to verify that everything public in the codebase is covered by doc comments. If `doxygen` is not installed, the hook will raise a warning suggesting to install `doxygen` for future commits.
@@ -102,7 +102,7 @@ The button for that is near the bottom of the PR's page on GitHub.
 This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent.
 
 ## Formatting
-Code must conform to `clang-format` version 17, unless the result would be unreasonably difficult to read or maintain.
+Code must conform to `clang-format` version 18, unless the result would be unreasonably difficult to read or maintain.
 In most cases the pre-commit hook will take care of formatting and will fix any issues automatically.
 To manually format your code, use `clang-format -i <your changed files>` for C++ files and `cmake-format -i <your changed files>` for CMake files.
 

README.md

@@ -36,10 +36,12 @@ Below are some useful docs to learn more about Clio.
 - [How to configure Clio and rippled](./docs/configure-clio.md)
 - [How to run Clio](./docs/run-clio.md)
 - [Logging](./docs/logging.md)
+- [Troubleshooting guide](./docs/trouble_shooting.md)
 
 **General reference material:**
 
 - [API reference](https://xrpl.org/http-websocket-apis.html)
+- [Developer docs](https://xrplf.github.io/clio)
 - [Clio documentation](https://xrpl.org/the-clio-server.html#the-clio-server)
 
 ## 🆘 Help

benchmarks/CMakeLists.txt

@@ -0,0 +1,16 @@
+add_executable(clio_benchmark)
+
+target_sources(
+  clio_benchmark
+  PRIVATE # Common
+          Main.cpp
+          Playground.cpp
+          # ExecutionContext
+          util/async/ExecutionContextBenchmarks.cpp
+)
+
+include(deps/gbench)
+
+target_include_directories(clio_benchmark PRIVATE .)
+target_link_libraries(clio_benchmark PUBLIC clio benchmark::benchmark_main)
+set_target_properties(clio_benchmark PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

cmake/Build.cpp.in

@@ -22,7 +22,7 @@
 #include <string>
 
 namespace Build {
-static constexpr char versionString[] = "@VERSION@";
+static constexpr char versionString[] = "@CLIO_VERSION@";
 
 std::string const&
 getClioVersionString()

cmake/CheckCompiler.cmake

@@ -1,20 +1,20 @@
 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
-    message(FATAL_ERROR "Clang 14+ required for building clio")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16)
+    message(FATAL_ERROR "Clang 16+ required for building clio")
   endif ()
   set(is_clang TRUE)
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
-    message(FATAL_ERROR "AppleClang 14+ required for building clio")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15)
+    message(FATAL_ERROR "AppleClang 15+ required for building clio")
   endif ()
   set(is_appleclang TRUE)
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11)
-    message(FATAL_ERROR "GCC 11+ required for building clio")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12)
+    message(FATAL_ERROR "GCC 12+ required for building clio")
   endif ()
   set(is_gcc TRUE)
 else ()
-  message(FATAL_ERROR "Supported compilers: AppleClang 14+, Clang 14+, GCC 11+")
+  message(FATAL_ERROR "Supported compilers: AppleClang 15+, Clang 16+, GCC 12+")
 endif ()
 
 if (san)

cmake/ClangTidy.cmake

@@ -8,7 +8,7 @@ if (lint)
     endif ()
     message(STATUS "Using clang-tidy from CLIO_CLANG_TIDY_BIN")
   else ()
-    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-17" "clang-tidy" REQUIRED)
+    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-18" "clang-tidy" REQUIRED)
   endif ()
 
   if (NOT _CLANG_TIDY_BIN)

cmake/ClioVersion.cmake

@@ -22,24 +22,27 @@ endif ()
 
 if (NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name
                                                              # YYYYMMDDHMS-<branch>-<git-rev>
-  execute_process(COMMAND date +%Y%m%d%H%M%S OUTPUT_VARIABLE DATE OUTPUT_STRIP_TRAILING_WHITESPACE)
-  set(VERSION "${DATE}-${BRANCH}-${REV}")
+  set(GIT_COMMAND show -s --date=format:%Y%m%d%H%M%S --format=%cd)
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE DATE
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(CLIO_VERSION "${DATE}-${BRANCH}-${REV}")
   set(DOC_CLIO_VERSION "develop")
 else ()
   set(GIT_COMMAND describe --tags)
   execute_process(
-    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE TAG_VERSION
+    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE CLIO_TAG_VERSION
     OUTPUT_STRIP_TRAILING_WHITESPACE
   )
-  set(VERSION "${TAG_VERSION}-${REV}")
-  set(DOC_CLIO_VERSION "${TAG_VERSION}")
+  set(CLIO_VERSION "${CLIO_TAG_VERSION}")
+  set(DOC_CLIO_VERSION "${CLIO_TAG_VERSION}")
 endif ()
 
 if (CMAKE_BUILD_TYPE MATCHES Debug)
-  set(VERSION "${VERSION}+DEBUG")
+  set(CLIO_VERSION "${CLIO_VERSION}+DEBUG")
 endif ()
 
-message(STATUS "Build version: ${VERSION}")
-set(clio_version "${VERSION}")
+message(STATUS "Build version: ${CLIO_VERSION}")
 
 configure_file(${CMAKE_CURRENT_LIST_DIR}/Build.cpp.in ${CMAKE_CURRENT_LIST_DIR}/../src/main/impl/Build.cpp)

cmake/CodeCoverage.cmake

@@ -352,10 +352,10 @@ function (append_coverage_compiler_flags)
 endfunction () # append_coverage_compiler_flags
 
 # Setup coverage for specific library
-function (append_coverage_compiler_flags_to_target name)
+function (append_coverage_compiler_flags_to_target name mode)
   separate_arguments(_flag_list NATIVE_COMMAND "${COVERAGE_COMPILER_FLAGS}")
-  target_compile_options(${name} PRIVATE ${_flag_list})
+  target_compile_options(${name} ${mode} ${_flag_list})
   if (CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    target_link_libraries(${name} PRIVATE gcov)
+    target_link_libraries(${name} ${mode} gcov)
   endif ()
 endfunction ()

cmake/Settings.cmake

@@ -17,6 +17,13 @@ set(COMPILER_FLAGS
     -pedantic
     -Wpedantic
     -Wunused
+    # FIXME: The following bunch are needed for gcc12 atm.
+    -Wno-missing-requires
+    -Wno-restrict
+    -Wno-null-dereference
+    -Wno-maybe-uninitialized
+    -Wno-unknown-warning-option # and this to work with clang
+    # TODO: Address these and others in https://github.com/XRPLF/clio/issues/1273
 )
 
 # TODO: reenable when we change CI #884 if (is_gcc AND NOT lint) list(APPEND COMPILER_FLAGS -Wduplicated-branches
@@ -35,4 +42,4 @@ endif ()
 # See https://github.com/cpp-best-practices/cppbestpractices/blob/master/02-Use_the_Tools_Available.md#gcc--clang for
 # the flags description
 
-target_compile_options(clio PUBLIC ${COMPILER_FLAGS})
+target_compile_options(clio_options INTERFACE ${COMPILER_FLAGS})

cmake/SourceLocation.cmake

@@ -2,10 +2,10 @@ include(CheckIncludeFileCXX)
 
 check_include_file_cxx("source_location" SOURCE_LOCATION_AVAILABLE)
 if (SOURCE_LOCATION_AVAILABLE)
-  target_compile_definitions(clio PUBLIC "HAS_SOURCE_LOCATION")
+  target_compile_definitions(clio_options INTERFACE "HAS_SOURCE_LOCATION")
 endif ()
 
 check_include_file_cxx("experimental/source_location" EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)
 if (EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)
-  target_compile_definitions(clio PUBLIC "HAS_EXPERIMENTAL_SOURCE_LOCATION")
+  target_compile_definitions(clio_options INTERFACE "HAS_EXPERIMENTAL_SOURCE_LOCATION")
 endif ()

cmake/deps/Boost.cmake

@@ -0,0 +1,4 @@
+set(Boost_USE_STATIC_LIBS ON)
+set(Boost_USE_STATIC_RUNTIME ON)
+
+find_package(Boost 1.82 REQUIRED CONFIG COMPONENTS program_options coroutine system log log_setup)

cmake/deps/OpenSSL.cmake

@@ -1,3 +1,3 @@
-find_package(OpenSSL 1.1.1 REQUIRED)
+find_package(OpenSSL 1.1.1 REQUIRED CONFIG)
 
 set_target_properties(OpenSSL::SSL PROPERTIES INTERFACE_COMPILE_DEFINITIONS OPENSSL_NO_SSL2)

cmake/deps/Threads.cmake

@@ -1,2 +1,2 @@
 set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(Threads)
+find_package(Threads REQUIRED)

cmake/deps/cassandra.cmake

@@ -0,0 +1 @@
+find_package(cassandra-cpp-driver REQUIRED CONFIG)

cmake/deps/gbench.cmake

@@ -0,0 +1 @@
+find_package(benchmark REQUIRED CONFIG)

cmake/deps/libbacktrace.cmake

@@ -0,0 +1,3 @@
+target_compile_definitions(clio_options INTERFACE BOOST_STACKTRACE_LINK)
+target_compile_definitions(clio_options INTERFACE BOOST_STACKTRACE_USE_BACKTRACE)
+find_package(libbacktrace REQUIRED CONFIG)

cmake/deps/libfmt.cmake

@@ -0,0 +1 @@
+find_package(fmt REQUIRED CONFIG)

cmake/deps/libxrpl.cmake

@@ -0,0 +1 @@
+find_package(xrpl REQUIRED CONFIG)

cmake/install/install.cmake

@@ -8,6 +8,6 @@ string(REGEX REPLACE "./clio_log" "/var/log/clio/" config "${config}")
 file(WRITE ${CMAKE_BINARY_DIR}/install-config.json "${config}")
 install(FILES ${CMAKE_BINARY_DIR}/install-config.json DESTINATION etc RENAME config.json)
 
-configure_file("${CMAKE_SOURCE_DIR}/CMake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service")
+configure_file("${CMAKE_SOURCE_DIR}/cmake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service")
 
 install(FILES "${CMAKE_BINARY_DIR}/clio.service" DESTINATION /lib/systemd/system)

conanfile.py

@@ -1,7 +1,6 @@
 from conan import ConanFile
 from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
 
-
 class Clio(ConanFile):
     name = 'clio'
     license = 'ISC'
@@ -10,31 +9,35 @@ class Clio(ConanFile):
     description = 'Clio RPC server'
     settings = 'os', 'compiler', 'build_type', 'arch'
     options = {
-        'fPIC': [True, False],
+        'static': [True, False],              # static linkage
+        'fPIC': [True, False],                # unused?
         'verbose': [True, False],
-        'tests': [True, False],     # build unit tests; create `clio_tests` binary
-        'benchmark': [True, False], # build benchmarks; create `clio_benchmarks` binary
-        'docs': [True, False],      # doxygen API docs; create custom target 'docs'
-        'packaging': [True, False], # create distribution packages
-        'coverage': [True, False],  # build for test coverage report; create custom target `clio_tests-ccov`
-        'lint': [True, False],      # run clang-tidy checks during compilation
+        'tests': [True, False],               # build unit tests; create `clio_tests` binary
+        'integration_tests': [True, False],   # build integration tests; create `clio_integration_tests` binary
+        'benchmark': [True, False],           # build benchmarks; create `clio_benchmarks` binary
+        'docs': [True, False],                # doxygen API docs; create custom target 'docs'
+        'packaging': [True, False],           # create distribution packages
+        'coverage': [True, False],            # build for test coverage report; create custom target `clio_tests-ccov`
+        'lint': [True, False],                # run clang-tidy checks during compilation
     }
 
     requires = [
         'boost/1.82.0',
         'cassandra-cpp-driver/2.17.0',
         'fmt/10.1.1',
-        'protobuf/3.21.12',
+        'protobuf/3.21.9',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.2.0-b1',
+        'xrpl/2.2.0',
         'libbacktrace/cci.20210118'
     ]
 
     default_options = {
+        'static': False,
         'fPIC': True,
         'verbose': False,
         'tests': False,
+        'integration_tests': False,
         'benchmark': False,
         'packaging': False,
         'coverage': False,
@@ -42,6 +45,7 @@ class Clio(ConanFile):
         'docs': False,
         
         'xrpl/*:tests': False,
+        'xrpl/*:rocksdb': False,
         'cassandra-cpp-driver/*:shared': False,
         'date/*:header_only': True,
         'grpc/*:shared': False,
@@ -56,11 +60,11 @@ class Clio(ConanFile):
     }
 
     exports_sources = (
-        'CMakeLists.txt', 'CMake/*', 'src/*'
+        'CMakeLists.txt', 'cmake/*', 'src/*'
     )
 
     def requirements(self):
-        if self.options.tests:
+        if self.options.tests or self.options.integration_tests:
             self.requires('gtest/1.14.0')
         if self.options.benchmark:
             self.requires('benchmark/1.8.3')
@@ -79,11 +83,14 @@ class Clio(ConanFile):
     def generate(self):
         tc = CMakeToolchain(self)
         tc.variables['verbose'] = self.options.verbose
+        tc.variables['static'] = self.options.static
         tc.variables['tests'] = self.options.tests
+        tc.variables['integration_tests'] = self.options.integration_tests
         tc.variables['coverage'] = self.options.coverage
         tc.variables['lint'] = self.options.lint
         tc.variables['docs'] = self.options.docs
         tc.variables['packaging'] = self.options.packaging
+        tc.variables['benchmark'] = self.options.benchmark
         tc.generate()
 
     def build(self):

docker/centos/Dockerfile

@@ -1,49 +0,0 @@
-# FROM centos:7 as deps
-FROM centos:7 as build
-
-ENV CLIO_DIR=/opt/clio/
-# ENV OPENSSL_DIR=/opt/openssl
-
-RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl
-RUN yum install -y devtoolset-11
-ENV version=3.16
-ENV build=3
-# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
-COPY docker/shared/install_cmake.sh /install_cmake.sh
-RUN /install_cmake.sh 3.16.3 /usr/local
-RUN source /opt/rh/devtoolset-11/enable
-WORKDIR /tmp
-# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR
-COPY docker/centos/build_git_centos7.sh build_git_centos7.sh
-
-RUN ./build_git_centos7.sh
-RUN git clone https://github.com/openssl/openssl
-WORKDIR /tmp/openssl
-RUN git checkout OpenSSL_1_1_1q
-#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic
-RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \
-    make -j $(nproc) && \
-    make install_sw
-WORKDIR /tmp
-# FROM centos:7 as build
-
-RUN git clone https://github.com/xrplf/clio.git
-COPY docker/shared/build_boost.sh build_boost.sh
-ENV OPENSSL_ROOT=/opt/local/openssl
-ENV BOOST_ROOT=/boost
-RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0
-RUN yum install -y bison flex
-RUN yum install -y rpmdevtools rpmlint
-RUN source /opt/rh/devtoolset-11/enable && cd /tmp/clio && \
-    cmake -B build -DBUILD_TESTS=1 && \
-    cmake --build build --parallel $(nproc)
-RUN mkdir output
-RUN strip clio/build/clio_server && strip clio/build/clio_tests
-RUN cp clio/build/clio_tests output/ && cp clio/build/clio_server output/
-RUN cp clio/docs/examples/config/example-config.json output/example-config.json
-
-FROM centos:7
-COPY --from=build /tmp/output /clio
-RUN mkdir -p /opt/clio/etc && mv /clio/docs/examples/config/example-config.json /opt/clio/etc/config.json
-
-CMD ["/clio/clio_server", "/opt/clio/etc/config.json"]

docker/centos/build_git_centos7.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-GIT_VERSION="2.37.1"
-curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz
-tar zxvf git-${GIT_VERSION}.tar.gz
-cd git-${GIT_VERSION}
-
-yum install -y centos-release-scl epel-release
-yum update -y
-yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel  libcurl-devel
-
-source /opt/rh/devtoolset-11/enable
-make configure
-./configure
-make git -j$(nproc)
-make install git
-git  --version | cut -d ' ' -f3

docker/centos/install_cmake.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-CMAKE_VERSION=${1:-"3.16.3"}
-cd /tmp
-URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
-curl -OJLs $URL
-tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
-ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake

docker/ci/dockerfile

@@ -1,42 +1,46 @@
-FROM ubuntu:focal
+FROM rippleci/clio_clang:16
 ARG DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
 
 SHELL ["/bin/bash", "-c"]
 USER root
-WORKDIR /root/
+WORKDIR /root
 
-ENV GCC_VERSION=11 \
-    CCACHE_VERSION=4.8.3 \
-    LLVM_TOOLS_VERSION=17 \
+ENV CCACHE_VERSION=4.8.3 \
+    LLVM_TOOLS_VERSION=18 \
     GH_VERSION=2.40.0 \
     DOXYGEN_VERSION=1.10.0
  
 # Add repositories
 RUN apt-get -qq update \
     && apt-get -qq install -y --no-install-recommends --no-install-suggests gnupg wget curl software-properties-common \
-    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
-    && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - \
-    && apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' \
     && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM_TOOLS_VERSION} main" >> /etc/apt/sources.list \
     && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 
 # Install packages
 RUN apt update -qq \
-    && apt install -y --no-install-recommends --no-install-suggests cmake python3 python3-pip sudo git \
-    ninja-build make pkg-config libzstd-dev libzstd1 g++-${GCC_VERSION} flex bison jq graphviz \
+    && apt install -y --no-install-recommends --no-install-suggests python3 python3-pip git git-lfs make ninja-build flex bison jq graphviz \
     clang-format-${LLVM_TOOLS_VERSION} clang-tidy-${LLVM_TOOLS_VERSION} clang-tools-${LLVM_TOOLS_VERSION} \
-    && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-${GCC_VERSION} 100 \
     && update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-${LLVM_TOOLS_VERSION} 100 \
-    && apt-get clean && apt remove -y software-properties-common \
-    && pip3 install  -q --upgrade --no-cache-dir pip \
-    && pip3 install  -q --no-cache-dir conan==1.62 gcovr cmake-format
+    && pip3 install -q --upgrade --no-cache-dir pip && pip3 install -q --no-cache-dir conan==1.62 gcovr cmake cmake-format \
+    && apt-get clean && apt remove -y software-properties-common
+
+# Install gcc-12 and make ldconfig aware of the new libstdc++ location (for gcc)
+# Note: Clang is using libc++ instead
+COPY --from=rippleci/clio_gcc:12.3.0 /gcc12.deb /
+RUN apt update && apt-get install -y binutils libc6-dev \
+    && dpkg -i /gcc12.deb \
+    && rm -rf /gcc12.deb \
+    && ldconfig
+
+# Rewire to use gcc-12 as default compiler
+RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 100 \
+    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
+    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100
 
 WORKDIR /tmp
 
@@ -46,34 +50,51 @@ RUN wget "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/
     && cd "ccache-${CCACHE_VERSION}" \
     && mkdir build && cd build \
     && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install
+    && cmake --build . --target install \
+    && rm -rf /tmp/* /var/tmp/*
 
-# Install doxygen from sounce
+# Install doxygen from source
 RUN wget "https://github.com/doxygen/doxygen/releases/download/Release_${DOXYGEN_VERSION//./_}/doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
     && tar xf "doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
     && cd "doxygen-${DOXYGEN_VERSION}" \
     && mkdir build && cd build \
     && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install
+    && cmake --build . --target install \
+    && rm -rf /tmp/* /var/tmp/*
 
 # Install gh
 RUN wget https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
     && tar xf gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
-    && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh
+    && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh \
+    && rm -rf /tmp/* /var/tmp/*
 
-# Clean up
-RUN rm -rf  /tmp/* /var/tmp/*
-
-WORKDIR /root/
+WORKDIR /root
 # Using root by default is not very secure but github checkout action doesn't work with any other user
 # https://github.com/actions/checkout/issues/956
 # And Github Actions doc recommends using root
 # https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
 
 # Setup conan
-RUN conan profile new default --detect \
-    && conan profile update settings.compiler.cppstd=20 default \
-    && conan profile update settings.compiler.libcxx=libstdc++11 default \
-    && conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+RUN conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
 
+# Note: intentionally leaving cppstd=20
+RUN conan profile new gcc --detect \
+    && conan profile update settings.compiler=gcc gcc \
+    && conan profile update settings.compiler.version=12 gcc \
+    && conan profile update settings.compiler.cppstd=20 gcc \
+    && conan profile update settings.compiler.libcxx=libstdc++11 gcc \
+    && conan profile update env.CC=/usr/bin/gcc-12 gcc \
+    && conan profile update env.CXX=/usr/bin/g++-12 gcc \
+    && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/gcc-12\", \"cpp\": \"/usr/bin/g++-12\"}" gcc
 
+RUN conan profile new clang --detect \
+    && conan profile update settings.compiler=clang clang \
+    && conan profile update settings.compiler.version=16 clang \
+    && conan profile update settings.compiler.cppstd=20 clang \
+    && conan profile update settings.compiler.libcxx=libc++ clang \
+    && conan profile update env.CC=/usr/bin/clang-16 clang \
+    && conan profile update env.CXX=/usr/bin/clang++-16 clang \
+    && conan profile update env.CXXFLAGS="-DBOOST_ASIO_DISABLE_CONCEPTS" clang \
+    && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/clang-16\", \"cpp\": \"/usr/bin/clang++-16\"}" clang 
+
+RUN echo "include(gcc)" >> .conan/profiles/default

docker/clio_docker/centos/build_boost.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -exu
-
-#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static
-# it's either those or link=static that halves the failures. probably link=static
-BOOST_VERSION=$1
-BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _)
-echo "BOOST_VERSION: ${BOOST_VERSION}"
-echo "BOOST_VERSION_: ${BOOST_VERSION_}"
-curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz"
-tar zxf "boost_${BOOST_VERSION_}.tar.gz"
-cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc)
-mkdir -p /boost && mv boost /boost && mv stage /boost

docker/clio_docker/centos/build_git_centos7.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-GIT_VERSION="2.37.1"
-curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz
-tar zxvf git-${GIT_VERSION}.tar.gz
-cd git-${GIT_VERSION}
-
-yum install -y centos-release-scl epel-release
-yum update -y
-yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel  libcurl-devel
-
-source /opt/rh/devtoolset-11/enable
-make configure
-./configure
-make git -j$(nproc)
-make install git
-git  --version | cut -d ' ' -f3

docker/clio_docker/centos/dockerfile

@@ -1,34 +0,0 @@
-FROM centos:7
-
-ENV CLIO_DIR=/opt/clio/
-# ENV OPENSSL_DIR=/opt/openssl
-
-RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl
-RUN yum install -y devtoolset-11
-ENV version=3.16
-ENV build=3
-# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
-COPY install_cmake.sh /install_cmake.sh
-RUN /install_cmake.sh 3.16.3 /usr/local
-RUN source /opt/rh/devtoolset-11/enable
-WORKDIR /tmp
-# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR
-COPY build_git_centos7.sh build_git_centos7.sh
-
-RUN ./build_git_centos7.sh
-RUN git clone https://github.com/openssl/openssl
-WORKDIR /tmp/openssl
-RUN git checkout OpenSSL_1_1_1q
-#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic
-RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \
-    make -j $(nproc) && \
-    make install_sw
-WORKDIR /tmp
-RUN git clone https://github.com/xrplf/clio.git
-COPY build_boost.sh build_boost.sh
-ENV OPENSSL_ROOT=/opt/local/openssl
-ENV BOOST_ROOT=/boost
-RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0
-RUN yum install -y bison flex
-RUN source /opt/rh/devtoolset-11/enable && \
-    cd /tmp/clio && cmake -B build -Dtests=0 -Dlocal_libarchive=1 -Dunity=0 -DBUILD_TESTS=0 && cmake --build build --parallel $(nproc)

docker/clio_docker/centos/install_cmake.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-CMAKE_VERSION=${1:-"3.16.3"}
-cd /tmp
-URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
-curl -OJLs $URL
-tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
-ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake

docker/compilers/clang-16/dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:focal 
+ARG DEBIAN_FRONTEND=noninteractive
+ARG TARGETARCH
+
+SHELL ["/bin/bash", "-c"]
+USER root
+WORKDIR /root
+
+ENV CLANG_VERSION=16
+
+RUN apt update -qq \
+    && apt install -qq -y --no-install-recommends --no-install-suggests \
+    wget software-properties-common gnupg
+
+RUN wget https://apt.llvm.org/llvm.sh \
+    && chmod +x llvm.sh \
+    && ./llvm.sh ${CLANG_VERSION} \
+    && rm -rf llvm.sh \
+    && apt-get install -y libc++-16-dev libc++abi-16-dev

docker/compilers/gcc-12/control.m4

@@ -0,0 +1,6 @@
+Package: gcc-12-ubuntu-UBUNTUVERSION
+Version: VERSION
+Architecture: TARGETARCH
+Maintainer: Alex Kremer <akremer@ripple.com>
+Description: Gcc VERSION build for ubuntu UBUNTUVERSION
+Depends: binutils, libc6-dev

docker/compilers/gcc-12/dockerfile

@@ -0,0 +1,74 @@
+FROM ubuntu:focal as build
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG TARGETARCH
+ARG UBUNTU_VERSION=20.04
+ARG GCC_VERSION=12.3.0
+ARG BUILD_VERSION=1
+
+RUN apt update && apt install -y wget build-essential file flex libz-dev libzstd-dev
+RUN wget https://gcc.gnu.org/pub/gcc/releases/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz \
+    && tar xf gcc-$GCC_VERSION.tar.gz \
+    && cd /gcc-$GCC_VERSION && ./contrib/download_prerequisites
+
+RUN mkdir /${TARGETARCH}-gcc-12
+WORKDIR /${TARGETARCH}-gcc-12
+RUN /gcc-$GCC_VERSION/configure \
+    --with-pkgversion="clio-build-$BUILD_VERSION https://github.com/XRPLF/clio" \
+    --enable-languages=c,c++ \
+    --prefix=/usr \
+    --with-gcc-major-version-only \
+    --program-suffix=-12 \
+    --enable-shared \
+    --enable-linker-build-id \
+    --libexecdir=/usr/lib \
+    --without-included-gettext \
+    --enable-threads=posix \
+    --libdir=/usr/lib \
+    --disable-nls \
+    --enable-clocale=gnu \
+    --enable-libstdcxx-backtrace=yes \
+    --enable-libstdcxx-debug \
+    --enable-libstdcxx-time=yes \
+    --with-default-libstdcxx-abi=new \
+    --enable-gnu-unique-object \
+    --disable-vtable-verify \
+    --enable-plugin \
+    --enable-default-pie \
+    --with-system-zlib \
+    --enable-libphobos-checking=release \
+    --with-target-system-zlib=auto \
+    --disable-werror \
+    --enable-cet \
+    --disable-multilib \
+    --without-cuda-driver \
+    --enable-checking=release \
+    && make -j`nproc` \
+    && make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION \
+    && mkdir -p /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64 \
+    && mv /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/lib64/libstdc++.so.6.0.30-gdb.py /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64/libstdc++.so.6.0.30-gdb.py
+
+# Generate deb
+WORKDIR /
+COPY control.m4 /
+COPY ld.so.conf /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/etc/ld.so.conf.d/1-gcc-12.conf
+
+RUN mkdir /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/DEBIAN \
+    && m4 -P -DUBUNTU_VERSION=$UBUNTU_VERSION -DVERSION=$GCC_VERSION-$BUILD_VERSION -DTARGETARCH=$TARGETARCH control.m4 > /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/DEBIAN/control \
+    && dpkg-deb --build --root-owner-group /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION /gcc12.deb
+
+# Create final image
+FROM ubuntu:focal as gcc
+COPY --from=build /gcc12.deb /
+
+# Make gcc-12 available but also leave gcc12.deb for others to copy if needed
+RUN apt update && apt-get install -y binutils libc6-dev \
+    && dpkg -i /gcc12.deb 
+
+RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 100 \
+    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
+    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100

docker/compilers/gcc-12/ld.so.conf

@@ -0,0 +1,2 @@
+# Path to the directory containing libstdc++.so.6
+/usr/lib64

docker/develop/compose.yaml

@@ -0,0 +1,16 @@
+version: '3.7'
+services:
+  clio_develop:
+    image: rippleci/clio_ci:latest
+    volumes:
+      - clio_develop_conan_data:/root/.conan/data
+      - clio_develop_ccache:/root/.ccache
+      - ../../:/root/clio
+      - clio_develop_build:/root/clio/build_docker
+    working_dir: /root/clio/build_docker
+    tty: true
+
+volumes:
+  clio_develop_conan_data:
+  clio_develop_ccache:
+  clio_develop_build:

docker/develop/run

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+script_dir=$(dirname $0)
+
+pushd $script_dir > /dev/null
+
+function start_container {
+    if [ -z "$(docker ps -q -f name=clio_develop)" ]; then
+        docker compose up -d
+    fi
+}
+
+function run {
+    start_container
+    docker compose exec clio_develop "$@"
+}
+
+function stop_container {
+    docker compose down
+}
+
+function open_terminal {
+    start_container
+    docker compose exec clio_develop /bin/bash
+}
+
+function print_help {
+    cat <<EOF
+run: Run a command inside the development container.
+
+Usage:
+    run [options or command]
+
+If no options are provided, the command will be executed inside the container.
+
+Options:
+-h, --help      Show this help message and exit.
+-t, --terminal  Open a terminal inside the container.
+-s, --stop      Stop the container.
+EOF
+}
+
+case $1 in
+    -h|--help)
+        print_help ;;
+
+    -t|--terminal)
+        open_terminal ;;
+
+    -s|--stop)
+        stop_container ;;
+
+    -*)
+        echo "Unknown option: $1"
+        print_help ;;
+
+    *)
+        run "$@" ;;
+esac
+
+popd > /dev/null
+

docker/shared/build_boost.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -exu
-
-#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static
-# it's either those or link=static that halves the failures. probably link=static
-BOOST_VERSION=$1
-BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _)
-echo "BOOST_VERSION: ${BOOST_VERSION}"
-echo "BOOST_VERSION_: ${BOOST_VERSION_}"
-curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz"
-tar zxf "boost_${BOOST_VERSION_}.tar.gz"
-cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc)
-mkdir -p /boost && mv boost /boost && mv stage /boost

docker/shared/install_cmake.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-CMAKE_VERSION=${1:-"3.16.3"}
-cd /tmp
-URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
-curl -OJLs $URL
-tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
-ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake

docker/shared/install_openssl.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-set -e

docker/ubuntu/Dockerfile

@@ -1,24 +0,0 @@
-FROM ubuntu:20.04 AS boost
-
-RUN apt-get update && apt-get install -y build-essential
-ARG BOOST_VERSION_=1_75_0
-ARG BOOST_VERSION=1.75.0
-COPY docker/shared/build_boost.sh .
-RUN apt install -y curl
-RUN ./build_boost.sh ${BOOST_VERSION}
-ENV BOOST_ROOT=/boost
-
-FROM ubuntu:20.04 AS build
-ENV BOOST_ROOT=/boost
-COPY --from=boost /boost /boost
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install --no-install-recommends -y build-essential software-properties-common pkg-config libssl-dev wget curl gpg git zlib1g-dev bison flex autoconf lsb-release
-RUN apt install -y gpg-agent
-RUN wget https://apt.llvm.org/llvm.sh
-RUN chmod +x llvm.sh && ./llvm.sh 14 && ./llvm.sh 15
-# COPY . /clio
-## Install cmake
-ARG CMAKE_VERSION=3.16.3
-COPY docker/shared/install_cmake.sh .
-RUN ./install_cmake.sh ${CMAKE_VERSION}
-ENV PATH="/opt/local/cmake/bin:$PATH"

docs/CMakeLists.txt

@@ -1,8 +1,8 @@
-cmake_minimum_required(VERSION 3.16.3)
+cmake_minimum_required(VERSION 3.20)
 project(docs)
 
-include(${CMAKE_CURRENT_SOURCE_DIR}/../CMake/ClioVersion.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/ClioVersion.cmake)
 
 # Generate `docs` target for doxygen documentation
 # Note: use `cmake --build . --target docs` from your `build` directory to generate the documentation
-include(${CMAKE_CURRENT_SOURCE_DIR}/../CMake/Docs.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/Docs.cmake)

docs/build-clio.md

@@ -6,19 +6,22 @@ Clio is built with [CMake](https://cmake.org/) and uses [Conan](https://conan.io
 
 - [Python 3.7](https://www.python.org/downloads/)
 - [Conan 1.55](https://conan.io/downloads.html)
-- [CMake 3.16](https://cmake.org/download/)
+- [CMake 3.20](https://cmake.org/download/)
 - [**Optional**] [GCovr](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html): needed for code coverage generation
 - [**Optional**] [CCache](https://ccache.dev/): speeds up compilation if you are going to compile Clio often
 
 | Compiler    | Version |
 |-------------|---------|
-| GCC         | 11      |
-| Clang       | 14      |
-| Apple Clang | 14.0.3  |
+| GCC         | 12.3    |
+| Clang       | 16      |
+| Apple Clang | 15      |
 
 ### Conan Configuration
 
-Clio does not require anything but default settings in your (`~/.conan/profiles/default`) Conan profile. It's best to have no extra flags specified.
+Clio does not require anything other than `compiler.cppstd=20` in your (`~/.conan/profiles/default`) Conan profile.
+
+> [!NOTE]
+> Although Clio is built using C++23, it's required to set `compiler.cppstd=20` for the time being as some of Clio's dependencies are not yet capable of building under C++23.
 
 > Mac example:
 
@@ -29,10 +32,12 @@ os_build=Macos
 arch=armv8
 arch_build=armv8
 compiler=apple-clang
-compiler.version=14
+compiler.version=15
 compiler.libcxx=libc++
 build_type=Release
 compiler.cppstd=20
+[conf]
+tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]
 ```
 
 > Linux example:
@@ -44,7 +49,7 @@ os_build=Linux
 arch=x86_64
 arch_build=x86_64
 compiler=gcc
-compiler.version=11
+compiler.version=12
 compiler.libcxx=libstdc++11
 build_type=Release
 compiler.cppstd=20
@@ -88,6 +93,35 @@ If successful, `conan install` will find the required packages and `cmake` will
 > [!TIP]
 > To generate a Code Coverage report, include `-o coverage=True` in the `conan install` command above, along with `-o tests=True` to enable tests. After running the `cmake` commands, execute `make clio_tests-ccov`. The coverage report will be found at `clio_tests-llvm-cov/index.html`.
 
+> [!NOTE]
+> If you've built Clio before and the build is now failing, it's likely due to updated dependencies. Try deleting the build folder and then rerunning the Conan and CMake commands mentioned above.
+
+### Generating API docs for Clio
+
+The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen on your system.
+
+To generate the API docs:
+
+1. First, include `-o docs=True` in the conan install command. For example:
+
+    ```sh
+    mkdir build && cd build
+    conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False -o docs=True
+    ```
+
+2. Once that has completed successfully, run the `cmake` command and add the `--target docs` option:
+
+   ```sh
+   cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
+   cmake --build . --parallel 8 --target docs
+   ```
+
+3. Go to `build/docs/html` to view the generated files.
+
+    Open the `index.html` file in your browser to see the documentation pages.
+
+    ![API index page](./img/doxygen-docs-output.png "API index page")
+
 ## Building Clio with Docker
 
 It is also possible to build Clio using [Docker](https://www.docker.com/) if you don't want to install all the dependencies on your machine.

docs/configure-clio.md

@@ -27,7 +27,7 @@ If you're running Clio and `rippled` on separate machines, in addition to uncomm
 
 2. Open a public, unencrypted WebSocket port on your `rippled` server.
 
-3. In the `rippled` config, change the IP specified for `secure_gateway`, under the `port_grpc` section, to the IP of your Clio server. This entry can take the form of a comma-separated list if you are running multiple Clio nodes.
+3. In the `rippled` config, change the IP specified for `secure_gateway`, under the `port_grpc` and websocket server sections, to the IP of your Clio server. This entry can take the form of a comma-separated list if you are running multiple Clio nodes.
 
 ## Ledger sequence
 
@@ -97,3 +97,11 @@ To enable the caching for a source, `forwarding_cache_timeout` value should be a
 
 `forwarding_cache_timeout` defines for how long (in seconds) a cache entry will be valid after being placed into the cache.
 Zero value turns off the cache feature.
+
+## Graceful shutdown (not fully implemented yet)
+
+Clio can be gracefully shut down by sending a `SIGINT` (Ctrl+C) or `SIGTERM` signal.
+The process will stop accepting new connections and will wait for the time specified in `graceful_period` config value (or 10 seconds by default).
+If Clio finishes all the scheduled operations before the end of the period, it will stop immediately.
+Otherwise, it will wait for the period to finish and then exit without finishing operations.
+If Clio receives a second signal during the period, it will stop immediately.

docs/coverage-report.md

@@ -25,16 +25,7 @@ The above steps are automated into a single target `coverage_report`. The instru
 
 In case of a spurious failure of unit tests, it is possible to re-run the `coverage_report` target without rebuilding the `clio_tests` binary (since it is simply a dependency of the coverage report target).
 
-The default coverage report format is `html-details`, but developers can override it to any of the formats listed in `CMake/CodeCoverage.cmake` by setting `CODE_COVERAGE_REPORT_FORMAT` variable in `cmake`. For example, CI is setting this parameter to `xml` for the [codecov](https://codecov.io) integration.
-
-If some unit tests predictably fail (e.g., due to absence of a Cassandra database), it is possible to set unit tests options in the `CODE_COVERAGE_TESTS_ARGS` cmake variable, as demonstrated below:
-
-```sh
-cd .build
-conan install .. --output-folder . --build missing --settings build_type=Debug -o tests=True -o coverage=True
-cmake -DCODE_COVERAGE_REPORT_FORMAT=json-details -DCMAKE_BUILD_TYPE=Debug -DCODE_COVERAGE_TESTS_ARGS="--gtest_filter=-BackendCassandra*" -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake ..
-cmake --build . --target coverage_report
-```
+The default coverage report format is `html-details`, but developers can override it to any of the formats listed in `cmake/CodeCoverage.cmake` by setting `CODE_COVERAGE_REPORT_FORMAT` variable in `cmake`. For example, CI is setting this parameter to `xml` for the [codecov](https://codecov.io) integration.
 
 After the `coverage_report` target is completed, the generated coverage report will be stored inside the build directory as either:
 

docs/examples/config/example-config.json

@@ -35,7 +35,10 @@
             "grpc_port": "50051"
         }
     ],
-    "forwarding_cache_timeout": 0.250, // in seconds, could be 0, which means no cache
+    "forwarding": {
+        "cache_timeout": 0.250, // in seconds, could be 0, which means no cache
+        "request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds)
+    },
     "dos_guard": {
         // Comma-separated list of IPs to exclude from rate limiting
         "whitelist": [
@@ -64,8 +67,10 @@
         "admin_password": "xrp",
         // If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests
         // It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time
-        "local_amdin": false
+        "local_admin": false
     },
+    // Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet.
+    "graceful_period": 10.0,
     // Overrides log level on a per logging channel.
     // Defaults to global "log_level" for each unspecified channel.
     "log_channels": [
@@ -94,6 +99,15 @@
             "log_level": "trace"
         }
     ],
+    "cache": {
+        // Configure this to use either "num_diffs", "num_cursors_from_diff", or "num_cursors_from_account". By default, Clio uses "num_diffs".
+        "num_diffs": 32, // Generate the cursors from the latest ledger diff, then use the cursors to partition the ledger to load concurrently. The cursors number is affected by the busyness of the network.
+        // "num_cursors_from_diff": 3200, // Read the cursors from the diff table until we have enough cursors to partition the ledger to load concurrently. 
+        // "num_cursors_from_account": 3200, // Read the cursors from the account table until we have enough cursors to partition the ledger to load concurrently.
+        "num_markers": 48, // The number of markers is the number of coroutines to load the cache concurrently.
+        "page_fetch_size": 512, // The number of rows to load for each page.
+        "load": "async" // "sync" to load cache synchronously  or "async" to load cache asynchronously or "none"/"no" to turn off the cache.
+    },
     "prometheus": {
         "enabled": true,
         "compress_reply": true

docs/examples/infrastructure/grafana/clio_dashboard.json

@@ -18,6 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
+  "id": 1,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -85,9 +86,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "10.2.0",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -105,6 +108,372 @@
       "title": "Service state",
       "type": "stat"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "blue",
+                  "index": 0,
+                  "text": "No"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 1,
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 3,
+        "x": 3,
+        "y": 0
+      },
+      "id": 14,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "read_only",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Read only",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "blue",
+                  "index": 0,
+                  "text": "No"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 1,
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 3,
+        "x": 6,
+        "y": 0
+      },
+      "id": 15,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "etl_writing",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Writing data to DB",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "No"
+                },
+                "1": {
+                  "color": "red",
+                  "index": 1,
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 3,
+        "x": 9,
+        "y": 0
+      },
+      "id": 16,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "etl_amendment_blocked",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Amendment blocked",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "timestamp(etl_last_publish_seconds) - etl_last_publish_seconds",
+          "format": "time_series",
+          "instant": false,
+          "legendFormat": "ledger age",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Ledger Age",
+      "transformations": [
+        {
+          "id": "filterByValue",
+          "options": {
+            "filters": [
+              {
+                "config": {
+                  "id": "lower",
+                  "options": {
+                    "value": 31500000
+                  }
+                },
+                "fieldName": "ledger age"
+              }
+            ],
+            "match": "all",
+            "type": "include"
+          }
+        }
+      ],
+      "type": "timeseries"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -166,9 +535,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 9,
-        "x": 3,
-        "y": 0
+        "w": 12,
+        "x": 0,
+        "y": 8
       },
       "id": 2,
       "options": {
@@ -263,7 +632,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 0
+        "y": 8
       },
       "id": 9,
       "options": {
@@ -358,7 +727,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 8
+        "y": 16
       },
       "id": 11,
       "options": {
@@ -453,7 +822,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 8
+        "y": 16
       },
       "id": 6,
       "options": {
@@ -550,7 +919,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 16
+        "y": 24
       },
       "id": 10,
       "options": {
@@ -645,7 +1014,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 16
+        "y": 24
       },
       "id": 8,
       "options": {
@@ -740,7 +1109,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 24
+        "y": 32
       },
       "id": 4,
       "options": {
@@ -839,7 +1208,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 24
+        "y": 32
       },
       "id": 12,
       "options": {
@@ -973,7 +1342,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 32
+        "y": 40
       },
       "id": 5,
       "options": {
@@ -1081,7 +1450,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 32
+        "y": 40
       },
       "id": 3,
       "options": {
@@ -1186,9 +1555,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 10,
+        "w": 12,
         "x": 0,
-        "y": 40
+        "y": 48
       },
       "id": 7,
       "options": {
@@ -1222,7 +1591,7 @@
     }
   ],
   "refresh": "5s",
-  "schemaVersion": 38,
+  "schemaVersion": 39,
   "tags": [],
   "templating": {
     "list": []
@@ -1235,6 +1604,6 @@
   "timezone": "",
   "title": "Clio",
   "uid": "aeaae84e-c194-47b2-ad65-86e45eebb815",
-  "version": 3,
+  "version": 1,
   "weekStart": ""
 }

docs/img/doxygen-docs-output.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16e6f0a44b2dc7462f813f07ba754575e93033bc8bbfbe1d28f8bf1927915028
+size 394711

docs/trouble_shooting.md

@@ -0,0 +1,47 @@
+# Troubleshooting Guide
+This guide will help you troubleshoot common issues of Clio.
+
+## Can't connect to DB
+If you see the error log message `Could not connect to Cassandra: No hosts available`, this means that Clio can't connect to the database. Check the following:
+- Make sure the database is running at the specified address and port.
+- Make sure the database is accessible from the machine where Clio is running.
+You can use [cqlsh](https://pypi.org/project/cqlsh/) to check the connection to the database.
+If you would like to run a local ScyllaDB, you can call:
+```sh
+docker run --rm -p 9042:9042 --name clio-scylla -d scylladb/scylla 
+```
+
+## Check the server status of Clio
+To check if Clio is syncing with rippled:
+```sh
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep seq
+```
+If Clio is syncing with rippled, the `seq` value will be increasing.
+
+## Clio fails to start
+If you see the error log message `Failed to fetch ETL state from...`, this means the configured rippled node is not reachable. Check the following:
+- Make sure the rippled node is running at the specified address and port.
+- Make sure the rippled node is accessible from the machine where Clio is running.
+
+If you would like to run Clio without an avaliable rippled node, you can add below setting to Clio's configuration file:
+```
+"allow_no_etl": true
+```
+
+## Clio is not added to secure_gateway in rippled's config
+If you see the warning message `AsyncCallData is_unlimited is false.`, this means that Clio is not added to the `secure_gateway` of `port_grpc` session in the rippled configuration file. It will slow down the sync process. Please add Clio's IP to the `secure_gateway` in the rippled configuration file for both grpc and ws port.
+
+## Clio is slow
+To speed up the response time, Clio has a cache inside. However, cache can take time to warm up. If you see slow response time, you can firstly check if cache is still loading.
+You can check the cache status by calling:
+```sh
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_full
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_enabled
+```
+If `is_full` is false, it means the cache is still loading. Normally, the Clio can respond quicker if cache finishs loading. If `is_enabled` is false, it means the cache is disabled in the configuration file or there is data corruption in the database. 
+
+## Receive error message `Too many requests`
+If client sees the error message `Too many requests`, this means that the client is blocked by Clio's DosGuard protection. You may want to add the client's IP to the whitelist in the configuration file, Or update other your DosGuard settings.
+
+
+

src/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_subdirectory(util)
+add_subdirectory(data)
+add_subdirectory(etl)
+add_subdirectory(feed)
+add_subdirectory(rpc)
+add_subdirectory(web)
+add_subdirectory(main)

src/data/BackendCounters.hpp

@@ -39,39 +39,17 @@ namespace data {
 template <typename T>
 concept SomeBackendCounters = requires(T a) {
     typename T::PtrType;
-    {
-        a.registerTooBusy()
-    } -> std::same_as<void>;
-    {
-        a.registerWriteSync(std::chrono::steady_clock::time_point{})
-    } -> std::same_as<void>;
-    {
-        a.registerWriteSyncRetry()
-    } -> std::same_as<void>;
-    {
-        a.registerWriteStarted()
-    } -> std::same_as<void>;
-    {
-        a.registerWriteFinished(std::chrono::steady_clock::time_point{})
-    } -> std::same_as<void>;
-    {
-        a.registerWriteRetry()
-    } -> std::same_as<void>;
-    {
-        a.registerReadStarted(std::uint64_t{})
-    } -> std::same_as<void>;
-    {
-        a.registerReadFinished(std::chrono::steady_clock::time_point{}, std::uint64_t{})
-    } -> std::same_as<void>;
-    {
-        a.registerReadRetry(std::uint64_t{})
-    } -> std::same_as<void>;
-    {
-        a.registerReadError(std::uint64_t{})
-    } -> std::same_as<void>;
-    {
-        a.report()
-    } -> std::same_as<boost::json::object>;
+    { a.registerTooBusy() } -> std::same_as<void>;
+    { a.registerWriteSync(std::chrono::steady_clock::time_point{}) } -> std::same_as<void>;
+    { a.registerWriteSyncRetry() } -> std::same_as<void>;
+    { a.registerWriteStarted() } -> std::same_as<void>;
+    { a.registerWriteFinished(std::chrono::steady_clock::time_point{}) } -> std::same_as<void>;
+    { a.registerWriteRetry() } -> std::same_as<void>;
+    { a.registerReadStarted(std::uint64_t{}) } -> std::same_as<void>;
+    { a.registerReadFinished(std::chrono::steady_clock::time_point{}, std::uint64_t{}) } -> std::same_as<void>;
+    { a.registerReadRetry(std::uint64_t{}) } -> std::same_as<void>;
+    { a.registerReadError(std::uint64_t{}) } -> std::same_as<void>;
+    { a.report() } -> std::same_as<boost::json::object>;
 };
 
 /**

src/data/BackendInterface.cpp

@@ -90,7 +90,7 @@ BackendInterface::fetchLedgerObject(
     auto obj = cache_.get(key, sequence);
     if (obj) {
         LOG(gLog.trace()) << "Cache hit - " << ripple::strHex(key);
-        return *obj;
+        return obj;
     }
 
     LOG(gLog.trace()) << "Cache miss - " << ripple::strHex(key);
@@ -297,16 +297,23 @@ BackendInterface::fetchLedgerPage(
     std::uint32_t const limit,
     bool outOfOrder,
     boost::asio::yield_context yield
-) const
+)
 {
     LedgerPage page;
 
     std::vector<ripple::uint256> keys;
     bool reachedEnd = false;
+
     while (keys.size() < limit && !reachedEnd) {
-        ripple::uint256 const& curCursor = !keys.empty() ? keys.back() : (cursor ? *cursor : firstKey);
+        ripple::uint256 const& curCursor = [&]() {
+            if (!keys.empty())
+                return keys.back();
+            return (cursor ? *cursor : firstKey);
+        }();
+
         std::uint32_t const seq = outOfOrder ? range->maxSequence : ledgerSequence;
         auto succ = fetchSuccessorKey(curCursor, seq, yield);
+
         if (!succ) {
             reachedEnd = true;
         } else {
@@ -326,6 +333,9 @@ BackendInterface::fetchLedgerPage(
                 msg << " - " << ripple::strHex(keys[j]);
             }
             LOG(gLog.error()) << msg.str();
+
+            if (corruptionDetector_.has_value())
+                corruptionDetector_->onCorruptionDetected();
         }
     }
     if (!keys.empty() && !reachedEnd)
@@ -350,14 +360,42 @@ BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context
     ripple::SerialIter it(bytes->data(), bytes->size());
     ripple::SLE const sle{it, key};
 
-    if (sle.getFieldIndex(ripple::sfBaseFee) != -1)
-        fees.base = sle.getFieldU64(ripple::sfBaseFee);
+    // XRPFees amendment introduced new fields for fees calculations.
+    // New fields are set and the old fields are removed via `set_fees` tx.
+    // Fallback to old fields if `set_fees` was not yet used to update the fields on this tx.
+    auto hasNewFields = false;
+    {
+        auto const baseFeeXRP = sle.at(~ripple::sfBaseFeeDrops);
+        auto const reserveBaseXRP = sle.at(~ripple::sfReserveBaseDrops);
+        auto const reserveIncrementXRP = sle.at(~ripple::sfReserveIncrementDrops);
 
-    if (sle.getFieldIndex(ripple::sfReserveBase) != -1)
-        fees.reserve = sle.getFieldU32(ripple::sfReserveBase);
+        if (baseFeeXRP)
+            fees.base = baseFeeXRP->xrp();
 
-    if (sle.getFieldIndex(ripple::sfReserveIncrement) != -1)
-        fees.increment = sle.getFieldU32(ripple::sfReserveIncrement);
+        if (reserveBaseXRP)
+            fees.reserve = reserveBaseXRP->xrp();
+
+        if (reserveIncrementXRP)
+            fees.increment = reserveIncrementXRP->xrp();
+
+        hasNewFields = baseFeeXRP || reserveBaseXRP || reserveIncrementXRP;
+    }
+
+    if (not hasNewFields) {
+        // Fallback to old fields
+        auto const baseFee = sle.at(~ripple::sfBaseFee);
+        auto const reserveBase = sle.at(~ripple::sfReserveBase);
+        auto const reserveIncrement = sle.at(~ripple::sfReserveIncrement);
+
+        if (baseFee)
+            fees.base = baseFee.value();
+
+        if (reserveBase)
+            fees.reserve = reserveBase.value();
+
+        if (reserveIncrement)
+            fees.increment = reserveIncrement.value();
+    }
 
     return fees;
 }

src/data/BackendInterface.hpp

@@ -22,6 +22,7 @@
 #include "data/DBHelpers.hpp"
 #include "data/LedgerCache.hpp"
 #include "data/Types.hpp"
+#include "etl/CorruptionDetector.hpp"
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/executor_work_guard.hpp>
@@ -44,6 +45,7 @@
 #include <string>
 #include <thread>
 #include <type_traits>
+#include <utility>
 #include <vector>
 
 namespace data {
@@ -138,6 +140,7 @@ protected:
     mutable std::shared_mutex rngMtx_;
     std::optional<LedgerRange> range;
     LedgerCache cache_;
+    std::optional<etl::CorruptionDetector<LedgerCache>> corruptionDetector_;
 
 public:
     BackendInterface() = default;
@@ -162,6 +165,17 @@ public:
         return cache_;
     }
 
+    /**
+     * @brief Sets the corruption detector.
+     *
+     * @param detector The corruption detector to set
+     */
+    void
+    setCorruptionDetector(etl::CorruptionDetector<LedgerCache> detector)
+    {
+        corruptionDetector_ = std::move(detector);
+    }
+
     /**
      * @brief Fetches a specific ledger by sequence number.
      *
@@ -199,6 +213,19 @@ public:
     std::optional<LedgerRange>
     fetchLedgerRange() const;
 
+    /**
+     * @brief Fetch the specified number of account root object indexes by page, the accounts need to exist for seq.
+     *
+     * @param number The number of accounts to fetch
+     * @param pageSize The maximum number of accounts per page
+     * @param seq The accounts need to exist for this sequence
+     * @param yield The coroutine context
+     * @return A vector of ripple::uint256 representing the account roots
+     */
+    virtual std::vector<ripple::uint256>
+    fetchAccountRoots(std::uint32_t number, std::uint32_t pageSize, std::uint32_t seq, boost::asio::yield_context yield)
+        const = 0;
+
     /**
      * @brief Updates the range of sequences that are stored in the DB.
      *
@@ -422,7 +449,7 @@ public:
         std::uint32_t limit,
         bool outOfOrder,
         boost::asio::yield_context yield
-    ) const;
+    );
 
     /**
      * @brief Fetches the successor object.

src/data/CMakeLists.txt

@@ -0,0 +1,17 @@
+add_library(clio_data)
+target_sources(
+  clio_data
+  PRIVATE BackendCounters.cpp
+          BackendInterface.cpp
+          LedgerCache.cpp
+          cassandra/impl/Future.cpp
+          cassandra/impl/Cluster.cpp
+          cassandra/impl/Batch.cpp
+          cassandra/impl/Result.cpp
+          cassandra/impl/Tuple.cpp
+          cassandra/impl/SslContext.cpp
+          cassandra/Handle.cpp
+          cassandra/SettingsProvider.cpp
+)
+
+target_link_libraries(clio_data PUBLIC cassandra-cpp-driver::cassandra-cpp-driver clio_util)

src/data/CassandraBackend.hpp

@@ -40,6 +40,7 @@
 #include <ripple/basics/base_uint.h>
 #include <ripple/basics/strHex.h>
 #include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/Indexes.h>
 #include <ripple/protocol/LedgerHeader.h>
 #include <ripple/protocol/nft.h>
 
@@ -171,11 +172,6 @@ public:
             if (--numRows == 0) {
                 LOG(log_.debug()) << "Setting cursor";
                 cursor = data;
-
-                // forward queries by ledger/tx sequence `>=`
-                // so we have to advance the index by one
-                if (forward)
-                    ++cursor->transactionIndex;
             }
         }
 
@@ -210,13 +206,13 @@ public:
     }
 
     void
-    writeLedger(ripple::LedgerHeader const& ledgerInfo, std::string&& blob) override
+    writeLedger(ripple::LedgerHeader const& ledgerHeader, std::string&& blob) override
     {
-        executor_.write(schema_->insertLedgerHeader, ledgerInfo.seq, std::move(blob));
+        executor_.write(schema_->insertLedgerHeader, ledgerHeader.seq, std::move(blob));
 
-        executor_.write(schema_->insertLedgerHash, ledgerInfo.hash, ledgerInfo.seq);
+        executor_.write(schema_->insertLedgerHash, ledgerHeader.hash, ledgerHeader.seq);
 
-        ledgerSequence_ = ledgerInfo.seq;
+        ledgerSequence_ = ledgerHeader.seq;
     }
 
     std::optional<std::uint32_t>
@@ -560,7 +556,7 @@ public:
         if (auto const res = executor_.read(yield, schema_->selectObject, key, sequence); res) {
             if (auto const result = res->template get<Blob>(); result) {
                 if (result->size())
-                    return *result;
+                    return result;
             } else {
                 LOG(log_.debug()) << "Could not fetch ledger object - no rows";
             }
@@ -596,7 +592,7 @@ public:
             if (auto const result = res->template get<ripple::uint256>(); result) {
                 if (*result == lastKey)
                     return std::nullopt;
-                return *result;
+                return result;
             }
 
             LOG(log_.debug()) << "Could not fetch successor - no rows";
@@ -693,6 +689,50 @@ public:
         return results;
     }
 
+    std::vector<ripple::uint256>
+    fetchAccountRoots(std::uint32_t number, std::uint32_t pageSize, std::uint32_t seq, boost::asio::yield_context yield)
+        const override
+    {
+        std::vector<ripple::uint256> liveAccounts;
+        std::optional<ripple::AccountID> lastItem;
+
+        while (liveAccounts.size() < number) {
+            Statement const statement = lastItem ? schema_->selectAccountFromToken.bind(*lastItem, Limit{pageSize})
+                                                 : schema_->selectAccountFromBegining.bind(Limit{pageSize});
+
+            auto const res = executor_.read(yield, statement);
+            if (res) {
+                auto const& results = res.value();
+                if (not results.hasRows()) {
+                    LOG(log_.debug()) << "No rows returned";
+                    break;
+                }
+                // The results should not contain duplicates, we just filter out deleted accounts
+                std::vector<ripple::uint256> fullAccounts;
+                for (auto [account] : extract<ripple::AccountID>(results)) {
+                    fullAccounts.push_back(ripple::keylet::account(account).key);
+                    lastItem = account;
+                }
+                auto const objs = doFetchLedgerObjects(fullAccounts, seq, yield);
+
+                for (auto i = 0u; i < fullAccounts.size(); i++) {
+                    if (not objs[i].empty()) {
+                        if (liveAccounts.size() < number) {
+                            liveAccounts.push_back(fullAccounts[i]);
+                        } else {
+                            break;
+                        }
+                    }
+                }
+            } else {
+                LOG(log_.error()) << "Could not fetch account from account_tx: " << res.error();
+                break;
+            }
+        }
+
+        return liveAccounts;
+    }
+
     std::vector<LedgerObject>
     fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override
     {
@@ -732,9 +772,7 @@ public:
             std::cend(keys),
             std::cbegin(objs),
             std::back_inserter(results),
-            [](auto const& key, auto const& obj) {
-                return LedgerObject{key, obj};
-            }
+            [](auto const& key, auto const& obj) { return LedgerObject{key, obj}; }
         );
 
         return results;

src/data/LedgerCache.cpp

@@ -45,6 +45,7 @@ LedgerCache::waitUntilCacheContainsSeq(uint32_t seq)
 {
     if (disabled_)
         return;
+
     std::unique_lock lock(mtx_);
     cv_.wait(lock, [this, seq] { return latestSeq_ >= seq; });
     return;
@@ -89,8 +90,9 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
 std::optional<LedgerObject>
 LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
 {
-    if (!full_)
+    if (disabled_ or not full_)
         return {};
+
     std::shared_lock const lck{mtx_};
     ++successorReqCounter_.get();
     if (seq != latestSeq_)
@@ -105,8 +107,9 @@ LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
 std::optional<LedgerObject>
 LedgerCache::getPredecessor(ripple::uint256 const& key, uint32_t seq) const
 {
-    if (!full_)
+    if (disabled_ or not full_)
         return {};
+
     std::shared_lock const lck{mtx_};
     if (seq != latestSeq_)
         return {};
@@ -120,6 +123,9 @@ LedgerCache::getPredecessor(ripple::uint256 const& key, uint32_t seq) const
 std::optional<Blob>
 LedgerCache::get(ripple::uint256 const& key, uint32_t seq) const
 {
+    if (disabled_)
+        return {};
+
     std::shared_lock const lck{mtx_};
     if (seq > latestSeq_)
         return {};
@@ -139,6 +145,12 @@ LedgerCache::setDisabled()
     disabled_ = true;
 }
 
+bool
+LedgerCache::isDisabled() const
+{
+    return disabled_;
+}
+
 void
 LedgerCache::setFull()
 {

src/data/LedgerCache.hpp

@@ -133,6 +133,12 @@ public:
     void
     setDisabled();
 
+    /**
+     * @return true if the cache is disabled; false otherwise
+     */
+    bool
+    isDisabled() const;
+
     /**
      * @brief Sets the full flag to true.
      *

src/data/README.md

@@ -1,42 +1,75 @@
 # Backend
 
-## Background
-The backend of Clio is responsible for handling the proper reading and writing of past ledger data from and to a given database. As of right now, Cassandra and ScyllaDB are the only supported databases that are production-ready. Support for database types can be easily extended by creating new implementations which implements the virtual methods of `BackendInterface.h`. Then, use the Factory Object Design Pattern to simply add logic statements to `BackendFactory.h` that return the new database interface for a specific `type` in Clio's configuration file. 
+The backend of Clio is responsible for handling the proper reading and writing of past ledger data from and to a given database. Currently, Cassandra and ScyllaDB are the only supported databases that are production-ready.
+
+To support additional database types, you can create new classes that implement the virtual methods in [BackendInterface.h](https://github.com/XRPLF/clio/blob/develop/src/data/BackendInterface.hpp). Then, leveraging the Factory Object Design Pattern, modify [BackendFactory.h](https://github.com/XRPLF/clio/blob/develop/src/data/BackendFactory.hpp) with logic that returns the new database interface if the relevant `type` is provided in Clio's configuration file.
 
 ## Data Model
-The data model used by Clio to read and write ledger data is different from what Rippled uses. Rippled uses a novel data structure named [*SHAMap*](https://github.com/ripple/rippled/blob/master/src/ripple/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record. Rippled nodes can also generate a proof-tree by forming a subtree with all the path nodes and their neighbors, which can then be used to prove the existnce of the leaf node data to other Rippled nodes. In short, the main purpose of the SHAMap data structure is to facilitate the fast validation of data integrity between different decentralized Rippled nodes.
 
-Since Clio only extracts past validated ledger data from a group of trusted Rippled nodes, it can be safely assumed that these ledger data are correct without the need to validate with other  nodes in the XRP peer-to-peer network. Because of this, Clio is able to use a flattened data model to store the past validated ledger data, which allows for direct record lookup with much faster constant time operations. 
+The data model used by Clio to read and write ledger data is different from what `rippled` uses. `rippled` uses a novel data structure named [*SHAMap*](https://github.com/ripple/rippled/blob/master/src/ripple/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record.
 
-There are three main types of data in each XRP ledger version, they are [Ledger Header](https://xrpl.org/ledger-header.html), [Transaction Set](https://xrpl.org/transaction-formats.html) and [State Data](https://xrpl.org/ledger-object-types.html). Due to the structural differences of the different types of databases, Clio may choose to represent these data using a different schema for each unique database type. 
+`rippled` nodes can also generate a proof-tree by forming a subtree with all the path nodes and their neighbors, which can then be used to prove the existence of the leaf node data to other `rippled` nodes. In short, the main purpose of the SHAMap data structure is to facilitate the fast validation of data integrity between different decentralized `rippled` nodes.
 
-**Keywords**  
-*Sequence*: A unique incrementing identification number used to label the different ledger versions.  
-*Hash*: The SHA512-half (calculate SHA512 and take the first 256 bits) hash of various ledger data like the entire ledger or specific ledger objects.
-*Ledger Object*: The [binary-encoded](https://xrpl.org/serialization.html) STObject containing specific data (i.e. metadata, transaction data).  
-*Metadata*: The data containing [detailed information](https://xrpl.org/transaction-metadata.html#transaction-metadata) of the outcome of a specific transaction, regardless of whether the transaction was successful.  
-*Transaction data*: The data containing the [full details](https://xrpl.org/transaction-common-fields.html) of a specific transaction.  
-*Object Index*: The pseudo-random unique identifier of a ledger object, created by hashing the data of the object.  
+Since Clio only extracts past validated ledger data from a group of trusted `rippled` nodes, it can be safely assumed that the ledger data is correct without the need to validate with other nodes in the XRP peer-to-peer network. Because of this, Clio is able to use a flattened data model to store the past validated ledger data, which allows for direct record lookup with much faster constant time operations.
+
+There are three main types of data in each XRP Ledger version:
+
+- [Ledger Header](https://xrpl.org/ledger-header.html)
+
+- [Transaction Set](https://xrpl.org/transaction-formats.html)
+
+- [State Data](https://xrpl.org/ledger-object-types.html)
+
+Due to the structural differences of the different types of databases, Clio may choose to represent these data types using a different schema for each unique database type.
+
+### Keywords  
+
+**Sequence**: A unique incrementing identification number used to label the different ledger versions.
+
+**Hash**: The SHA512-half (calculate SHA512 and take the first 256 bits) hash of various ledger data like the entire ledger or specific ledger objects.
+
+**Ledger Object**: The [binary-encoded](https://xrpl.org/serialization.html) STObject containing specific data (i.e. metadata, transaction data).  
+
+**Metadata**: The data containing [detailed information](https://xrpl.org/transaction-metadata.html#transaction-metadata) of the outcome of a specific transaction, regardless of whether the transaction was successful.  
+
+**Transaction data**: The data containing the [full details](https://xrpl.org/transaction-common-fields.html) of a specific transaction.  
+
+**Object Index**: The pseudo-random unique identifier of a ledger object, created by hashing the data of the object.  
 
 ## Cassandra Implementation
-Cassandra is a distributed wide-column NoSQL database designed to handle large data throughput with high availability and no single point of failure. By leveraging Cassandra, Clio will be able to quickly and reliably scale up when needed simply by adding more Cassandra nodes to the Cassandra cluster configuration.
 
-In Cassandra, Clio will be creating 9 tables to store the ledger data, they are `ledger_transactions`, `transactions`, `ledger_hashes`, `ledger_range`, `objects`, `ledgers`, `diff`, `account_tx`, and `successor`.  Their schemas and how they work are detailed below.
+Cassandra is a distributed wide-column NoSQL database designed to handle large data throughput with high availability and no single point of failure. By leveraging Cassandra, Clio is able to quickly and reliably scale up when needed simply by adding more Cassandra nodes to the Cassandra cluster configuration.
 
-*Note, if you would like visually explore the data structure of the Cassandra database, you can first run Clio server with database `type` configured as `cassandra` to fill ledger data from Rippled nodes into Cassandra, then use a GUI database management tool like [Datastax's Opcenter](https://docs.datastax.com/en/install/6.0/install/opscInstallOpsc.html) to interactively view it.* 
+In Cassandra, Clio creates 9 tables to store the ledger data:
 
+- `ledger_transactions`
+- `transactions`
+- `ledger_hashes`
+- `ledger_range`
+- `objects`
+- `ledgers`
+- `diff`
+- `account_tx`
+- `successor`
+
+Their schemas and how they work are detailed in the following sections.
+
+> **Note**: If you would like visually explore the data structure of the Cassandra database, run the Clio server with the database `type` configured as `cassandra` to fill ledger data from the `rippled` nodes into Cassandra. Then, use a GUI database management tool like [Datastax's Opcenter](https://docs.datastax.com/en/install/6.0/install/opscInstallOpsc.html) to interactively view it.
 
 ### ledger_transactions
+
 ```
 CREATE TABLE clio.ledger_transactions (  
 	ledger_sequence bigint,  # The sequence number of the ledger version
 	hash blob,               # Hash of all the transactions on this ledger version
 	PRIMARY KEY (ledger_sequence, hash)  
 ) WITH CLUSTERING ORDER BY (hash ASC) ...
- ```
-This table stores the hashes of all transactions in a given ledger sequence ordered by the hash value in ascending order. 
+```
+
+This table stores the hashes of all transactions in a given ledger sequence and is sorted by the hash value in ascending order.
 
 ### transactions
+
 ```
 CREATE TABLE clio.transactions (  
 	hash blob PRIMARY KEY,   # The transaction hash
@@ -45,29 +78,36 @@ CREATE TABLE clio.transactions (
 	metadata blob,           # Metadata of the transaction
 	transaction blob         # Data of the transaction
 ) ...
- ```
+```
+
 This table stores the full transaction and metadata of each ledger version with the transaction hash as the primary key.
 
-To look up all the transactions that were validated in a ledger version with sequence `n`, one can first get the all the transaction hashes in that ledger version by querying `SELECT * FROM ledger_transactions WHERE ledger_sequence = n;`. Then, iterate through the list of hashes and query `SELECT * FROM transactions WHERE hash = one_of_the_hash_from_the_list;` to get the detailed transaction data.  
+To lookup all the transactions that were validated in a ledger version with sequence `n`, first get the all the transaction hashes in that ledger version by querying `SELECT * FROM ledger_transactions WHERE ledger_sequence = n;`. Then, iterate through the list of hashes and query `SELECT * FROM transactions WHERE hash = one_of_the_hash_from_the_list;` to get the detailed transaction data.  
 
 ### ledger_hashes
+
 ```
 CREATE TABLE clio.ledger_hashes (
 	hash blob PRIMARY KEY,  # Hash of entire ledger version's data
 	sequence bigint         # The sequence of the ledger version
 ) ...
- ```
+```
+
 This table stores the hash of all ledger versions by their sequences. 
+
 ### ledger_range
+
 ```
 CREATE TABLE clio.ledger_range (
 	is_latest boolean PRIMARY KEY,  # Whether this sequence is the stopping range
 	sequence bigint                 # The sequence number of the starting/stopping range
 ) ...
- ```
-This table marks the range of ledger versions that is stored on this specific Cassandra node. Because of its nature, there are only two records in this table with `false` and `true` values for `is_latest`, marking the starting and ending sequence of the ledger range. 
+```
+
+This table marks the range of ledger versions that is stored on this specific Cassandra node. Because of its nature, there are only two records in this table with `false` and `true` values for `is_latest`, marking the starting and ending sequence of the ledger range.
 
 ### objects
+
 ```
 CREATE TABLE clio.objects (
 	key blob,         # Object index of the object
@@ -75,31 +115,37 @@ CREATE TABLE clio.objects (
 	object blob,      # Data of the object
 	PRIMARY KEY (key, sequence)
 ) WITH CLUSTERING ORDER BY (sequence DESC) ...
- ```
-This table stores the specific data of all objects that ever existed on the XRP network, even if they are deleted (which is represented with a special `0x` value). The records are ordered by descending sequence, where the newest validated ledger objects are at the top. 
+```
 
-This table is updated when all data for a given ledger sequence has been written to the various tables in the database. For each ledger, many associated records are written to different tables. This table is used as a synchronization mechanism, to prevent the application from reading data from a ledger for which all data has not yet been fully written.
+The `objects` table stores the specific data of all objects that ever existed on the XRP network, even if they are deleted (which is represented with a special `0x` value). The records are ordered by descending sequence, where the newest validated ledger objects are at the top.
+
+The table is updated when all data for a given ledger sequence has been written to the various tables in the database. For each ledger, many associated records are written to different tables. This table is used as a synchronization mechanism, to prevent the application from reading data from a ledger for which all data has not yet been fully written.
 
 ### ledgers
+
 ```
 CREATE TABLE clio.ledgers (
 	sequence bigint PRIMARY KEY,  # Sequence of the ledger version
 	header blob                   # Data of the header
 ) ...
- ```
+```
+
 This table stores the ledger header data of specific ledger versions by their sequence.
 
 ### diff
+
 ```
 CREATE TABLE clio.diff (
 	seq bigint,  # Sequence of the ledger version
 	key blob,    # Hash of changes in the ledger version
 	PRIMARY KEY (seq, key)
 ) WITH CLUSTERING ORDER BY (key ASC) ...
- ```
+```
+
 This table stores the object index of all the changes in each ledger version.
 
 ### account_tx
+
 ```
 CREATE TABLE clio.account_tx (
 	account blob,
@@ -107,10 +153,12 @@ CREATE TABLE clio.account_tx (
 	hash blob,                              # Hash of the transaction
 	PRIMARY KEY (account, seq_idx)
 ) WITH CLUSTERING ORDER BY (seq_idx DESC) ...
- ```
+```
+
 This table stores the list of transactions affecting a given account. This includes transactions made by the account, as well as transactions received.
 
 ### successor
+
 ```
 CREATE TABLE clio.successor (
 	key blob,    # Object index
@@ -118,30 +166,35 @@ CREATE TABLE clio.successor (
 	next blob,   # Index of the next object that existed in this sequence
 	PRIMARY KEY (key, seq)
 ) WITH CLUSTERING ORDER BY (seq ASC) ...
-  ```
-This table is the important backbone of how histories of ledger objects are stored in Cassandra. The successor table stores the object index of all ledger objects that were validated on the XRP network along with the ledger sequence that the object was upated on. Due to the unique nature of the table with each key being ordered by the sequence, by tracing through the table with a specific sequence number, Clio can recreate a Linked List data structure that represents all the existing ledger object at that ledger sequence. The special value of `0x00...00` and `0xFF...FF` are used to label the head and tail of the Linked List in the successor table. The diagram below showcases how tracing through the same table but with different sequence parameter filtering can result in different Linked List data representing the corresponding past state of the ledger objects. A query like `SELECT * FROM successor WHERE key = ? AND seq <= n ORDER BY seq DESC LIMIT 1;` can effectively trace through the successor table and get the Linked List of a specific sequence `n`.
+```
+
+This table is the important backbone of how histories of ledger objects are stored in Cassandra. The `successor` table stores the object index of all ledger objects that were validated on the XRP network along with the ledger sequence that the object was updated on.
+
+As each key is ordered by the sequence, which is achieved by tracing through the table with a specific sequence number, Clio can recreate a Linked List data structure that represents all the existing ledger objects at that ledger sequence. The special values of `0x00...00` and `0xFF...FF` are used to label the *head* and *tail* of the Linked List in the successor table.
+
+The diagram below showcases how tracing through the same table, but with different sequence parameter filtering, can result in different Linked List data representing the corresponding past state of the ledger objects. A query like `SELECT * FROM successor WHERE key = ? AND seq <= n ORDER BY seq DESC LIMIT 1;` can effectively trace through the successor table and get the Linked List of a specific sequence `n`.
 
 ![Successor Table Trace Diagram](https://raw.githubusercontent.com/Shoukozumi/clio/9b2ea3efb6b164b02e9a5f0ef6717065a70f078c/src/backend/README.png)
-*P.S.: The `diff` is `(DELETE 0x00...02, CREATE 0x00...03)` for `seq=1001` and `(CREATE 0x00...04)` for `seq=1002`, which is both accurately reflected with the Linked List trace*
 
-In each new ledger version with sequence `n`, a ledger object `v` can either be **created**, **modified**, or **deleted**. For all three of these operations, the procedure to update the successor table can be broken down in to two steps: 
- 1. Trace through the Linked List of the previous sequence to to find the ledger object `e` with the greatest object index smaller or equal than the `v`'s index. Save `e`'s `next` value (the index of the next ledger object) as `w`.
+> **Note**: The `diff` is `(DELETE 0x00...02, CREATE 0x00...03)` for `seq=1001` and `(CREATE 0x00...04)` for `seq=1002`, which is both accurately reflected with the Linked List trace.
+
+In each new ledger version with sequence `n`, a ledger object `v` can either be **created**, **modified**, or **deleted**.
+
+For all three of these operations, the procedure to update the successor table can be broken down into two steps:
+
+ 1. Trace through the Linked List of the previous sequence to find the ledger object `e` with the greatest object index smaller or equal than the `v`'s index. Save `e`'s `next` value (the index of the next ledger object) as `w`.
+
  2. If `v` is...
 	 1. Being **created**, add two new records of `seq=n` with one being `e` pointing to `v`, and `v` pointing to `w` (Linked List insertion operation).
 	 2. Being **modified**, do nothing.
 	 3. Being **deleted**, add a record of `seq=n` with `e` pointing to `v`'s `next` value (Linked List deletion operation).
 
-### NFT data model
-In `rippled` NFTs are stored in NFTokenPage ledger objects. This object is
-implemented to save ledger space and has the property that it gives us O(1)
-lookup time for an NFT, assuming we know who owns the NFT at a particular
-ledger. However, if we do not know who owns the NFT at a specific ledger
-height we have no alternative in rippled other than scanning the entire
-ledger. Because of this tradeoff, clio implements a special NFT indexing data
-structure that allows clio users to query NFTs quickly, while keeping
-rippled's space-saving optimizations.
+## NFT data model
+
+In `rippled` NFTs are stored in `NFTokenPage` ledger objects. This object is implemented to save ledger space and has the property that it gives us O(1) lookup time for an NFT, assuming we know who owns the NFT at a particular ledger. However, if we do not know who owns the NFT at a specific ledger height we have no alternative but to scan the entire ledger in `rippled`. Because of this tradeoff, Clio implements a special NFT indexing data structure that allows Clio users to query NFTs quickly, while keeping rippled's space-saving optimizations.
+
+### nf_tokens
 
-#### nf_tokens
 ```
 CREATE TABLE clio.nf_tokens (
 	token_id blob,         # The NFT's ID
@@ -151,21 +204,21 @@ CREATE TABLE clio.nf_tokens (
 	PRIMARY KEY (token_id, sequence)
 ) WITH CLUSTERING ORDER BY (sequence DESC) ...
 ```
-This table indexes NFT IDs with their owner at a given ledger. So
+
+This table indexes NFT IDs with their owner at a given ledger.
+
+The example query below shows how you could search for the owner of token `N` at ledger `Y` and see whether the token was burned.
+
 ```
 SELECT * FROM nf_tokens
 WHERE token_id = N AND seq <= Y
 ORDER BY seq DESC LIMIT 1;
 ```
-will give you the owner of token N at ledger Y and whether it was burned. If
-the token is burned, the owner field indicates the account that owned the
-token at the time it was burned; it does not indicate the person who burned
-the token, necessarily. If you need to determine who burned the token you can
-use the `nft_history` API, which will give you the NFTokenBurn transaction
-that burned this token, along with the account that submitted that
-transaction.
 
-#### issuer_nf_tokens_v2
+If the token is burned, the owner field indicates the account that owned the token at the time it was burned; it does not indicate the person who burned the token, necessarily. If you need to determine who burned the token you can use the `nft_history` API, which will give you the `NFTokenBurn` transaction that burned this token, along with the account that submitted that transaction.
+
+### issuer_nf_tokens_v2
+
 ```
 CREATE TABLE clio.issuer_nf_tokens_v2 (
 	issuer blob,       # The NFT issuer's account ID
@@ -174,13 +227,12 @@ CREATE TABLE clio.issuer_nf_tokens_v2 (
 	PRIMARY KEY (issuer, taxon, token_id)
 ) WITH CLUSTERING ORDER BY (taxon ASC, token_id ASC) ...
 ```
-This table indexes token IDs against their issuer and issuer/taxon
-combination. This is useful for determining all the NFTs a specific account
-issued, or all the NFTs a specific account issued with a specific taxon. It is
-not useful to know all the NFTs with a given taxon while excluding issuer, since the
-meaning of a taxon is left to an issuer.
 
-#### nf_token_uris
+This table indexes token IDs against their issuer and issuer/taxon
+combination. This is useful for determining all the NFTs a specific account issued, or all the NFTs a specific account issued with a specific taxon. It is not useful to know all the NFTs with a given taxon while excluding issuer, since the meaning of a taxon is left to an issuer.
+
+### nf_token_uris
+
 ```
 CREATE TABLE clio.nf_token_uris (
 	token_id blob,    # The NFT's ID
@@ -189,23 +241,17 @@ CREATE TABLE clio.nf_token_uris (
 	PRIMARY KEY (token_id, sequence)
 ) WITH CLUSTERING ORDER BY (sequence DESC) ...
 ```
-This table is used to store an NFT's URI. Without storing this here, we would
-need to traverse the NFT owner's entire set of NFTs to find the URI, again due
-to the way that NFTs are stored in rippled. Furthermore, instead of storing
-this in the `nf_tokens` table, we store it here to save space. A given NFT
-will have only one entry in this table (see caveat below), written to this
-table as soon as clio sees the NFTokenMint transaction, or when clio loads an
-NFTokenPage from the initial ledger it downloaded. However, the `nf_tokens`
-table is written to every time an NFT changes ownership, or if it is burned.
 
-Given this, why do we have to store the sequence? Unfortunately there is an
-extreme edge case where a given NFT ID can be burned, and then re-minted with
-a different URI. This is extremely unlikely, and might be fixed in a future
-version to rippled, but just in case we can handle that edge case by allowing
-a given NFT ID to have a new URI assigned in this case, without removing the
-prior URI.
+This table is used to store an NFT's URI. Without storing this here, we would need to traverse the NFT owner's entire set of NFTs to find the URI, again due to the way that NFTs are stored in `rippled`. Furthermore, instead of storing this in the `nf_tokens` table, we store it here to save space.
+
+A given NFT will have only one entry in this table (see caveat below), and will be written to this table as soon as Clio sees the `NFTokenMint` transaction, or when Clio loads an `NFTokenPage` from the initial ledger it downloaded. However, the `nf_tokens` table is written to every time an NFT changes ownership, or if it is burned.
+
+> **Why do we have to store the sequence?**
+>
+> Unfortunately there is an extreme edge case where a given NFT ID can be burned, and then re-minted with a different URI. This is extremely unlikely, and might be fixed in a future version of `rippled`. Currently, Clio handles this edge case by allowing the NFT ID to have a new URI assigned, without removing the prior URI.
+
+### nf_token_transactions
 
-#### nf_token_transactions
 ```
 CREATE TABLE clio.nf_token_transactions (
 	token_id blob,                  # The NFT's ID
@@ -214,7 +260,5 @@ CREATE TABLE clio.nf_token_transactions (
 	PRIMARY KEY (token_id, seq_idx)
 ) WITH CLUSTERING ORDER BY (seq_idx DESC) ...
 ```
-This table is the NFT equivalent of `account_tx`. It's motivated by the exact
-same reasons and serves the analogous purpose here. It drives the
-`nft_history` API.
 
+The `nf_token_transactions` table serves as the NFT counterpart to `account_tx`, inspired by the same motivations and fulfilling a similar role within this context. It drives the `nft_history` API.

src/data/cassandra/Concepts.hpp

@@ -41,21 +41,10 @@ namespace data::cassandra {
  */
 template <typename T>
 concept SomeSettingsProvider = requires(T a) {
-    {
-        a.getSettings()
-    } -> std::same_as<Settings>;
-    {
-        a.getKeyspace()
-    } -> std::same_as<std::string>;
-    {
-        a.getTablePrefix()
-    } -> std::same_as<std::optional<std::string>>;
-    {
-        a.getReplicationFactor()
-    } -> std::same_as<uint16_t>;
-    {
-        a.getTtl()
-    } -> std::same_as<uint16_t>;
+    { a.getSettings() } -> std::same_as<Settings>;
+    { a.getKeyspace() } -> std::same_as<std::string>;
+    { a.getTablePrefix() } -> std::same_as<std::optional<std::string>>;
+    { a.getReplicationFactor() } -> std::same_as<uint16_t>;
 };
 
 /**
@@ -71,42 +60,18 @@ concept SomeExecutionStrategy = requires(
     PreparedStatement prepared,
     boost::asio::yield_context token
 ) {
-    {
-        T(settings, handle)
-    };
-    {
-        a.sync()
-    } -> std::same_as<void>;
-    {
-        a.isTooBusy()
-    } -> std::same_as<bool>;
-    {
-        a.writeSync(statement)
-    } -> std::same_as<ResultOrError>;
-    {
-        a.writeSync(prepared)
-    } -> std::same_as<ResultOrError>;
-    {
-        a.write(prepared)
-    } -> std::same_as<void>;
-    {
-        a.write(std::move(statements))
-    } -> std::same_as<void>;
-    {
-        a.read(token, prepared)
-    } -> std::same_as<ResultOrError>;
-    {
-        a.read(token, statement)
-    } -> std::same_as<ResultOrError>;
-    {
-        a.read(token, statements)
-    } -> std::same_as<ResultOrError>;
-    {
-        a.readEach(token, statements)
-    } -> std::same_as<std::vector<Result>>;
-    {
-        a.stats()
-    } -> std::same_as<boost::json::object>;
+    { T(settings, handle) };
+    { a.sync() } -> std::same_as<void>;
+    { a.isTooBusy() } -> std::same_as<bool>;
+    { a.writeSync(statement) } -> std::same_as<ResultOrError>;
+    { a.writeSync(prepared) } -> std::same_as<ResultOrError>;
+    { a.write(prepared) } -> std::same_as<void>;
+    { a.write(std::move(statements)) } -> std::same_as<void>;
+    { a.read(token, prepared) } -> std::same_as<ResultOrError>;
+    { a.read(token, statement) } -> std::same_as<ResultOrError>;
+    { a.read(token, statements) } -> std::same_as<ResultOrError>;
+    { a.readEach(token, statements) } -> std::same_as<std::vector<Result>>;
+    { a.stats() } -> std::same_as<boost::json::object>;
 };
 
 /**
@@ -114,12 +79,8 @@ concept SomeExecutionStrategy = requires(
  */
 template <typename T>
 concept SomeRetryPolicy = requires(T a, boost::asio::io_context ioc, CassandraError err, uint32_t attempt) {
-    {
-        T(ioc)
-    };
-    {
-        a.shouldRetry(err)
-    } -> std::same_as<bool>;
+    { T(ioc) };
+    { a.shouldRetry(err) } -> std::same_as<bool>;
     {
         a.retry([]() {})
     } -> std::same_as<void>;

src/data/cassandra/Schema.hpp

@@ -98,10 +98,8 @@ public:
                      PRIMARY KEY (key, sequence) 
                   ) 
              WITH CLUSTERING ORDER BY (sequence DESC) 
-              AND default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "objects"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "objects")
         ));
 
         statements.emplace_back(fmt::format(
@@ -114,10 +112,8 @@ public:
                  transaction blob, 
                     metadata blob 
                   ) 
-             WITH default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "transactions"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "transactions")
         ));
 
         statements.emplace_back(fmt::format(
@@ -128,10 +124,8 @@ public:
                         hash blob, 
                      PRIMARY KEY (ledger_sequence, hash) 
                   ) 
-             WITH default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "ledger_transactions"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "ledger_transactions")
         ));
 
         statements.emplace_back(fmt::format(
@@ -143,10 +137,8 @@ public:
                    next blob, 
                 PRIMARY KEY (key, seq) 
                   ) 
-             WITH default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "successor"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "successor")
         ));
 
         statements.emplace_back(fmt::format(
@@ -157,10 +149,8 @@ public:
                     key blob,
                 PRIMARY KEY (seq, key) 
                   ) 
-             WITH default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "diff"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "diff")
         ));
 
         statements.emplace_back(fmt::format(
@@ -173,10 +163,8 @@ public:
                     PRIMARY KEY (account, seq_idx) 
                   ) 
              WITH CLUSTERING ORDER BY (seq_idx DESC)
-              AND default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "account_tx"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "account_tx")
         ));
 
         statements.emplace_back(fmt::format(
@@ -186,10 +174,8 @@ public:
                     sequence bigint PRIMARY KEY,
                       header blob
                   ) 
-             WITH default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "ledgers"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "ledgers")
         ));
 
         statements.emplace_back(fmt::format(
@@ -199,10 +185,8 @@ public:
                     hash blob PRIMARY KEY,
                 sequence bigint
                   ) 
-             WITH default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "ledger_hashes"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "ledger_hashes")
         ));
 
         statements.emplace_back(fmt::format(
@@ -227,10 +211,8 @@ public:
                      PRIMARY KEY (token_id, sequence) 
                   ) 
              WITH CLUSTERING ORDER BY (sequence DESC)
-              AND default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "nf_tokens"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "nf_tokens")
         ));
 
         statements.emplace_back(fmt::format(
@@ -243,10 +225,8 @@ public:
                      PRIMARY KEY (issuer, taxon, token_id)
                   ) 
              WITH CLUSTERING ORDER BY (taxon ASC, token_id ASC)
-              AND default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "issuer_nf_tokens_v2"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "issuer_nf_tokens_v2")
         ));
 
         statements.emplace_back(fmt::format(
@@ -259,10 +239,8 @@ public:
                      PRIMARY KEY (token_id, sequence)
                   ) 
              WITH CLUSTERING ORDER BY (sequence DESC)
-              AND default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "nf_token_uris"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "nf_token_uris")
         ));
 
         statements.emplace_back(fmt::format(
@@ -275,10 +253,8 @@ public:
                      PRIMARY KEY (token_id, seq_idx) 
                   ) 
              WITH CLUSTERING ORDER BY (seq_idx DESC)
-              AND default_time_to_live = {}
             )",
-            qualifiedTableName(settingsProvider_.get(), "nf_token_transactions"),
-            settingsProvider_.get().getTtl()
+            qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")
         ));
 
         return statements;
@@ -586,6 +562,32 @@ public:
             ));
         }();
 
+        PreparedStatement selectAccountFromBegining = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                SELECT account 
+                  FROM {}               
+                 WHERE token(account) > 0
+                   PER PARTITION LIMIT 1 
+                 LIMIT ?
+                )",
+                qualifiedTableName(settingsProvider_.get(), "account_tx")
+            ));
+        }();
+
+        PreparedStatement selectAccountFromToken = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                SELECT account 
+                  FROM {}               
+                 WHERE token(account) > token(?)
+                   PER PARTITION LIMIT 1 
+                 LIMIT ?
+                )",
+                qualifiedTableName(settingsProvider_.get(), "account_tx")
+            ));
+        }();
+
         PreparedStatement selectAccountTxForward = [this]() {
             return handle_.get().prepare(fmt::format(
                 R"(

src/data/cassandra/SettingsProvider.cpp

@@ -68,12 +68,11 @@ tag_invoke(boost::json::value_to_tag<Settings::SecureConnectionBundle>, boost::j
 }
 }  // namespace impl
 
-SettingsProvider::SettingsProvider(util::Config const& cfg, uint16_t ttl)
+SettingsProvider::SettingsProvider(util::Config const& cfg)
     : config_{cfg}
     , keyspace_{cfg.valueOr<std::string>("keyspace", "clio")}
     , tablePrefix_{cfg.maybeValue<std::string>("table_prefix")}
     , replicationFactor_{cfg.valueOr<uint16_t>("replication_factor", 3)}
-    , ttl_{ttl}
     , settings_{parseSettings()}
 {
 }

src/data/cassandra/SettingsProvider.hpp

@@ -39,7 +39,6 @@ class SettingsProvider {
     std::string keyspace_;
     std::optional<std::string> tablePrefix_;
     uint16_t replicationFactor_;
-    uint16_t ttl_;
     Settings settings_;
 
 public:
@@ -47,9 +46,8 @@ public:
      * @brief Create a settings provider from the specified config.
      *
      * @param cfg The config of Clio to use
-     * @param ttl Time to live setting
      */
-    explicit SettingsProvider(util::Config const& cfg, uint16_t ttl = 0);
+    explicit SettingsProvider(util::Config const& cfg);
 
     /**
      * @return The cluster settings
@@ -60,7 +58,7 @@ public:
     /**
      * @return The specified keyspace
      */
-    [[nodiscard]] inline std::string
+    [[nodiscard]] std::string
     getKeyspace() const
     {
         return keyspace_;
@@ -69,7 +67,7 @@ public:
     /**
      * @return The optional table prefix to use in all queries
      */
-    [[nodiscard]] inline std::optional<std::string>
+    [[nodiscard]] std::optional<std::string>
     getTablePrefix() const
     {
         return tablePrefix_;
@@ -78,21 +76,12 @@ public:
     /**
      * @return The replication factor
      */
-    [[nodiscard]] inline uint16_t
+    [[nodiscard]] uint16_t
     getReplicationFactor() const
     {
         return replicationFactor_;
     }
 
-    /**
-     * @return The default time to live to use in all `create` queries
-     */
-    [[nodiscard]] inline uint16_t
-    getTtl() const
-    {
-        return ttl_;
-    }
-
 private:
     [[nodiscard]] std::optional<std::string>
     parseOptionalCertificate() const;

src/data/cassandra/Types.hpp

@@ -19,9 +19,8 @@
 
 #pragma once
 
-#include "util/Expected.hpp"
-
 #include <cstdint>
+#include <expected>
 
 namespace data::cassandra {
 
@@ -59,8 +58,8 @@ struct Limit {
 class Handle;
 class CassandraError;
 
-using MaybeError = util::Expected<void, CassandraError>;
-using ResultOrError = util::Expected<Result, CassandraError>;
-using Error = util::Unexpected<CassandraError>;
+using MaybeError = std::expected<void, CassandraError>;
+using ResultOrError = std::expected<Result, CassandraError>;
+using Error = std::unexpected<CassandraError>;
 
 }  // namespace data::cassandra

src/data/cassandra/impl/Batch.cpp

@@ -23,10 +23,10 @@
 #include "data/cassandra/Types.hpp"
 #include "data/cassandra/impl/ManagedObject.hpp"
 #include "data/cassandra/impl/Statement.hpp"
-#include "util/Expected.hpp"
 
 #include <cassandra.h>
 
+#include <expected>
 #include <stdexcept>
 #include <vector>