Upgrade xrpl to 2.2.0 (#1443)

Fixes #1156

.clang-format

@@ -33,12 +33,13 @@ DisableFormat:   false
 ExperimentalAutoDetectBinPacking: false
 FixNamespaceComments: true
 ForEachMacros:   [ Q_FOREACH,  BOOST_FOREACH ]
+IncludeBlocks: Regroup
 IncludeCategories:
-  - Regex:           '^<(BeastConfig)'
-    Priority:        0
-  - Regex:           '^<(ripple)/'
+  - Regex:           '^".*"$'
+    Priority:        1
+  - Regex:           '^<.*\.(h|hpp)>$'
     Priority:        2
-  - Regex:           '^<(boost)/'
+  - Regex:           '^<.*>$'
     Priority:        3
   - Regex:           '.*'
     Priority:        4

.clang-tidy

@@ -7,6 +7,7 @@ Checks: '-*,
   bugprone-copy-constructor-init,
   bugprone-dangling-handle,
   bugprone-dynamic-static-initializers,
+  bugprone-empty-catch,
   bugprone-fold-init-type,
   bugprone-forward-declaration-namespace,
   bugprone-inaccurate-erase,
@@ -20,11 +21,15 @@ Checks: '-*,
   bugprone-misplaced-pointer-arithmetic-in-alloc,
   bugprone-misplaced-widening-cast,
   bugprone-move-forwarding-reference,
+  bugprone-multiple-new-in-one-expression,
   bugprone-multiple-statement-macro,
   bugprone-no-escape,
+  bugprone-non-zero-enum-to-bool-conversion,
   bugprone-parent-virtual-call,
   bugprone-posix-return,
   bugprone-redundant-branch-condition,
+  bugprone-reserved-identifier,
+  bugprone-unused-return-value,
   bugprone-shared-ptr-array-mismatch,
   bugprone-signal-handler,
   bugprone-signed-char-misuse,
@@ -45,6 +50,7 @@ Checks: '-*,
   bugprone-suspicious-semicolon,
   bugprone-suspicious-string-compare,
   bugprone-swapped-arguments,
+  bugprone-switch-missing-default-case,
   bugprone-terminating-continue,
   bugprone-throw-keyword-missing,
   bugprone-too-small-loop-variable,
@@ -52,18 +58,23 @@ Checks: '-*,
   bugprone-undelegated-constructor,
   bugprone-unhandled-exception-at-new,
   bugprone-unhandled-self-assignment,
+  bugprone-unique-ptr-array-mismatch,
+  bugprone-unsafe-functions,
   bugprone-unused-raii,
-  bugprone-unused-return-value,
   bugprone-use-after-move,
   bugprone-virtual-near-miss,
   cppcoreguidelines-init-variables,
-  cppcoreguidelines-prefer-member-initializer,
+  cppcoreguidelines-misleading-capture-default-by-value,
   cppcoreguidelines-pro-type-member-init,
   cppcoreguidelines-pro-type-static-cast-downcast,
+  cppcoreguidelines-rvalue-reference-param-not-moved,
+  cppcoreguidelines-use-default-member-init,
   cppcoreguidelines-virtual-class-destructor,
   llvm-namespace-comment,
   misc-const-correctness,
   misc-definitions-in-headers,
+  misc-header-include-cycle,
+  misc-include-cleaner,
   misc-misplaced-const,
   misc-redundant-expression,
   misc-static-assert,
@@ -75,6 +86,7 @@ Checks: '-*,
   modernize-make-shared,
   modernize-make-unique,
   modernize-pass-by-value,
+  modernize-type-traits,
   modernize-use-emplace,
   modernize-use-equals-default,
   modernize-use-equals-delete,
@@ -112,7 +124,10 @@ Checks: '-*,
 
 CheckOptions:
   readability-braces-around-statements.ShortStatementLines: 2
+  bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true
+  bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc;::std::expected
+  misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*'
 
-HeaderFilterRegex: '^.*/(src|unitests)/.*\.(h|hpp)$'
+HeaderFilterRegex: '^.*/(src|unittests)/.*\.(h|hpp)$'
 WarningsAsErrors: '*'
 

.clangd

@@ -0,0 +1,5 @@
+Diagnostics:
+  UnusedIncludes: Strict
+  MissingIncludes: Strict
+  Includes:
+    IgnoreHeader: ".*/(detail|impl)/.*"

.codecov.yml

@@ -0,0 +1,11 @@
+coverage:
+  status:
+    project:
+      default:
+        target: 50%
+        threshold: 2%
+
+    patch:
+      default:
+        target: 20% # Need to bump this number https://docs.codecov.com/docs/commit-status#patch-status
+        threshold: 2%

.githooks/ensure_release_tag

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# Pushing a release branch requires an annotated tag at the released commit
-branch=$(git rev-parse --abbrev-ref HEAD)
-
-if [[ $branch =~ master ]]; then
-  # check if HEAD commit is tagged
-  if ! git describe --exact-match HEAD; then
-    echo "Commits to master must be tagged"
-    exit 1
-  fi
-elif [[ $branch =~ release/* ]]; then
-  IFS=/ read -r branch rel_ver <<< ${branch}
-  tag=$(git describe --tags --abbrev=0)
-  if [[ "${rel_ver}" != "${tag}" ]]; then
-    echo "release/${rel_ver} branches must have annotated tag ${rel_ver}"
-    echo "git tag -am\"${rel_ver}\" ${rel_ver}"
-    exit 1
-  fi
-fi

.githooks/pre-commit

@@ -7,22 +7,58 @@ sources="src unittests"
 formatter="clang-format -i"
 version=$($formatter --version | grep -o '[0-9\.]*')
 
-if [[ "16.0.0" > "$version" ]]; then
+if [[ "17.0.0" > "$version" ]]; then
     cat <<EOF
 
                                     ERROR
 -----------------------------------------------------------------------------
-            A minimum of version 16 of `clang-format` is required.
+            A minimum of version 17 of `which clang-format` is required.
             Your version is $version.
             Please fix paths and run again.
 -----------------------------------------------------------------------------
 
+EOF
+    exit 3
+fi
+
+# check there is no .h headers, only .hpp
+wrong_headers=$(find $sources -name "*.h" | sed 's/^/        - /')
+if [[ ! -z "$wrong_headers" ]]; then
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+       Found .h headers in the source code. Please rename them to .hpp:
+
+$wrong_headers
+-----------------------------------------------------------------------------
+
 EOF
     exit 2
 fi
 
+function grep_code {
+    grep -l "${1}" ${sources} -r --include \*.hpp --include \*.cpp
+}
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    # make all includes to be <...> style
+    grep_code '#include ".*"' | xargs sed -i '' -E 's|#include "(.*)"|#include <\1>|g'
+
+    # make local includes to be "..." style
+    main_src_dirs=$(find ./src -maxdepth 1 -type d -exec basename {} \; | tr '\n' '|' | sed 's/|$//' | sed 's/|/\\|/g')
+    grep_code "#include <\($main_src_dirs\)/.*>" | xargs sed -i '' -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g"
+else
+    # make all includes to be <...> style
+    grep_code '#include ".*"' | xargs sed -i -E 's|#include "(.*)"|#include <\1>|g'
+
+    # make local includes to be "..." style
+    main_src_dirs=$(find ./src -type d -maxdepth 1 -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g')
+    grep_code "#include <\($main_src_dirs\)/.*>" | xargs sed -i -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g"
+fi
+
 first=$(git diff $sources)
-find $sources -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -print0 | xargs -0 $formatter
+find $sources -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.ipp' \) -print0 | xargs -0 $formatter
 second=$(git diff $sources)
 changes=$(diff <(echo "$first") <(echo "$second") | wc -l | sed -e 's/^[[:space:]]*//')
 
@@ -39,4 +75,3 @@ EOF
     exit 1
 fi
 
-.githooks/ensure_release_tag

.github/actions/build_clio/action.yml

@@ -1,37 +1,18 @@
 name: Build clio
 description: Build clio in build directory
 inputs:
-  conan_profile:
-    description: Conan profile name
-    required: true
-    default: default
-  conan_cache_hit:
-    description: Whether conan cache has been downloaded
-    required: true
+  target:
+    description: Build target name
+    default: all
 runs:
   using: composite
   steps:
-    - name: Get number of threads on mac
-      id: mac_threads
-      if: ${{ runner.os == 'macOS' }}
-      shell: bash
-      run: echo "num=$(($(sysctl -n hw.logicalcpu) - 2))" >> $GITHUB_OUTPUT
-
-    - name: Get number of threads on Linux
-      id: linux_threads
-      if: ${{ runner.os == 'Linux' }}
-      shell: bash
-      run: echo "num=$(($(nproc) - 2))" >> $GITHUB_OUTPUT
+    - name: Get number of threads
+      uses: ./.github/actions/get_number_of_threads
+      id: number_of_threads
 
     - name: Build Clio
       shell: bash
-      env:
-        BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '' }}"
-        LINT: "${{ runner.os == 'Linux' && 'True' || 'False' }}"
       run: |
-        mkdir -p build
         cd build
-        threads_num=${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }}
-        conan install .. -of . -b $BUILD_OPTION -s build_type=Release -o clio:tests=True -o clio:lint=$LINT --profile ${{ inputs.conan_profile }}
-        cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release .. -G Ninja
-        cmake --build . --parallel $threads_num
+        cmake --build . --parallel ${{ steps.number_of_threads.outputs.threads_number }} --target ${{ inputs.target }}

.github/actions/clang_format/action.yml

@@ -1,23 +1,27 @@
 name: Check format
-description: Check format using clang-format-16
+description: Check format using clang-format-17
 runs:
   using: composite
   steps:
     - name: Add llvm repo
       run: |
-        echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main' | sudo tee -a /etc/apt/sources.list
+        echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-17 main' | sudo tee -a /etc/apt/sources.list
         wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
       shell: bash
 
     - name: Install packages
       run: |
         sudo apt update -qq
-        sudo apt install -y jq clang-format-16
+        sudo apt install -y jq clang-format-17
+        sudo rm /usr/bin/clang-format
+        sudo ln -s /usr/bin/clang-format-17 /usr/bin/clang-format
       shell: bash
 
     - name: Run formatter
+      continue-on-error: true
+      id: run_formatter
       run: |
-        find src unittests -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.ipp' \) -print0 | xargs -0 clang-format-16 -i
+        ./.githooks/pre-commit
       shell: bash
 
     - name: Check for differences
@@ -25,3 +29,8 @@ runs:
       shell: bash
       run: |
         git diff --color --exit-code | tee "clang-format.patch"
+
+    - name: Fail job
+      if: ${{ steps.run_formatter.outcome != 'success' }}
+      shell: bash
+      run: exit 1

.github/actions/code_coverage/action.yml

@@ -0,0 +1,33 @@
+name: Generate code coverage report
+description: Run tests, generate code coverage report and upload it to codecov.io
+runs:
+  using: composite
+  steps:
+    - name: Run tests
+      shell: bash
+      run: |
+        build/clio_tests --backend_host=scylladb
+
+    - name: Run gcovr
+      shell: bash
+      run: |
+        gcovr -e unittests --xml build/coverage_report.xml -j8 --exclude-throw-branches
+
+    - name: Archive coverage report
+      uses: actions/upload-artifact@v3
+      with:
+        name: coverage-report.xml
+        path: build/coverage_report.xml
+        retention-days: 30
+
+    - name: Upload coverage report
+      uses: wandalen/wretry.action@v1.3.0
+      with:
+        action: codecov/codecov-action@v3
+        with: |
+          files: build/coverage_report.xml 
+          fail_ci_if_error: true
+          verbose: true
+          token: ${{ env.CODECOV_TOKEN }}
+        attempt_limit: 5
+        attempt_delay: 10000

.github/actions/generate/action.yml

@@ -0,0 +1,41 @@
+name: Run conan and cmake
+description: Run conan and cmake
+inputs:
+  conan_profile:
+    description: Conan profile name
+    required: true
+  conan_cache_hit:
+    description: Whether conan cache has been downloaded
+    required: true
+    default: 'false'
+  build_type:
+    description: Build type for third-party libraries and clio. Could be 'Release', 'Debug'
+    required: true
+    default: 'Release'
+  code_coverage:
+    description: Whether conan's coverage option should be on or not
+    required: true
+    default: 'false'
+runs:
+  using: composite
+  steps:
+    - name: Create build directory
+      shell: bash
+      run: mkdir -p build
+
+    - name: Run conan
+      shell: bash
+      env:
+        BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '' }}"
+        CODE_COVERAGE: "${{ inputs.code_coverage == 'true' && 'True' || 'False' }}"
+      run: |
+        cd build
+        conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:tests=True -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
+
+    - name: Run cmake
+      shell: bash
+      env:
+        BUILD_TYPE: "${{ inputs.build_type }}"
+      run: |
+        cd build
+        cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} ${{ inputs.extra_cmake_args }} .. -G Ninja

.github/actions/get_number_of_threads/action.yml

@@ -0,0 +1,26 @@
+name: Get number of threads
+description: Determines number of threads to use on macOS and Linux
+outputs:
+  threads_number:
+    description: Number of threads to use
+    value: ${{ steps.number_of_threads_export.outputs.num }}
+runs:
+  using: composite
+  steps:
+    - name: Get number of threads on mac
+      id: mac_threads
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: echo "num=$(($(sysctl -n hw.logicalcpu) - 2))" >> $GITHUB_OUTPUT
+
+    - name: Get number of threads on Linux
+      id: linux_threads
+      if: ${{ runner.os == 'Linux' }}
+      shell: bash
+      run: echo "num=$(($(nproc) - 2))" >> $GITHUB_OUTPUT
+
+    - name: Export output variable
+      shell: bash
+      id: number_of_threads_export
+      run: |
+        echo "num=${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }}" >> $GITHUB_OUTPUT

.github/actions/prepare_runner/action.yml

@@ -0,0 +1,47 @@
+name: Prepare runner
+description: Install packages, set environment variables, create directories
+inputs:
+  disable_ccache:
+    description: Whether ccache should be disabled
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Install packages on mac
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: |
+          brew install llvm@14 pkg-config ninja bison cmake ccache jq gh
+
+    - name: Fix git permissions on Linux
+      if: ${{ runner.os == 'Linux' }}
+      shell: bash
+      run: git config --global --add safe.directory $PWD
+
+    - name: Set env variables for macOS
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: |
+        echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
+        echo "CONAN_USER_HOME=${{ github.workspace }}" >> $GITHUB_ENV
+
+    - name: Set env variables for Linux
+      if: ${{ runner.os == 'Linux' }}
+      shell: bash
+      run: |
+        echo "CCACHE_DIR=/root/.ccache" >> $GITHUB_ENV
+        echo "CONAN_USER_HOME=/root/" >> $GITHUB_ENV
+
+    - name: Set CCACHE_DISABLE=1
+      if: ${{ inputs.disable_ccache == 'true' }}
+      shell: bash
+      run: |
+        echo "CCACHE_DISABLE=1" >> $GITHUB_ENV
+
+    - name: Create directories
+      shell: bash
+      run: |
+        mkdir -p $CCACHE_DIR
+        mkdir -p $CONAN_USER_HOME/.conan
+
+

.github/actions/restore_cache/action.yml

@@ -7,6 +7,14 @@ inputs:
   ccache_dir:
     description: Path to .ccache directory
     required: true
+  build_type:
+    description: Current build type (e.g. Release, Debug)
+    required: true
+    default: Release
+  code_coverage:
+    description: Whether code coverage is on
+    required: true
+    default: 'false'
 outputs:
   conan_hash:
     description: Hash to use as a part of conan cache key
@@ -40,11 +48,12 @@ runs:
       id: conan_cache
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-develop-${{ steps.conan_hash.outputs.hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ steps.conan_hash.outputs.hash }}
 
     - name: Restore ccache cache
       uses: actions/cache/restore@v3
       id: ccache_cache
+      if: ${{ env.CCACHE_DISABLE != '1' }}
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}

.github/actions/save_cache/action.yml

@@ -16,6 +16,16 @@ inputs:
   ccache_cache_hit:
     description: Whether conan cache has been downloaded
     required: true
+  ccache_cache_miss_rate:
+    description: How many cache misses happened
+  build_type:
+    description: Current build type (e.g. Release, Debug)
+    required: true
+    default: Release
+  code_coverage:
+    description: Whether code coverage is on
+    required: true
+    default: 'false'
 runs:
   using: composite
   steps:
@@ -34,13 +44,13 @@ runs:
       uses: actions/cache/save@v3
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-develop-${{ inputs.conan_hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ inputs.conan_hash }}
 
     - name: Save ccache cache
-      if: ${{ inputs.ccache_cache_hit != 'true' }}
+      if: ${{ inputs.ccache_cache_hit != 'true' || inputs.ccache_cache_miss_rate == '100.0' }}
       uses: actions/cache/save@v3
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
 
 

.github/actions/setup_conan/action.yml

@@ -31,9 +31,6 @@ runs:
       shell: bash
       id: conan_setup_linux
       run: |
-        conan profile new default --detect
-        conan profile update settings.compiler.cppstd=20 default
-        conan profile update settings.compiler.libcxx=libstdc++11 default
         echo "created_conan_profile=default" >> $GITHUB_OUTPUT
 
     - name: Export output variable

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: Build Clio
+name: Build
 on:
   push:
     branches: [master, release/*, develop]
@@ -8,28 +8,54 @@ on:
 
 jobs:
   lint:
-    name: Lint
+    name: Check format
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Run clang-format
         uses: ./.github/actions/clang_format
 
-  build_mac:
-    name: Build macOS
+  build:
+    name: Build
     needs: lint
-    runs-on: [self-hosted, macOS]
-    env:
-      CCACHE_DIR: ${{ github.workspace }}/.ccache
-      CONAN_USER_HOME: ${{ github.workspace }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-            fetch-depth: 0
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            build_type: Release
+            code_coverage: false
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            build_type: Debug
+            code_coverage: true
+          # - os: macOS
+          #   build_type: Release
+          #   code_coverage: false
+    runs-on: [self-hosted, "${{ matrix.os }}"]
+    container: ${{ matrix.container }}
 
-      - name: Install packages
-        run: |
-          brew install llvm@14 pkg-config ninja bison cmake ccache jq
+    services:
+      scylladb:
+        image: ${{ (matrix.code_coverage) && 'scylladb/scylla' || '' }}
+        options: >-
+          --health-cmd "cqlsh -e 'describe cluster'" 
+          --health-interval 10s 
+          --health-timeout 5s 
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+          disable_ccache: false
 
       - name: Setup conan
         uses: ./.github/actions/setup_conan
@@ -41,88 +67,44 @@ jobs:
         with:
           conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
           ccache_dir: ${{ env.CCACHE_DIR }}
+          build_type: ${{ matrix.build_type }}
+          code_coverage: ${{ matrix.code_coverage }}
 
-      - name: Build Clio
-        uses: ./.github/actions/build_clio
+      - name: Run conan and cmake
+        uses: ./.github/actions/generate
         with:
           conan_profile: ${{ steps.conan.outputs.conan_profile }}
           conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
-
-      - name: Strip tests
-        run: strip build/clio_tests
-
-      - name: Upload clio_tests
-        uses: actions/upload-artifact@v3
-        with:
-          name: clio_tests_mac
-          path: build/clio_tests
-
-      - name: Save cache
-        uses: ./.github/actions/save_cache
-        with:
-          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
-          conan_hash: ${{ steps.restore_cache.outputs.conan_hash }}
-          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
-          ccache_dir: ${{ env.CCACHE_DIR }}
-          ccache_cache_hit: ${{ steps.restore_cache.outputs.ccache_cache_hit }}
-
-  build_linux:
-    name: Build linux
-    needs: lint
-    runs-on: [self-hosted, Linux]
-    container:
-      image: conanio/gcc11:1.61.0
-      options: --user root
-    env:
-      CCACHE_DIR: /root/.ccache
-      CONAN_USER_HOME: /root/
-    steps:
-      - name: Get Clio
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: Add llvm repo
-        run: |
-          echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-16 main' >> /etc/apt/sources.list
-          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-
-      - name: Install packages
-        run: |
-          apt update -qq
-          apt install -y jq clang-tidy-16
-
-      - name: Install ccache
-        run: |
-          wget https://github.com/ccache/ccache/releases/download/v4.8.3/ccache-4.8.3-linux-x86_64.tar.xz
-          tar xf ./ccache-4.8.3-linux-x86_64.tar.xz
-          mv ./ccache-4.8.3-linux-x86_64/ccache /usr/bin/ccache
-
-      - name: Fix git permissions
-        run: git config --global --add safe.directory $PWD
-
-      - name: Setup conan
-        uses: ./.github/actions/setup_conan
-
-      - name: Restore cache
-        uses: ./.github/actions/restore_cache
-        id: restore_cache
-        with:
-          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
-          ccache_dir: ${{ env.CCACHE_DIR }}
+          build_type: ${{ matrix.build_type }}
+          code_coverage: ${{ matrix.code_coverage }}
 
       - name: Build Clio
         uses: ./.github/actions/build_clio
-        with:
-          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
+
+      - name: Show ccache's statistics
+        shell: bash
+        id: ccache_stats
+        run: |
+          ccache -s > /tmp/ccache.stats
+          miss_rate=$(cat /tmp/ccache.stats | grep 'Misses' | head -n1 | sed 's/.*(\(.*\)%).*/\1/')
+          echo "miss_rate=${miss_rate}" >> $GITHUB_OUTPUT
+          cat /tmp/ccache.stats
 
       - name: Strip tests
+        if: ${{ !matrix.code_coverage }}
         run: strip build/clio_tests
 
-      - name: Upload clio_tests
+      - name: Upload clio_server
         uses: actions/upload-artifact@v3
         with:
-          name: clio_tests_linux
+          name: clio_server_${{ runner.os }}_${{ matrix.build_type }}
+          path: build/clio_server
+
+      - name: Upload clio_tests
+        if: ${{ !matrix.code_coverage }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: clio_tests_${{ runner.os }}
           path: build/clio_tests
 
       - name: Save cache
@@ -133,26 +115,36 @@ jobs:
           conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
           ccache_dir: ${{ env.CCACHE_DIR }}
           ccache_cache_hit: ${{ steps.restore_cache.outputs.ccache_cache_hit }}
+          ccache_cache_miss_rate: ${{ steps.ccache_stats.outputs.miss_rate }}
+          build_type: ${{ matrix.build_type }}
+          code_coverage: ${{ matrix.code_coverage }}
+
+      # TODO: This is not a part of build process but it is the easiest way to do it here.
+      # It will be refactored in https://github.com/XRPLF/clio/issues/1075
+      - name: Run code coverage
+        if: ${{ matrix.code_coverage }}
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        uses: ./.github/actions/code_coverage
+
+  test:
+    name: Run Tests
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+          # - os: macOS
+    runs-on: [self-hosted, "${{ matrix.os }}"]
+    container: ${{ matrix.container }}
 
-  test_mac:
-    needs: build_mac
-    runs-on: [self-hosted, macOS]
     steps:
       - uses: actions/download-artifact@v3
         with:
-          name: clio_tests_mac
-      - name: Run clio_tests
-        run: |
-          chmod +x ./clio_tests
-          ./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
-
-  test_linux:
-    needs: build_linux
-    runs-on: [self-hosted, x-heavy]
-    steps:
-      - uses: actions/download-artifact@v3
-        with:
-          name: clio_tests_linux
+          name: clio_tests_${{ runner.os }}
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests

.github/workflows/clang-tidy.yml

@@ -0,0 +1,116 @@
+name: Clang-tidy check
+on:
+  schedule:
+    - cron: "0 6 * * 1-5"
+  workflow_dispatch:
+  pull_request:
+    branches: [develop]
+    paths:
+      - .clang_tidy
+      - .github/workflows/clang-tidy.yml
+  workflow_call:
+
+jobs:
+  clang_tidy:
+    runs-on: [self-hosted, Linux]
+    container:
+      image: rippleci/clio_ci:latest
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+          disable_ccache: true
+
+      - name: Setup conan
+        uses: ./.github/actions/setup_conan
+        id: conan
+
+      - name: Restore cache
+        uses: ./.github/actions/restore_cache
+        id: restore_cache
+        with:
+          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          ccache_dir: ${{ env.CCACHE_DIR }}
+
+      - name: Run conan and cmake
+        uses: ./.github/actions/generate
+        with:
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
+          build_type: Release
+
+      - name: Get number of threads
+        uses: ./.github/actions/get_number_of_threads
+        id: number_of_threads
+
+      - name: Run clang-tidy
+        continue-on-error: true
+        shell: bash
+        id: run_clang_tidy
+        run: |
+          run-clang-tidy-17 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
+
+      - name: Run pre-commit hook
+        if: ${{ steps.run_clang_tidy.outcome != 'success' }}
+        continue-on-error: true
+        shell: bash
+        run: ./.githooks/pre-commit
+
+      - name: Print issues found
+        if: ${{ steps.run_clang_tidy.outcome != 'success' }}
+        shell: bash
+        run: |
+          sed -i '/error\||/!d' ./output.txt
+          cat output.txt
+          rm output.txt
+
+      - name: Create an issue
+        if: ${{ steps.run_clang_tidy.outcome != 'success' }}
+        id: create_issue
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          echo -e 'Clang-tidy found issues in the code:\n' > issue.md
+          echo -e "List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/" >> issue.md
+          gh issue create --assignee 'cindyyan317,godexsoft,kuznetsss' --label bug --title 'Clang-tidy found bugs in code🐛' --body-file ./issue.md > create_issue.log
+          created_issue=$(cat create_issue.log | sed 's|.*/||')
+          echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
+          rm create_issue.log issue.md
+
+      - uses: crazy-max/ghaction-import-gpg@v5
+        with:
+          gpg_private_key: ${{ secrets.ACTIONS_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.ACTIONS_GPG_PASSPHRASE }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+
+      - name: Create PR with fixes
+        if: ${{ steps.run_clang_tidy.outcome != 'success' }}
+        uses: peter-evans/create-pull-request@v5
+        env:
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+        with:
+          commit-message: "[CI] clang-tidy auto fixes"
+          committer: Clio CI <skuznetsov@ripple.com>
+          branch: "clang_tidy/autofix"
+          branch-suffix: timestamp
+          delete-branch: true
+          title: "[CI] clang-tidy auto fixes"
+          body: "Fixes #${{ steps.create_issue.outputs.created_issue }}. Please review and commit clang-tidy fixes."
+          reviewers: "cindyyan317,godexsoft,kuznetsss"
+
+      - name: Fail the job
+        if: ${{ steps.run_clang_tidy.outcome != 'success' }}
+        shell: bash
+        run: exit 1

.github/workflows/clang-tidy_on_fix_merged.yml

@@ -0,0 +1,29 @@
+name: Restart clang-tidy workflow
+on:
+  push:
+    branches: [develop]
+  workflow_dispatch:
+
+jobs:
+  restart_clang_tidy:
+    runs-on: ubuntu-20.04
+
+    permissions:
+      actions: write
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check last commit matches clang-tidy auto fixes
+        id: check
+        shell: bash
+        run: |
+          passed=$(if [[ $(git log -1 --pretty=format:%s | grep '\[CI\] clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi)
+          echo "passed=$passed" >> $GITHUB_OUTPUT
+
+      - name: Run clang-tidy workflow
+        if: ${{ contains(steps.check.outputs.passed, 'true') }}
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: gh workflow run clang-tidy.yml

.github/workflows/nightly.yml

@@ -0,0 +1,138 @@
+name: Nightly release
+on:
+  schedule:
+    - cron:  '0 5 * * 1-5'
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build clio
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # - os: macOS
+          #   build_type: Release
+          - os: heavy
+            build_type: Release
+            container:
+              image: rippleci/clio_ci:latest
+          - os: heavy
+            build_type: Debug
+            container:
+              image: rippleci/clio_ci:latest
+    runs-on: [self-hosted, "${{ matrix.os }}"]
+    container: ${{ matrix.container }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            fetch-depth: 0
+
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+            disable_ccache: true
+
+      - name: Setup conan
+        uses: ./.github/actions/setup_conan
+        id: conan
+
+      - name: Run conan and cmake
+        uses: ./.github/actions/generate
+        with:
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
+          build_type: ${{ matrix.build_type }}
+
+      - name: Build Clio
+        uses: ./.github/actions/build_clio
+
+      - name: Strip tests
+        run: strip build/clio_tests
+
+      - name: Upload clio_tests
+        uses: actions/upload-artifact@v3
+        with:
+          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
+          path: build/clio_tests
+
+      - name: Compress clio_server
+        shell: bash
+        run: |
+            cd build
+            tar czf ./clio_server_${{ runner.os }}_${{ matrix.build_type }}.tar.gz ./clio_server
+
+      - name: Upload clio_server
+        uses: actions/upload-artifact@v3
+        with:
+          name: clio_server_${{ runner.os }}_${{ matrix.build_type }}
+          path: build/clio_server_${{ runner.os }}_${{ matrix.build_type }}.tar.gz
+
+
+  run_tests:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # - os: macOS
+          #   build_type: Release
+          - os: heavy
+            build_type: Release
+          - os: heavy
+            build_type: Debug
+    runs-on: [self-hosted, "${{ matrix.os }}"]
+
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
+
+      - name: Run clio_tests
+        run: |
+          chmod +x ./clio_tests
+          ./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
+
+  nightly_release:
+    needs: run_tests
+    runs-on: ubuntu-20.04
+    env:
+      GH_REPO: ${{ github.repository }}
+      GH_TOKEN: ${{ github.token }}
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v3
+        with:
+          path: nightly_release
+
+      - name: Prepare files
+        shell: bash
+        run: |
+          cp ${{ github.workspace }}/.github/workflows/nightly_notes.md "${RUNNER_TEMP}/nightly_notes.md"
+          cd nightly_release
+          rm -r clio_tests*
+          for d in $(ls); do
+            archive_name=$(ls $d)
+            mv ${d}/${archive_name} ./
+            rm -r $d
+            sha256sum ./$archive_name > ./${archive_name}.sha256sum
+            cat ./$archive_name.sha256sum >> "${RUNNER_TEMP}/nightly_notes.md"
+          done
+          echo '```' >> "${RUNNER_TEMP}/nightly_notes.md"
+
+      - name: Remove current nightly release and nightly tag
+        shell: bash
+        run: |
+          gh release delete nightly --yes || true
+          git push origin :nightly || true
+
+      - name: Publish nightly release
+        shell: bash
+        run: |
+          gh release create nightly --prerelease --title "Clio development (nightly) build" \
+            --target $GITHUB_SHA --notes-file "${RUNNER_TEMP}/nightly_notes.md" \
+            ./nightly_release/clio_server*

.github/workflows/nightly_notes.md

@@ -0,0 +1,6 @@
+> **Note:** Please remember that this is a development release and it is not recommended for production use.
+
+Changelog (including previous releases): https://github.com/XRPLF/clio/commits/nightly
+
+## SHA256 checksums
+```

.github/workflows/update_docker_ci.yml

@@ -0,0 +1,40 @@
+name: Update CI docker image
+on:
+  push:
+    branches: [develop]
+    paths:
+      - 'docker/ci/**'
+      - .github/workflows/update_docker_ci.yml
+  workflow_dispatch:
+
+jobs:
+  build_and_push:
+    name: Build and push docker image
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PW }}
+
+      - uses: actions/checkout@v4
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/metadata-action@v5
+        id: meta
+        with:
+          images: rippleci/clio_ci
+          tags: |
+            type=raw,value=latest
+            type=raw,value=gcc_11
+            type=raw,value=${{ env.GITHUB_SHA }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ github.workspace }}/docker/ci
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+

.gitignore

@@ -1,5 +1,6 @@
 *clio*.log
 /build*/
+.devcontainer
 .build
 .cache
 .vscode

CMake/Build.cpp.in

@@ -17,7 +17,9 @@
 */
 //==============================================================================
 
-#include <main/Build.h>
+#include "main/Build.hpp"
+
+#include <string>
 
 namespace Build {
 static constexpr char versionString[] = "@VERSION@";

CMake/ClangTidy.cmake

@@ -8,7 +8,7 @@ if (lint)
     endif ()
     message (STATUS "Using clang-tidy from CLIO_CLANG_TIDY_BIN")
   else ()
-    find_program (_CLANG_TIDY_BIN NAMES "clang-tidy-16" "clang-tidy" REQUIRED)
+    find_program (_CLANG_TIDY_BIN NAMES "clang-tidy-17" "clang-tidy" REQUIRED)
   endif ()
 
   if (NOT _CLANG_TIDY_BIN)

CMake/CodeCoverage.cmake

@@ -0,0 +1,440 @@
+# Copyright (c) 2012 - 2017, Lars Bilke
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+#    may be used to endorse or promote products derived from this software without
+#    specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# CHANGES:
+#
+# 2012-01-31, Lars Bilke
+# - Enable Code Coverage
+#
+# 2013-09-17, Joakim Söderberg
+# - Added support for Clang.
+# - Some additional usage instructions.
+#
+# 2016-02-03, Lars Bilke
+# - Refactored functions to use named parameters
+#
+# 2017-06-02, Lars Bilke
+# - Merged with modified version from github.com/ufz/ogs
+#
+# 2019-05-06, Anatolii Kurotych
+# - Remove unnecessary --coverage flag
+#
+# 2019-12-13, FeRD (Frank Dana)
+# - Deprecate COVERAGE_LCOVR_EXCLUDES and COVERAGE_GCOVR_EXCLUDES lists in favor
+#   of tool-agnostic COVERAGE_EXCLUDES variable, or EXCLUDE setup arguments.
+# - CMake 3.4+: All excludes can be specified relative to BASE_DIRECTORY
+# - All setup functions: accept BASE_DIRECTORY, EXCLUDE list
+# - Set lcov basedir with -b argument
+# - Add automatic --demangle-cpp in lcovr, if 'c++filt' is available (can be
+#   overridden with NO_DEMANGLE option in setup_target_for_coverage_lcovr().)
+# - Delete output dir, .info file on 'make clean'
+# - Remove Python detection, since version mismatches will break gcovr
+# - Minor cleanup (lowercase function names, update examples...)
+#
+# 2019-12-19, FeRD (Frank Dana)
+# - Rename Lcov outputs, make filtered file canonical, fix cleanup for targets
+#
+# 2020-01-19, Bob Apthorpe
+# - Added gfortran support
+#
+# 2020-02-17, FeRD (Frank Dana)
+# - Make all add_custom_target()s VERBATIM to auto-escape wildcard characters
+#   in EXCLUDEs, and remove manual escaping from gcovr targets
+#
+# 2021-01-19, Robin Mueller
+# - Add CODE_COVERAGE_VERBOSE option which will allow to print out commands which are run
+# - Added the option for users to set the GCOVR_ADDITIONAL_ARGS variable to supply additional
+#   flags to the gcovr command
+#
+# 2020-05-04, Mihchael Davis
+#     - Add -fprofile-abs-path to make gcno files contain absolute paths
+#     - Fix BASE_DIRECTORY not working when defined
+#     - Change BYPRODUCT from folder to index.html to stop ninja from complaining about double defines
+#
+# 2021-05-10, Martin Stump
+#     - Check if the generator is multi-config before warning about non-Debug builds
+#
+# 2022-02-22, Marko Wehle
+#     - Change gcovr output from -o <filename> for --xml <filename> and --html <filename> output respectively.
+#       This will allow for Multiple Output Formats at the same time by making use of GCOVR_ADDITIONAL_ARGS, e.g. GCOVR_ADDITIONAL_ARGS "--txt".
+#
+# 2022-09-28, Sebastian Mueller
+#     - fix append_coverage_compiler_flags_to_target to correctly add flags
+#     - replace "-fprofile-arcs -ftest-coverage" with "--coverage" (equivalent)
+#
+# 2023-12-15, Bronek Kozicki
+#     - remove setup_target_for_coverage_lcov (slow) and setup_target_for_coverage_fastcov (no support for Clang)
+#     - fix Clang support by adding find_program( ... llvm-cov )
+#     - add Apple Clang support by adding execute_process( COMMAND xcrun -f llvm-cov ... )
+#     - add CODE_COVERAGE_GCOV_TOOL to explicitly select gcov tool and disable find_program
+#     - replace both functions setup_target_for_coverage_gcovr_* with single setup_target_for_coverage_gcovr
+#     - add support for all gcovr output formats
+#
+# USAGE:
+#
+# 1. Copy this file into your cmake modules path.
+#
+# 2. Add the following line to your CMakeLists.txt (best inside an if-condition
+#    using a CMake option() to enable it just optionally):
+#      include(CodeCoverage)
+#
+# 3. Append necessary compiler flags for all supported source files:
+#      append_coverage_compiler_flags()
+#    Or for specific target:
+#      append_coverage_compiler_flags_to_target(YOUR_TARGET_NAME)
+#
+# 3.a (OPTIONAL) Set appropriate optimization flags, e.g. -O0, -O1 or -Og
+#
+# 4. If you need to exclude additional directories from the report, specify them
+#    using full paths in the COVERAGE_EXCLUDES variable before calling
+#    setup_target_for_coverage_*().
+#    Example:
+#      set(COVERAGE_EXCLUDES
+#          '${PROJECT_SOURCE_DIR}/src/dir1/*'
+#          '/path/to/my/src/dir2/*')
+#    Or, use the EXCLUDE argument to setup_target_for_coverage_*().
+#    Example:
+#      setup_target_for_coverage_gcovr(
+#          NAME coverage
+#          EXECUTABLE testrunner
+#          EXCLUDE "${PROJECT_SOURCE_DIR}/src/dir1/*" "/path/to/my/src/dir2/*")
+#
+# 4.a NOTE: With CMake 3.4+, COVERAGE_EXCLUDES or EXCLUDE can also be set
+#     relative to the BASE_DIRECTORY (default: PROJECT_SOURCE_DIR)
+#     Example:
+#       set(COVERAGE_EXCLUDES "dir1/*")
+#       setup_target_for_coverage_gcovr(
+#           NAME coverage
+#           EXECUTABLE testrunner
+#           FORMAT html-details
+#           BASE_DIRECTORY "${PROJECT_SOURCE_DIR}/src"
+#           EXCLUDE "dir2/*")
+#
+# 4.b If you need to pass specific options to gcovr, specify them in
+#     GCOVR_ADDITIONAL_ARGS variable.
+#     Example:
+#       set (GCOVR_ADDITIONAL_ARGS --exclude-throw-branches --exclude-noncode-lines -s)
+#       setup_target_for_coverage_gcovr(
+#           NAME coverage
+#           EXECUTABLE testrunner
+#           EXCLUDE "src/dir1" "src/dir2")
+#
+# 5. Use the functions described below to create a custom make target which
+#    runs your test executable and produces a code coverage report.
+#
+# 6. Build a Debug build:
+#      cmake -DCMAKE_BUILD_TYPE=Debug ..
+#      make
+#      make my_coverage_target
+
+include(CMakeParseArguments)
+
+option(CODE_COVERAGE_VERBOSE "Verbose information" FALSE)
+
+# Check prereqs
+find_program( GCOVR_PATH gcovr PATHS ${CMAKE_SOURCE_DIR}/scripts/test)
+
+if (DEFINED CODE_COVERAGE_GCOV_TOOL)
+  set(GCOV_TOOL "${CODE_COVERAGE_GCOV_TOOL}")
+elseif (DEFINED ENV{CODE_COVERAGE_GCOV_TOOL})
+  set(GCOV_TOOL "$ENV{CODE_COVERAGE_GCOV_TOOL}")
+elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang")
+  if (APPLE)
+    execute_process( COMMAND xcrun -f llvm-cov
+      OUTPUT_VARIABLE LLVMCOV_PATH
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+  else()
+    find_program( LLVMCOV_PATH llvm-cov )
+  endif()
+  if(LLVMCOV_PATH)
+    set(GCOV_TOOL "${LLVMCOV_PATH} gcov")
+  endif()
+elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
+  find_program( GCOV_PATH gcov )
+  set(GCOV_TOOL "${GCOV_PATH}")
+endif()
+
+# Check supported compiler (Clang, GNU and Flang)
+get_property(LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES)
+foreach(LANG ${LANGUAGES})
+  if("${CMAKE_${LANG}_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang")
+    if("${CMAKE_${LANG}_COMPILER_VERSION}" VERSION_LESS 3)
+      message(FATAL_ERROR "Clang version must be 3.0.0 or greater! Aborting...")
+    endif()
+  elseif(NOT "${CMAKE_${LANG}_COMPILER_ID}" MATCHES "GNU"
+         AND NOT "${CMAKE_${LANG}_COMPILER_ID}" MATCHES "(LLVM)?[Ff]lang")
+    message(FATAL_ERROR "Compiler is not GNU or Flang! Aborting...")
+  endif()
+endforeach()
+
+set(COVERAGE_COMPILER_FLAGS "-g --coverage"
+    CACHE INTERNAL "")
+if(CMAKE_CXX_COMPILER_ID MATCHES "(GNU|Clang)")
+    include(CheckCXXCompilerFlag)
+    check_cxx_compiler_flag(-fprofile-abs-path HAVE_cxx_fprofile_abs_path)
+    if(HAVE_cxx_fprofile_abs_path)
+        set(COVERAGE_CXX_COMPILER_FLAGS "${COVERAGE_COMPILER_FLAGS} -fprofile-abs-path")
+    endif()
+    include(CheckCCompilerFlag)
+    check_c_compiler_flag(-fprofile-abs-path HAVE_c_fprofile_abs_path)
+    if(HAVE_c_fprofile_abs_path)
+        set(COVERAGE_C_COMPILER_FLAGS "${COVERAGE_COMPILER_FLAGS} -fprofile-abs-path")
+    endif()
+endif()
+
+set(CMAKE_Fortran_FLAGS_COVERAGE
+    ${COVERAGE_COMPILER_FLAGS}
+    CACHE STRING "Flags used by the Fortran compiler during coverage builds."
+    FORCE )
+set(CMAKE_CXX_FLAGS_COVERAGE
+    ${COVERAGE_COMPILER_FLAGS}
+    CACHE STRING "Flags used by the C++ compiler during coverage builds."
+    FORCE )
+set(CMAKE_C_FLAGS_COVERAGE
+    ${COVERAGE_COMPILER_FLAGS}
+    CACHE STRING "Flags used by the C compiler during coverage builds."
+    FORCE )
+set(CMAKE_EXE_LINKER_FLAGS_COVERAGE
+    ""
+    CACHE STRING "Flags used for linking binaries during coverage builds."
+    FORCE )
+set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE
+    ""
+    CACHE STRING "Flags used by the shared libraries linker during coverage builds."
+    FORCE )
+mark_as_advanced(
+    CMAKE_Fortran_FLAGS_COVERAGE
+    CMAKE_CXX_FLAGS_COVERAGE
+    CMAKE_C_FLAGS_COVERAGE
+    CMAKE_EXE_LINKER_FLAGS_COVERAGE
+    CMAKE_SHARED_LINKER_FLAGS_COVERAGE )
+
+get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
+if(NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" OR GENERATOR_IS_MULTI_CONFIG))
+    message(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading")
+endif() # NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" OR GENERATOR_IS_MULTI_CONFIG)
+
+if(CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+    link_libraries(gcov)
+endif()
+
+# Defines a target for running and collection code coverage information
+# Builds dependencies, runs the given executable and outputs reports.
+# NOTE! The executable should always have a ZERO as exit code otherwise
+# the coverage generation will not complete.
+#
+# setup_target_for_coverage_gcovr(
+#     NAME ctest_coverage                    # New target name
+#     EXECUTABLE ctest -j ${PROCESSOR_COUNT} # Executable in PROJECT_BINARY_DIR
+#     DEPENDENCIES executable_target         # Dependencies to build first
+#     BASE_DIRECTORY "../"                   # Base directory for report
+#                                            #  (defaults to PROJECT_SOURCE_DIR)
+#     FORMAT "cobertura"                     # Output format, one of:
+#                                            #  xml cobertura sonarqube json-summary
+#                                            #  json-details coveralls csv txt
+#                                            #  html-single html-nested html-details
+#                                            #  (xml is an alias to cobertura;
+#                                            #  if no format is set, defaults to xml)
+#     EXCLUDE "src/dir1/*" "src/dir2/*"      # Patterns to exclude (can be relative
+#                                            #  to BASE_DIRECTORY, with CMake 3.4+)
+# )
+# The user can set the variable GCOVR_ADDITIONAL_ARGS to supply additional flags to the
+# GCVOR command.
+function(setup_target_for_coverage_gcovr)
+    set(options NONE)
+    set(oneValueArgs BASE_DIRECTORY NAME FORMAT)
+    set(multiValueArgs EXCLUDE EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES)
+    cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+    if(NOT GCOV_TOOL)
+        message(FATAL_ERROR "Could not find gcov or llvm-cov tool! Aborting...")
+    endif()
+
+    if(NOT GCOVR_PATH)
+        message(FATAL_ERROR "Could not find gcovr tool! Aborting...")
+    endif()
+
+    # Set base directory (as absolute path), or default to PROJECT_SOURCE_DIR
+    if(DEFINED Coverage_BASE_DIRECTORY)
+        get_filename_component(BASEDIR ${Coverage_BASE_DIRECTORY} ABSOLUTE)
+    else()
+        set(BASEDIR ${PROJECT_SOURCE_DIR})
+    endif()
+
+    if(NOT DEFINED Coverage_FORMAT)
+        set(Coverage_FORMAT xml)
+    endif()
+
+    if("--output" IN_LIST GCOVR_ADDITIONAL_ARGS)
+        message(FATAL_ERROR "Unsupported --output option detected in GCOVR_ADDITIONAL_ARGS! Aborting...")
+    else()
+        if((Coverage_FORMAT STREQUAL "html-details")
+            OR (Coverage_FORMAT STREQUAL "html-nested"))
+            set(GCOVR_OUTPUT_FILE ${PROJECT_BINARY_DIR}/${Coverage_NAME}/index.html)
+            set(GCOVR_CREATE_FOLDER ${PROJECT_BINARY_DIR}/${Coverage_NAME})
+        elseif(Coverage_FORMAT STREQUAL "html-single")
+            set(GCOVR_OUTPUT_FILE ${Coverage_NAME}.html)
+        elseif((Coverage_FORMAT STREQUAL "json-summary")
+            OR (Coverage_FORMAT STREQUAL "json-details")
+            OR (Coverage_FORMAT STREQUAL "coveralls"))
+            set(GCOVR_OUTPUT_FILE ${Coverage_NAME}.json)
+        elseif(Coverage_FORMAT STREQUAL "txt")
+            set(GCOVR_OUTPUT_FILE ${Coverage_NAME}.txt)
+        elseif(Coverage_FORMAT STREQUAL "csv")
+            set(GCOVR_OUTPUT_FILE ${Coverage_NAME}.csv)
+        else()
+            set(GCOVR_OUTPUT_FILE ${Coverage_NAME}.xml)
+        endif()
+    endif()
+
+    if ((Coverage_FORMAT STREQUAL "cobertura")
+        OR (Coverage_FORMAT STREQUAL "xml"))
+        list(APPEND GCOVR_ADDITIONAL_ARGS --cobertura "${GCOVR_OUTPUT_FILE}" )
+        list(APPEND GCOVR_ADDITIONAL_ARGS --cobertura-pretty )
+        set(Coverage_FORMAT cobertura) # overwrite xml
+    elseif(Coverage_FORMAT STREQUAL "sonarqube")
+        list(APPEND GCOVR_ADDITIONAL_ARGS --sonarqube "${GCOVR_OUTPUT_FILE}" )
+    elseif(Coverage_FORMAT STREQUAL "json-summary")
+        list(APPEND GCOVR_ADDITIONAL_ARGS --json-summary "${GCOVR_OUTPUT_FILE}" )
+        list(APPEND GCOVR_ADDITIONAL_ARGS --json-summary-pretty)
+    elseif(Coverage_FORMAT STREQUAL "json-details")
+        list(APPEND GCOVR_ADDITIONAL_ARGS --json "${GCOVR_OUTPUT_FILE}" )
+        list(APPEND GCOVR_ADDITIONAL_ARGS --json-pretty)
+    elseif(Coverage_FORMAT STREQUAL "coveralls")
+        list(APPEND GCOVR_ADDITIONAL_ARGS --coveralls "${GCOVR_OUTPUT_FILE}" )
+        list(APPEND GCOVR_ADDITIONAL_ARGS --coveralls-pretty)
+    elseif(Coverage_FORMAT STREQUAL "csv")
+        list(APPEND GCOVR_ADDITIONAL_ARGS --csv "${GCOVR_OUTPUT_FILE}" )
+    elseif(Coverage_FORMAT STREQUAL "txt")
+        list(APPEND GCOVR_ADDITIONAL_ARGS --txt "${GCOVR_OUTPUT_FILE}" )
+    elseif(Coverage_FORMAT STREQUAL "html-single")
+        list(APPEND GCOVR_ADDITIONAL_ARGS --html "${GCOVR_OUTPUT_FILE}" )
+        list(APPEND GCOVR_ADDITIONAL_ARGS --html-self-contained)
+    elseif(Coverage_FORMAT STREQUAL "html-nested")
+        list(APPEND GCOVR_ADDITIONAL_ARGS --html-nested "${GCOVR_OUTPUT_FILE}" )
+    elseif(Coverage_FORMAT STREQUAL "html-details")
+        list(APPEND GCOVR_ADDITIONAL_ARGS --html-details "${GCOVR_OUTPUT_FILE}" )
+    else()
+        message(FATAL_ERROR "Unsupported output style ${Coverage_FORMAT}! Aborting...")
+    endif()
+
+    # Collect excludes (CMake 3.4+: Also compute absolute paths)
+    set(GCOVR_EXCLUDES "")
+    foreach(EXCLUDE ${Coverage_EXCLUDE} ${COVERAGE_EXCLUDES} ${COVERAGE_GCOVR_EXCLUDES})
+        if(CMAKE_VERSION VERSION_GREATER 3.4)
+            get_filename_component(EXCLUDE ${EXCLUDE} ABSOLUTE BASE_DIR ${BASEDIR})
+        endif()
+        list(APPEND GCOVR_EXCLUDES "${EXCLUDE}")
+    endforeach()
+    list(REMOVE_DUPLICATES GCOVR_EXCLUDES)
+
+    # Combine excludes to several -e arguments
+    set(GCOVR_EXCLUDE_ARGS "")
+    foreach(EXCLUDE ${GCOVR_EXCLUDES})
+        list(APPEND GCOVR_EXCLUDE_ARGS "-e")
+        list(APPEND GCOVR_EXCLUDE_ARGS "${EXCLUDE}")
+    endforeach()
+
+    # Set up commands which will be run to generate coverage data
+    # Run tests
+    set(GCOVR_EXEC_TESTS_CMD
+        ${Coverage_EXECUTABLE} ${Coverage_EXECUTABLE_ARGS}
+    )
+
+    # Create folder
+    if(DEFINED GCOVR_CREATE_FOLDER)
+        set(GCOVR_FOLDER_CMD
+            ${CMAKE_COMMAND} -E make_directory ${GCOVR_CREATE_FOLDER})
+    else()
+        set(GCOVR_FOLDER_CMD echo) # dummy
+    endif()
+
+    # Running gcovr
+    set(GCOVR_CMD
+        ${GCOVR_PATH}
+        --gcov-executable ${GCOV_TOOL}
+        --gcov-ignore-parse-errors=negative_hits.warn_once_per_file
+        -r ${BASEDIR}
+        ${GCOVR_ADDITIONAL_ARGS}
+        ${GCOVR_EXCLUDE_ARGS}
+        --object-directory=${PROJECT_BINARY_DIR}
+    )
+
+    if(CODE_COVERAGE_VERBOSE)
+        message(STATUS "Executed command report")
+
+        message(STATUS "Command to run tests: ")
+        string(REPLACE ";" " " GCOVR_EXEC_TESTS_CMD_SPACED "${GCOVR_EXEC_TESTS_CMD}")
+        message(STATUS "${GCOVR_EXEC_TESTS_CMD_SPACED}")
+
+        if(NOT GCOVR_FOLDER_CMD STREQUAL "echo")
+            message(STATUS "Command to create a folder: ")
+            string(REPLACE ";" " " GCOVR_FOLDER_CMD_SPACED "${GCOVR_FOLDER_CMD}")
+            message(STATUS "${GCOVR_FOLDER_CMD_SPACED}")
+        endif()
+
+        message(STATUS "Command to generate gcovr coverage data: ")
+        string(REPLACE ";" " " GCOVR_CMD_SPACED "${GCOVR_CMD}")
+        message(STATUS "${GCOVR_CMD_SPACED}")
+    endif()
+
+    add_custom_target(${Coverage_NAME}
+        COMMAND ${GCOVR_EXEC_TESTS_CMD}
+        COMMAND ${GCOVR_FOLDER_CMD}
+        COMMAND ${GCOVR_CMD}
+
+        BYPRODUCTS ${GCOVR_OUTPUT_FILE}
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
+        DEPENDS ${Coverage_DEPENDENCIES}
+        VERBATIM # Protect arguments to commands
+        COMMENT "Running gcovr to produce code coverage report."
+    )
+
+    # Show info where to find the report
+    add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
+        COMMAND ;
+        COMMENT "Code coverage report saved in ${GCOVR_OUTPUT_FILE} formatted as ${Coverage_FORMAT}"
+    )
+endfunction() # setup_target_for_coverage_gcovr
+
+function(append_coverage_compiler_flags)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COVERAGE_COMPILER_FLAGS}" PARENT_SCOPE)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_COMPILER_FLAGS}" PARENT_SCOPE)
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${COVERAGE_COMPILER_FLAGS}" PARENT_SCOPE)
+    message(STATUS "Appending code coverage compiler flags: ${COVERAGE_COMPILER_FLAGS}")
+endfunction() # append_coverage_compiler_flags
+
+# Setup coverage for specific library
+function(append_coverage_compiler_flags_to_target name)
+    separate_arguments(_flag_list NATIVE_COMMAND "${COVERAGE_COMPILER_FLAGS}")
+    target_compile_options(${name} PRIVATE ${_flag_list})
+    if(CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
+        target_link_libraries(${name} PRIVATE gcov)
+    endif()
+endfunction()

CMake/Coverage.cmake

@@ -1,125 +0,0 @@
-# call add_coverage(module_name) to add coverage targets for the given module
-function (add_coverage module)
-  if ("${CMAKE_C_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang"
-     OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang")
-    message ("[Coverage] Building with llvm Code Coverage Tools")
-    # Using llvm gcov ; llvm install by xcode
-    set (LLVM_COV_PATH /Library/Developer/CommandLineTools/usr/bin)
-    if (NOT EXISTS ${LLVM_COV_PATH}/llvm-cov)
-      message (FATAL_ERROR "llvm-cov not found! Aborting.")
-    endif ()
-
-    # set Flags
-    target_compile_options (${module} PRIVATE 
-      -fprofile-instr-generate
-      -fcoverage-mapping)
-    
-    target_link_options (${module} PUBLIC 
-      -fprofile-instr-generate
-      -fcoverage-mapping)
-
-    target_compile_options (clio PRIVATE 
-      -fprofile-instr-generate
-      -fcoverage-mapping)
-    
-    target_link_options (clio PUBLIC 
-      -fprofile-instr-generate
-      -fcoverage-mapping)
-
-    # llvm-cov
-    add_custom_target (${module}-ccov-preprocessing
-      COMMAND LLVM_PROFILE_FILE=${module}.profraw $<TARGET_FILE:${module}>
-      COMMAND ${LLVM_COV_PATH}/llvm-profdata merge -sparse ${module}.profraw -o
-              ${module}.profdata
-      DEPENDS ${module})
-
-    add_custom_target (${module}-ccov-show
-      COMMAND ${LLVM_COV_PATH}/llvm-cov show $<TARGET_FILE:${module}>
-              -instr-profile=${module}.profdata -show-line-counts-or-regions
-      DEPENDS ${module}-ccov-preprocessing)
-
-    # add summary for CI parse
-    add_custom_target (${module}-ccov-report
-      COMMAND
-        ${LLVM_COV_PATH}/llvm-cov report $<TARGET_FILE:${module}>
-        -instr-profile=${module}.profdata
-        -ignore-filename-regex=".*_makefiles|.*unittests|.*_deps"
-        -show-region-summary=false
-      DEPENDS ${module}-ccov-preprocessing)
-
-    # exclude libs and unittests self
-    add_custom_target (${module}-ccov
-      COMMAND
-        ${LLVM_COV_PATH}/llvm-cov show $<TARGET_FILE:${module}>
-        -instr-profile=${module}.profdata -show-line-counts-or-regions
-        -output-dir=${module}-llvm-cov -format="html"
-        -ignore-filename-regex=".*_makefiles|.*unittests|.*_deps" > /dev/null 2>&1
-      DEPENDS ${module}-ccov-preprocessing)
-
-    add_custom_command (
-      TARGET ${module}-ccov
-      POST_BUILD
-      COMMENT
-        "Open ${module}-llvm-cov/index.html in your browser to view the coverage report."
-    )
-  elseif ("${CMAKE_C_COMPILER_ID}" MATCHES "GNU" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
-    message ("[Coverage] Building with Gcc Code Coverage Tools")
-
-    find_program (GCOV_PATH gcov)
-    if (NOT GCOV_PATH)
-      message (FATAL_ERROR "gcov not found! Aborting...")
-    endif () # NOT GCOV_PATH
-    find_program (GCOVR_PATH gcovr)
-    if (NOT GCOVR_PATH)
-      message (FATAL_ERROR "gcovr not found! Aborting...")
-    endif () # NOT GCOVR_PATH
-
-    set (COV_OUTPUT_PATH ${module}-gcc-cov)
-    target_compile_options (${module} PRIVATE -fprofile-arcs -ftest-coverage
-                                             -fPIC)
-    target_link_libraries (${module} PRIVATE gcov)
-
-    target_compile_options (clio PRIVATE -fprofile-arcs -ftest-coverage
-                                             -fPIC)
-    target_link_libraries (clio PRIVATE gcov)
-    # this target is used for CI as well generate the summary out.xml will send
-    # to github action to generate markdown, we can paste it to comments or
-    # readme
-    add_custom_target (${module}-ccov
-      COMMAND ${module} ${TEST_PARAMETER}
-      COMMAND rm -rf ${COV_OUTPUT_PATH}
-      COMMAND mkdir ${COV_OUTPUT_PATH}
-      COMMAND
-        gcovr -r ${CMAKE_SOURCE_DIR} --object-directory=${PROJECT_BINARY_DIR} -x
-        ${COV_OUTPUT_PATH}/out.xml --exclude='${CMAKE_SOURCE_DIR}/unittests/'
-        --exclude='${PROJECT_BINARY_DIR}/'
-      COMMAND
-        gcovr -r ${CMAKE_SOURCE_DIR} --object-directory=${PROJECT_BINARY_DIR}
-        --html ${COV_OUTPUT_PATH}/report.html
-        --exclude='${CMAKE_SOURCE_DIR}/unittests/'
-        --exclude='${PROJECT_BINARY_DIR}/'
-      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
-      COMMENT "Running gcovr to produce Cobertura code coverage report.")
-
-    # generate the detail report
-    add_custom_target (${module}-ccov-report
-      COMMAND ${module} ${TEST_PARAMETER}
-      COMMAND rm -rf ${COV_OUTPUT_PATH}
-      COMMAND mkdir ${COV_OUTPUT_PATH}
-      COMMAND
-        gcovr -r ${CMAKE_SOURCE_DIR} --object-directory=${PROJECT_BINARY_DIR}
-        --html-details ${COV_OUTPUT_PATH}/index.html
-        --exclude='${CMAKE_SOURCE_DIR}/unittests/'
-        --exclude='${PROJECT_BINARY_DIR}/'
-      WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
-      COMMENT "Running gcovr to produce Cobertura code coverage report.")
-    add_custom_command (
-      TARGET ${module}-ccov-report
-      POST_BUILD
-      COMMENT
-        "Open ${COV_OUTPUT_PATH}/index.html in your browser to view the coverage report."
-    )
-  else ()
-    message (FATAL_ERROR "Complier not support yet")
-  endif ()
-endfunction ()

CMake/Settings.cmake

@@ -17,16 +17,24 @@ set(COMPILER_FLAGS
   -pedantic
   -Wpedantic
   -Wunused
+   # FIXME: The following bunch are needed for gcc12 atm.
+   -Wno-missing-requires
+   -Wno-restrict
+   -Wno-null-dereference
+   -Wno-maybe-uninitialized
+   -Wno-unknown-warning-option # and this to work with clang
+   # TODO: Address these and others in https://github.com/XRPLF/clio/issues/1273
 )
 
-if (is_gcc AND NOT lint)
-  list(APPEND COMPILER_FLAGS
-    -Wduplicated-branches
-    -Wduplicated-cond
-    -Wlogical-op
-    -Wuseless-cast
-  )
-endif ()
+#TODO: reenable when we change CI #884
+# if (is_gcc AND NOT lint)
+#   list(APPEND COMPILER_FLAGS
+#     -Wduplicated-branches
+#     -Wduplicated-cond
+#     -Wlogical-op
+#     -Wuseless-cast
+#   )
+# endif ()
 
 if (is_clang)
   list(APPEND COMPILER_FLAGS

CMake/deps/libbacktrace.cmake

@@ -0,0 +1,3 @@
+target_compile_definitions(clio PUBLIC BOOST_STACKTRACE_LINK)
+target_compile_definitions(clio PUBLIC BOOST_STACKTRACE_USE_BACKTRACE)
+find_package(libbacktrace REQUIRED)

CMakeLists.txt

@@ -21,6 +21,12 @@ include (CMake/Ccache.cmake)
 include (CheckCXXCompilerFlag)
 include (CMake/ClangTidy.cmake)
 
+# Set coverage build options
+if (tests AND coverage)
+  include (CMake/CodeCoverage.cmake)
+  append_coverage_compiler_flags()
+endif()
+
 if (verbose)
   set (CMAKE_VERBOSE_MAKEFILE TRUE)
 endif ()
@@ -44,6 +50,7 @@ include (CMake/deps/OpenSSL.cmake)
 include (CMake/deps/Threads.cmake)
 include (CMake/deps/libfmt.cmake)
 include (CMake/deps/cassandra.cmake)
+include (CMake/deps/libbacktrace.cmake)
 
 # TODO: Include directory will be wrong when installed.
 target_include_directories (clio PUBLIC src)
@@ -56,11 +63,14 @@ target_link_libraries (clio
   PUBLIC Boost::system
   PUBLIC Boost::log
   PUBLIC Boost::log_setup
+  PUBLIC Boost::stacktrace_backtrace
   PUBLIC cassandra-cpp-driver::cassandra-cpp-driver
   PUBLIC fmt::fmt
   PUBLIC OpenSSL::Crypto
   PUBLIC OpenSSL::SSL
   PUBLIC xrpl::libxrpl
+  PUBLIC dl
+  PUBLIC libbacktrace::libbacktrace
 
   INTERFACE Threads::Threads
 )
@@ -95,12 +105,18 @@ target_sources (clio PRIVATE
   src/etl/impl/ForwardCache.cpp
   ## Feed
   src/feed/SubscriptionManager.cpp
+  src/feed/impl/TransactionFeed.cpp
+  src/feed/impl/LedgerFeed.cpp
+  src/feed/impl/ProposedTransactionFeed.cpp
+  src/feed/impl/SingleFeedBase.cpp
   ## Web
   src/web/impl/AdminVerificationStrategy.cpp
   src/web/IntervalSweepHandler.cpp
+  src/web/Resolver.cpp
   ## RPC
   src/rpc/Errors.cpp
   src/rpc/Factories.cpp
+  src/rpc/AMMHelpers.cpp
   src/rpc/RPCHelpers.cpp
   src/rpc/Counters.cpp
   src/rpc/WorkQueue.cpp
@@ -118,6 +134,7 @@ target_sources (clio PRIVATE
   src/rpc/handlers/AccountObjects.cpp
   src/rpc/handlers/AccountOffers.cpp
   src/rpc/handlers/AccountTx.cpp
+  src/rpc/handlers/AMMInfo.cpp
   src/rpc/handlers/BookChanges.cpp
   src/rpc/handlers/BookOffers.cpp
   src/rpc/handlers/DepositAuthorized.cpp
@@ -140,10 +157,21 @@ target_sources (clio PRIVATE
   src/util/log/Logger.cpp
   src/util/prometheus/Http.cpp
   src/util/prometheus/Label.cpp
-  src/util/prometheus/Metrics.cpp
+  src/util/prometheus/MetricBase.cpp
+  src/util/prometheus/MetricBuilder.cpp
+  src/util/prometheus/MetricsFamily.cpp
+  src/util/prometheus/OStream.cpp
   src/util/prometheus/Prometheus.cpp
   src/util/Random.cpp
-  src/util/Taggable.cpp)
+  src/util/requests/RequestBuilder.cpp
+  src/util/requests/Types.cpp
+  src/util/requests/WsConnection.cpp
+  src/util/requests/impl/SslContext.cpp
+  src/util/Taggable.cpp
+  src/util/TerminationHandler.cpp
+  src/util/TxUtils.cpp
+  src/util/LedgerUtils.cpp
+)
 
 # Clio server
 add_executable (clio_server src/main/Main.cpp)
@@ -165,15 +193,26 @@ if (tests)
     unittests/ProfilerTests.cpp
     unittests/JsonUtilTests.cpp
     unittests/DOSGuardTests.cpp
-    unittests/SubscriptionTests.cpp
-    unittests/SubscriptionManagerTests.cpp
+    unittests/util/TestGlobals.cpp
+    unittests/util/AssertTests.cpp
+    unittests/util/BatchingTests.cpp
+    unittests/util/TestHttpServer.cpp
     unittests/util/TestObject.cpp
+    unittests/util/TestWsServer.cpp
+    unittests/util/TxUtilTests.cpp
     unittests/util/StringUtils.cpp
+    unittests/util/LedgerUtilsTests.cpp
     unittests/util/prometheus/CounterTests.cpp
     unittests/util/prometheus/GaugeTests.cpp
+    unittests/util/prometheus/HistogramTests.cpp
     unittests/util/prometheus/HttpTests.cpp
     unittests/util/prometheus/LabelTests.cpp
-    unittests/util/prometheus/MetricsTests.cpp
+    unittests/util/prometheus/MetricBuilderTests.cpp
+    unittests/util/prometheus/MetricsFamilyTests.cpp
+    unittests/util/prometheus/OStreamTests.cpp
+    unittests/util/requests/RequestBuilderTests.cpp
+    unittests/util/requests/SslContextTests.cpp
+    unittests/util/requests/WsConnectionTests.cpp
     # ETL
     unittests/etl/ExtractionDataPipeTests.cpp
     unittests/etl/ExtractorTests.cpp
@@ -225,8 +264,10 @@ if (tests)
     unittests/rpc/handlers/BookChangesTests.cpp
     unittests/rpc/handlers/LedgerTests.cpp
     unittests/rpc/handlers/VersionHandlerTests.cpp
+    unittests/rpc/handlers/AMMInfoTests.cpp
     # Backend
     unittests/data/BackendFactoryTests.cpp
+    unittests/data/BackendInterfaceTests.cpp
     unittests/data/BackendCountersTests.cpp
     unittests/data/cassandra/BaseTests.cpp
     unittests/data/cassandra/BackendTests.cpp
@@ -239,7 +280,16 @@ if (tests)
     unittests/web/ServerTests.cpp
     unittests/web/RPCServerHandlerTests.cpp
     unittests/web/WhitelistHandlerTests.cpp
-    unittests/web/SweepHandlerTests.cpp)
+    unittests/web/SweepHandlerTests.cpp
+    # Feed
+    unittests/feed/SubscriptionManagerTests.cpp
+    unittests/feed/SingleFeedBaseTests.cpp
+    unittests/feed/ProposedTransactionFeedTests.cpp
+    unittests/feed/BookChangesFeedTests.cpp
+    unittests/feed/LedgerFeedTests.cpp
+    unittests/feed/TransactionFeedTests.cpp
+    unittests/feed/ForwardFeedTests.cpp
+    unittests/feed/TrackableSignalTests.cpp)
 
   include (CMake/deps/gtest.cmake)
 
@@ -253,12 +303,31 @@ if (tests)
   target_include_directories (${TEST_TARGET} PRIVATE unittests)
   target_link_libraries (${TEST_TARGET} PUBLIC clio gtest::gtest)
 
-  # Generate `clio_tests-ccov` if coverage is enabled
-  # Note: use `make clio_tests-ccov` to generate report
+  # Generate `coverage_report` target if coverage is enabled
   if (coverage)
-    target_compile_definitions(${TEST_TARGET} PRIVATE COVERAGE_ENABLED)
-    include (CMake/Coverage.cmake)
-    add_coverage (${TEST_TARGET})
+    if (DEFINED CODE_COVERAGE_REPORT_FORMAT)
+      set(CODE_COVERAGE_FORMAT ${CODE_COVERAGE_REPORT_FORMAT})
+    else()
+      set(CODE_COVERAGE_FORMAT html-details)
+    endif()
+
+    if (DEFINED CODE_COVERAGE_TESTS_ARGS)
+      set(TESTS_ADDITIONAL_ARGS ${CODE_COVERAGE_TESTS_ARGS})
+      separate_arguments(TESTS_ADDITIONAL_ARGS)
+    else()
+      set(TESTS_ADDITIONAL_ARGS "")
+    endif()
+
+    set (GCOVR_ADDITIONAL_ARGS --exclude-throw-branches -s)
+
+    setup_target_for_coverage_gcovr(
+        NAME coverage_report
+        FORMAT ${CODE_COVERAGE_FORMAT}
+        EXECUTABLE clio_tests
+        EXECUTABLE_ARGS --gtest_brief=1 ${TESTS_ADDITIONAL_ARGS}
+        EXCLUDE "unittests"
+        DEPENDENCIES clio_tests
+    )
   endif ()
 endif ()
 

CONTRIBUTING.md

@@ -62,6 +62,11 @@ git commit --amend -S
 git push --force
 ```
 
+## Use ccache (optional)
+Clio uses ccache to speed up compilation. If you want to use it, please make sure it is installed on your machine.
+CMake will automatically detect it and use it if it's available.
+
+
 ## Fixing issues found during code review
 While your code is in review, it's possible that some changes will be requested by reviewer(s).
 This section describes the process of adding your fixes.
@@ -91,7 +96,7 @@ The button for that is near the bottom of the PR's page on GitHub.
 This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent.
 
 ## Formatting
-Code must conform to `clang-format` version 16, unless the result would be unreasonably difficult to read or maintain.
+Code must conform to `clang-format` version 17, unless the result would be unreasonably difficult to read or maintain.
 To change your code to conform use `clang-format -i <your changed files>`.
 
 ## Avoid
@@ -126,6 +131,7 @@ Existing maintainers can resign, or be subject to a vote for removal at the behe
 
 * [cindyyan317](https://github.com/cindyyan317) (Ripple)
 * [godexsoft](https://github.com/godexsoft) (Ripple)
+* [kuznetsss](https://github.com/kuznetsss) (Ripple)
 * [legleux](https://github.com/legleux) (Ripple)
 
 ## Honorable ex-Maintainers

Doxyfile

@@ -1,6 +1,6 @@
 PROJECT_NAME           = "Clio"
 INPUT                  = ../src ../unittests
-EXCLUDE_PATTERNS       = *Test*.cpp *Test*.h
+EXCLUDE_PATTERNS       = *Test*.cpp *Test*.hpp
 RECURSIVE              = YES
 HAVE_DOT               = YES
 

README.md

@@ -1,4 +1,8 @@
 # Clio
+[![Build status](https://github.com/XRPLF/clio/actions/workflows/build.yml/badge.svg?branch=develop)](https://github.com/XRPLF/clio/actions/workflows/build.yml?query=branch%3Adevelop)
+[![Nightly release status](https://github.com/XRPLF/clio/actions/workflows/nightly.yml/badge.svg?branch=develop)](https://github.com/XRPLF/clio/actions/workflows/nightly.yml?query=branch%3Adevelop)
+[![Clang-tidy checks status](https://github.com/XRPLF/clio/actions/workflows/clang-tidy.yml/badge.svg?branch=develop)](https://github.com/XRPLF/clio/actions/workflows/clang-tidy.yml?query=branch%3Adevelop)
+[![Code coverage develop branch](https://codecov.io/gh/XRPLF/clio/branch/develop/graph/badge.svg?)](https://app.codecov.io/gh/XRPLF/clio)
 
 Clio is an XRP Ledger API server. Clio is optimized for RPC calls, over WebSocket or JSON-RPC. 
 Validated historical ledger and transaction data are stored in a more space-efficient format,
@@ -36,6 +40,7 @@ It is written in C++20 and therefore requires a modern compiler.
 - [Conan 1.55](https://conan.io/downloads.html)
 - [CMake 3.16](https://cmake.org/download/)
 - [**Optional**] [GCovr](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html) (needed for code coverage generation)
+- [**Optional**] [CCache](https://ccache.dev/) (speeds up compilation if you are going to compile Clio often)
 
 | Compiler    | Version |
 |-------------|---------|
@@ -81,6 +86,8 @@ conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactor
 ```
 Now you should be able to download prebuilt `xrpl` package on some platforms.
 
+You might need to edit the `~/.conan/remotes.json` file to ensure that this newly added artifactory is listed last. Otherwise you might see compilation errors when building the project with gcc version 13 (or newer).
+
 2. Remove old packages you may have cached: 
 ```sh 
 conan remove -f xrpl
@@ -91,7 +98,7 @@ conan remove -f xrpl
 Navigate to Clio's root directory and perform
 ```sh
 mkdir build && cd build
-conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True
+conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False
 cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
 cmake --build . --parallel 8 # or without the number if you feel extra adventurous
 ```
@@ -101,6 +108,18 @@ If all goes well, `conan install` will find required packages and `cmake` will d
 
 > **Tip:** To generate a Code Coverage report, include `-o coverage=True` in the `conan install` command above, along with `-o tests=True` to enable tests. After running the `cmake` commands, execute `make clio_tests-ccov`. The coverage report will be found at `clio_tests-llvm-cov/index.html`.
 
+## Building Clio with Docker
+
+It is possible to build Clio using docker if you don't want to install all the dependencies on your machine.
+```sh
+docker run -it rippleci/clio_ci:latest
+git clone https://github.com/XRPLF/clio
+mkdir build && cd build
+conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False
+cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
+cmake --build . --parallel 8 # or without the number if you feel extra adventurous
+```
+
 ## Running
 ```sh
 ./clio_server config.json
@@ -257,13 +276,13 @@ Exactly equal password gains admin rights for the request or a websocket connect
 ## Prometheus metrics collection
 
 Clio natively supports Prometheus metrics collection. It accepts Prometheus requests on the port configured in `server` section of config.
-Prometheus metrics are enabled by default. To disable it add `"prometheus_enabled": false` to the config.
+Prometheus metrics are enabled by default. To disable it add `"prometheus": { "enabled": false }` to the config.
 It is important to know that clio responds to Prometheus request only if they are admin requests, so Prometheus should be configured to send admin password in header.
 There is an example of docker-compose file, Prometheus and Grafana configs in [examples/infrastructure](examples/infrastructure).
 
 ## Using clang-tidy for static analysis
 
-Minimum clang-tidy version required is 16.0.
+Minimum clang-tidy version required is 17.0.
 Clang-tidy could be run by cmake during building the project.
 For that provide the option `-o lint=True` for `conan install` command:
 ```sh
@@ -273,9 +292,58 @@ By default cmake will try to find clang-tidy automatically in your system.
 To force cmake use desired binary set `CLIO_CLANG_TIDY_BIN` environment variable as path to clang-tidy binary.
 E.g.:
 ```sh
-export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@16/bin/clang-tidy
+export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@17/bin/clang-tidy
 ```
 
+## Coverage report
+
+Coverage report is intended for the developers using compilers GCC
+or Clang (including Apple Clang). It is generated by the build target `coverage_report`,
+which is only enabled when both `tests` and `coverage` options are set, e.g. with
+`-o coverage=True -o tests=True` in `conan`
+
+Prerequisites for the coverage report:
+
+- [gcovr tool](https://gcovr.com/en/stable/getting-started.html) (can be installed e.g. with `pip install gcovr`)
+- `gcov` for GCC (installed with the compiler by default) or
+- `llvm-cov` for Clang (installed with the compiler by default, also on Apple)
+- `Debug` build type
+
+Coverage report is created when the following steps are completed, in order:
+
+1. `clio_tests` binary built with the instrumentation data, enabled by the `coverage`
+   option mentioned above
+2. completed run of unit tests, which populates coverage capture data
+3. completed run of `gcovr` tool (which internally invokes either `gcov` or `llvm-cov`)
+   to assemble both instrumentation data and coverage capture data into a coverage report
+
+Above steps are automated into a single target `coverage_report`. The instrumented
+`clio_tests` binary can be also used for running regular unit tests. In case of a
+spurious failure of unit tests, it is possile to re-run `coverage_report` target without
+rebuilding the `clio_tests` binary (since it is simply a dependency of the coverage report target).
+
+The default coverage report format is `html-details`, but the developers
+can override it to any of the formats listed in `CMake/CodeCoverage.cmake`
+by setting `CODE_COVERAGE_REPORT_FORMAT` variable in `cmake`. For example, CI
+is setting this parameter to `xml` for the [codecov](codecov.io) integration.
+
+In case if some unit tests predictably fail e.g. due to absence of a Cassandra database, it is possible
+to set unit tests options in `CODE_COVERAGE_TESTS_ARGS` cmake variable, as demonstrated below:
+
+```
+cd .build
+conan install .. --output-folder . --build missing --settings build_type=Debug -o tests=True -o coverage=True
+cmake -DCODE_COVERAGE_REPORT_FORMAT=json-details -DCMAKE_BUILD_TYPE=Debug -DCODE_COVERAGE_TESTS_ARGS="--gtest_filter=-BackendCassandra*" -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake ..
+cmake --build . --target coverage_report
+```
+
+After `coverage_report` target is completed, the generated coverage report will be
+stored inside the build directory, as either of:
+
+- file named `coverage_report.*`, with a suitable extension for the report format, or
+- directory named `coverage_report`, with `index.html` and other files inside, for `html-details` or `html-nested` report formats.
+
+
 ## Developing against `rippled` in standalone mode
 
 If you wish you develop against a `rippled` instance running in standalone

conanfile.py

@@ -1,6 +1,6 @@
 from conan import ConanFile
 from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
-import re
+
 
 class Clio(ConanFile):
     name = 'clio'
@@ -11,7 +11,7 @@ class Clio(ConanFile):
     settings = 'os', 'compiler', 'build_type', 'arch'
     options = {
         'fPIC': [True, False],
-        'verbose': [True, False],   
+        'verbose': [True, False],
         'tests': [True, False],     # build unit tests; create `clio_tests` binary
         'docs': [True, False],      # doxygen API docs; create custom target 'docs'
         'packaging': [True, False], # create distribution packages
@@ -23,10 +23,11 @@ class Clio(ConanFile):
         'boost/1.82.0',
         'cassandra-cpp-driver/2.17.0',
         'fmt/10.1.1',
-        'protobuf/3.21.12',
+        'protobuf/3.21.9',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.0.0-b4',
+        'xrpl/2.2.0',
+        'libbacktrace/cci.20210118'
     ]
 
     default_options = {

docker/ci/dockerfile

@@ -0,0 +1,67 @@
+FROM ubuntu:focal
+ARG DEBIAN_FRONTEND=noninteractive
+ARG TARGETARCH
+
+USER root
+WORKDIR /root/
+
+ENV GCC_VERSION=11 \
+    CCACHE_VERSION=4.8.3 \
+    LLVM_TOOLS_VERSION=17 \
+    GH_VERSION=2.40.0
+ 
+# Add repositories
+RUN apt-get -qq update \
+    && apt-get -qq install -y --no-install-recommends --no-install-suggests gnupg wget curl software-properties-common \
+    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
+    && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - \
+    && apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' \
+    && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM_TOOLS_VERSION} main" >> /etc/apt/sources.list \
+    && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+
+# Install packages
+RUN apt update -qq \
+    && apt install -y --no-install-recommends --no-install-suggests cmake python3 python3-pip sudo git \
+    ninja-build make pkg-config libzstd-dev libzstd1 g++-${GCC_VERSION} jq \
+    clang-format-${LLVM_TOOLS_VERSION} clang-tidy-${LLVM_TOOLS_VERSION} clang-tools-${LLVM_TOOLS_VERSION} \
+    && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-${GCC_VERSION} 100 \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-${GCC_VERSION} 100 \
+    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-${GCC_VERSION} 100 \
+    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-${GCC_VERSION} 100 \
+    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-${GCC_VERSION} 100 \
+    && apt-get clean && apt remove -y software-properties-common \
+    && pip3 install  -q --upgrade --no-cache-dir pip \
+    && pip3 install  -q --no-cache-dir conan==1.62 gcovr
+
+# Install ccache from source
+WORKDIR /tmp
+RUN wget "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz" \
+    && tar xf "ccache-${CCACHE_VERSION}.tar.gz" \
+    && cd "ccache-${CCACHE_VERSION}" \
+    && mkdir build && cd build \
+    && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
+    && ninja && cp ./ccache /usr/bin/ccache
+
+# Install gh
+RUN wget https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
+    && tar xf gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
+    && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh
+
+# Clean up
+RUN rm -rf  /tmp/* /var/tmp/*
+
+WORKDIR /root/
+# Using root by default is not very secure but github checkout action doesn't work with any other user
+# https://github.com/actions/checkout/issues/956
+# And Github Actions doc recommends using root
+# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
+
+# Setup conan
+RUN conan profile new default --detect \
+    && conan profile update settings.compiler.cppstd=20 default \
+    && conan profile update settings.compiler.libcxx=libstdc++11 default \
+    && conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+
+

example-config.json

@@ -16,7 +16,8 @@
             //
             // Advanced options. USE AT OWN RISK:
             // ---
-            "core_connections_per_host": 1 // Defaults to 1
+            "core_connections_per_host": 1, // Defaults to 1
+            "write_batch_size": 20 // Defaults to 20
             //
             // Below options will use defaults from cassandra driver if left unspecified.
             // See https://docs.datastax.com/en/developer/cpp-driver/2.17/api/struct.CassCluster/ for details.
@@ -66,7 +67,7 @@
         // Max number of requests to queue up before rejecting further requests.
         // Defaults to 0, which disables the limit.
         "max_queue_size": 500,
-        // If request contains header with authorization, Clio will check if it matches this value's hash
+        // If request contains header with authorization, Clio will check if it matches the prefix 'Password ' + this value's sha256 hash
         // If matches, the request will be considered as admin request
         "admin_password": "xrp",
         // If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests
@@ -101,7 +102,10 @@
             "log_level": "trace"
         }
     ],
-    "prometheus_enabled": true,
+    "prometheus": {
+        "enabled": true,
+        "compress_reply": true
+    },
     "log_level": "info",
     // Log format (this is the default format)
     "log_format": "%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%",

examples/infrastructure/grafana/clio_dashboard.json

@@ -68,7 +68,7 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 5,
+        "w": 3,
         "x": 0,
         "y": 0
       },
@@ -105,102 +105,6 @@
       "title": "Service state",
       "type": "stat"
     },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "s"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 7,
-        "x": 5,
-        "y": 0
-      },
-      "id": 7,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": false
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "scrape_duration_seconds{job=\"clio\"}",
-          "instant": false,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Prometheus Request Processing Time",
-      "type": "timeseries"
-    },
     {
       "datasource": {
         "type": "prometheus",
@@ -262,8 +166,8 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
-        "x": 12,
+        "w": 9,
+        "x": 3,
         "y": 0
       },
       "id": 2,
@@ -296,102 +200,6 @@
       "title": "Work Queue Size",
       "type": "timeseries"
     },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "PBFA97CFB590B2093"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "µs"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 8
-      },
-      "id": 10,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "PBFA97CFB590B2093"
-          },
-          "editorMode": "code",
-          "expr": "rpc_method_duration_us{job=\"clio\"}",
-          "instant": false,
-          "legendFormat": "{{method}}",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "RPC Method Call Duration",
-      "type": "timeseries"
-    },
     {
       "datasource": {
         "type": "prometheus",
@@ -455,7 +263,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 8
+        "y": 0
       },
       "id": 9,
       "options": {
@@ -550,9 +358,9 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 16
+        "y": 8
       },
-      "id": 8,
+      "id": 11,
       "options": {
         "legend": {
           "calcs": [],
@@ -572,14 +380,206 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "rate(rpc_error_total_number{job=\"clio\"}[$__rate_interval])",
+          "expr": "subscriptions_current_number{job=\"clio\"}",
           "instant": false,
-          "legendFormat": "{{error_type}}",
+          "legendFormat": "{{collection}}{{stream}}",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "RPC Error Rate",
+      "title": "Subscriptions Number",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(increase(ledger_cache_counter_total_number{job=\"clio\",type=\"cache_hit\"}[1m])) / sum(increase(ledger_cache_counter_total_number{job=\"clio\",type=\"request\"}[1m]))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "ledger cache hit rate",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Ledger Cache Hit Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "µs"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 10,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "rpc_method_duration_us{job=\"clio\"}",
+          "instant": false,
+          "legendFormat": "{{method}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "RPC Method Call Duration",
       "type": "timeseries"
     },
     {
@@ -647,13 +647,13 @@
         "x": 12,
         "y": 16
       },
-      "id": 6,
+      "id": 8,
       "options": {
         "legend": {
           "calcs": [],
           "displayMode": "list",
           "placement": "bottom",
-          "showLegend": false
+          "showLegend": true
         },
         "tooltip": {
           "mode": "single",
@@ -667,15 +667,14 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "sum(increase(ledger_cache_counter_total_number{job=\"clio\",type=\"cache_hit\"}[1m])) / sum(increase(ledger_cache_counter_total_number{job=\"clio\",type=\"request\"}[1m]))",
-          "hide": false,
+          "expr": "rate(rpc_error_total_number{job=\"clio\"}[$__rate_interval])",
           "instant": false,
-          "legendFormat": "ledger cache hit rate",
+          "legendFormat": "{{error_type}}",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Ledger Cache Hit Rate",
+      "title": "RPC Error Rate",
       "type": "timeseries"
     },
     {
@@ -791,7 +790,7 @@
             "axisPlacement": "auto",
             "barAlignment": 0,
             "drawStyle": "line",
-            "fillOpacity": 0,
+            "fillOpacity": 10,
             "gradientMode": "none",
             "hideFrom": {
               "legend": false,
@@ -800,6 +799,9 @@
             },
             "insertNulls": false,
             "lineInterpolation": "linear",
+            "lineStyle": {
+              "fill": "solid"
+            },
             "lineWidth": 1,
             "pointSize": 5,
             "scaleDistribution": {
@@ -828,7 +830,8 @@
                 "value": 80
               }
             ]
-          }
+          },
+          "unit": "ms"
         },
         "overrides": []
       },
@@ -838,7 +841,7 @@
         "x": 12,
         "y": 24
       },
-      "id": 11,
+      "id": 12,
       "options": {
         "legend": {
           "calcs": [],
@@ -851,6 +854,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "10.2.0",
       "targets": [
         {
           "datasource": {
@@ -858,14 +862,52 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "subscriptions_current_number{job=\"clio\"}",
+          "expr": "histogram_quantile(0.50, sum(rate(backend_duration_milliseconds_histogram_bucket{job=\"clio\"}[$__interval])) by (le, operation))",
+          "hide": false,
           "instant": false,
-          "legendFormat": "{{collection}}{{stream}}",
+          "legendFormat": "{{operation}} 0.5 percentile",
           "range": true,
           "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.75, sum(rate(backend_duration_milliseconds_histogram_bucket{job=\"clio\"}[$__interval])) by (le, operation))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "{{operation}} 0.75 percentile",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.95, sum(rate(backend_duration_milliseconds_histogram_bucket{job=\"clio\"}[$__interval])) by (le, operation))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "{{operation}} 0.95 percentile",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "expr": "",
+          "hide": false,
+          "instant": false,
+          "range": true,
+          "refId": "D"
         }
       ],
-      "title": "Subscriptions Number",
+      "title": "DB operation duration",
       "type": "timeseries"
     },
     {
@@ -1081,6 +1123,102 @@
       ],
       "title": "DB Pending operations",
       "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 10,
+        "x": 0,
+        "y": 40
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "scrape_duration_seconds{job=\"clio\"}",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Prometheus Request Processing Time",
+      "type": "timeseries"
     }
   ],
   "refresh": "5s",
@@ -1097,6 +1235,6 @@
   "timezone": "",
   "title": "Clio",
   "uid": "aeaae84e-c194-47b2-ad65-86e45eebb815",
-  "version": 1,
+  "version": 3,
   "weekStart": ""
 }

metrics.py

@@ -1,183 +0,0 @@
-#!/usr/bin/python3
-import argparse
-
-from datetime import datetime
-
-def getTime(line):
-    bracketOpen = line.find("[")
-    bracketClose = line.find("]")
-    timestampSub = line[bracketOpen+1:bracketClose]
-    timestamp = datetime.strptime(timestampSub, '%Y-%m-%d %H:%M:%S.%f')
-    return timestamp.timestamp()
-
-def parseAccountTx(filename):
-
-
-    with open(filename) as f:
-        totalProcTime = 0.0
-        totalTxnTime = 0.0
-        numCalls = 0
-        for line in f:
-            if "executed stored_procedure" in line:
-                idx = line.find("in ")
-                idx = idx + 3
-                idx2 = line.find("num")
-                procTime = float(line[idx:idx2])
-                totalProcTime += procTime
-            if "fetchTransactions fetched" in line:
-                idx = line.find("took ")
-                idx = idx + 5
-                txnTime = float(line[idx:])
-                totalTxnTime += txnTime
-                numCalls = numCalls + 1
-        print(totalProcTime)
-        print(totalProcTime/numCalls)
-        print(totalTxnTime)
-        print(totalTxnTime/numCalls)
-            
-
-
-
-
-def parseLogs(filename, interval, minTxnCount = 0):
-
-    with open(filename) as f:
-
-        totalTime = 0
-        totalTxns = 0
-        totalObjs = 0
-        totalLoadTime = 0
-
-
-        start = 0
-        end = 0
-        totalLedgers = 0
-
-        intervalTime = 0
-        intervalTxns = 0
-        intervalObjs = 0
-        intervalLoadTime = 0
-
-        intervalStart = 0
-        intervalEnd = 0
-        intervalLedgers = 0
-        ledgersPerSecond = 0
-
-        print("ledgers, transactions, objects, loadTime, loadTime/ledger, ledgers/sec, txns/sec, objs/sec")
-        for line in f:
-            if "Load phase" in line:
-                sequenceIdx = line.find("Sequence : ")
-                hashIdx =  line.find(" Hash :")
-                sequence = line[sequenceIdx + len("Sequence : "):hashIdx]
-                txnCountSubstr = "txn count = "
-                objCountSubstr = ". object count = "
-                loadTimeSubstr = ". load time = "
-                txnsSubstr = ". load txns per second = "
-                objsSubstr = ". load objs per second = "
-                txnCountIdx = line.find(txnCountSubstr)
-                objCountIdx = line.find(objCountSubstr)
-                loadTimeIdx = line.find(loadTimeSubstr)
-                txnsIdx = line.find(txnsSubstr)
-                objsIdx = line.find(objsSubstr)
-                txnCount = line[txnCountIdx + len(txnCountSubstr):objCountIdx]
-                objCount = line[objCountIdx + len(objCountSubstr):loadTimeIdx]
-                loadTime = line[loadTimeIdx + len(loadTimeSubstr):txnsIdx]
-                txnsPerSecond = line[txnsIdx + len(txnsSubstr):objsIdx]
-                objsPerSecond = line[objsIdx + len(objsSubstr):-1]
-                if int(txnCount) >= minTxnCount:
-                    totalTime += float(loadTime);
-                    totalTxns += float(txnCount)
-                    totalObjs += float(objCount)
-                    intervalTime += float(loadTime)
-                    intervalTxns += float(txnCount)
-                    intervalObjs += float(objCount)
-
-                totalLoadTime += float(loadTime)
-                intervalLoadTime += float(loadTime)
-                
-
-                if start == 0:
-                    start = getTime(line)
-
-
-                prevEnd = end
-                end = getTime(line)
-
-                if intervalStart == 0:
-                    intervalStart = getTime(line)
-
-                intervalEnd = getTime(line)
-
-                totalLedgers+=1
-                intervalLedgers+=1
-                ledgersPerSecond = 0
-                if end != start:
-                    ledgersPerSecond = float(totalLedgers) / float((end - start))
-                intervalLedgersPerSecond = 0
-                if intervalEnd != intervalStart:
-                    intervalLedgersPerSecond = float(intervalLedgers) / float((intervalEnd - intervalStart))
-
-
-
-                if int(sequence) % interval == 0:
-
-                   # print("Sequence = " + sequence + " : [time, txCount, objCount, txPerSec, objsPerSec]")
-                   # print(loadTime + " , " 
-                   #     + txnCount + " , " 
-                   #     + objCount + " , " 
-                   #     + txnsPerSecond + " , " 
-                   #     + objsPerSecond)
-                   # print("Interval Aggregate ( " + str(interval) + " ) [ledgers, txns, objects, elapsedTime, ledgersPerSec, avgLoadTime, txPerSec, objsPerSec]: ")
-                    print(str(intervalLedgers) + " , " 
-                        + str(intervalTxns) + " , "
-                        + str(intervalObjs) + " , "
-                        + str(intervalLoadTime) + " , " 
-                        + str(intervalLoadTime/intervalLedgers) + " , " 
-                        + str(intervalLedgers/intervalLoadTime) + " , " 
-                        + str(intervalTxns/intervalLoadTime) + " , " 
-                        + str(intervalObjs/intervalLoadTime))
-                  #  print("Total Aggregate: [ledgers, txns, objects, elapsedTime, ledgersPerSec, avgLoadTime, txPerSec, objsPerSec]")
-                  #  print(str(totalLedgers) + " , " 
-                  #      + str(totalTxns) + " , "
-                  #      + str(totalObjs) + " , "
-                  #      + str(end-start) + " , " 
-                  #      + str(ledgersPerSecond) + " , " 
-                  #      + str(totalLoadTime/totalLedgers) + " , " 
-                  #      + str(totalTxns/totalTime) + " , " 
-                  #      + str(totalObjs/totalTime))
-                    if int(sequence) % interval == 0:
-                        intervalTime = 0
-                        intervalTxns = 0
-                        intervalObjs = 0
-                        intervalStart = 0
-                        intervalEnd = 0
-                        intervalLedgers = 0
-                        intervalLoadTime = 0
-        print("Total Aggregate: [ledgers, elapsedTime, ledgersPerSec, avgLoadTime, txPerSec, objsPerSec]")
-        print(totalLedgers)
-        print(totalLoadTime)
-        print(str(totalLedgers) + " : " 
-            + str(end-start) + " : " 
-            + str(ledgersPerSecond) + " : " 
-            + str(totalLoadTime/totalLedgers) + " : " 
-            + str(totalTxns/totalTime) + " : " 
-            + str(totalObjs/totalTime))
-
-
-    
-
-parser = argparse.ArgumentParser(description='parses logs')
-parser.add_argument("--filename")
-parser.add_argument("--interval",default=100000)
-parser.add_argument("--minTxnCount",default=0)
-parser.add_argument("--account_tx",default=False)
-
-args = parser.parse_args()
-
-def run(args):
-    if args.account_tx:
-        parseAccountTx(args.filename)
-    else:
-        parseLogs(args.filename, int(args.interval))
-
-run(args)

src/data/BackendCounters.cpp

@@ -17,12 +17,35 @@
 */
 //==============================================================================
 
-#include <data/BackendCounters.h>
+#include "data/BackendCounters.hpp"
 
-#include <util/prometheus/Prometheus.h>
+#include "util/Assert.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
+
+#include <boost/json/object.hpp>
+
+#include <chrono>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
 
 namespace data {
 
+namespace {
+
+std::vector<std::int64_t> const histogramBuckets{1, 2, 5, 10, 20, 50, 100, 200, 500, 700, 1000};
+
+std::int64_t
+durationInMillisecondsSince(std::chrono::steady_clock::time_point const startTime)
+{
+    return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startTime).count();
+}
+
+}  // namespace
+
 using namespace util::prometheus;
 
 BackendCounters::BackendCounters()
@@ -43,6 +66,18 @@ BackendCounters::BackendCounters()
       ))
     , asyncWriteCounters_{"write_async"}
     , asyncReadCounters_{"read_async"}
+    , readDurationHistogram_(PrometheusService::histogramInt(
+          "backend_duration_milliseconds_histogram",
+          Labels({Label{"operation", "read"}}),
+          histogramBuckets,
+          "The duration of backend read operations including retries"
+      ))
+    , writeDurationHistogram_(PrometheusService::histogramInt(
+          "backend_duration_milliseconds_histogram",
+          Labels({Label{"operation", "write"}}),
+          histogramBuckets,
+          "The duration of backend write operations including retries"
+      ))
 {
 }
 
@@ -60,9 +95,10 @@ BackendCounters::registerTooBusy()
 }
 
 void
-BackendCounters::registerWriteSync()
+BackendCounters::registerWriteSync(std::chrono::steady_clock::time_point const startTime)
 {
     ++writeSyncCounter_.get();
+    writeDurationHistogram_.get().observe(durationInMillisecondsSince(startTime));
 }
 
 void
@@ -78,9 +114,11 @@ BackendCounters::registerWriteStarted()
 }
 
 void
-BackendCounters::registerWriteFinished()
+BackendCounters::registerWriteFinished(std::chrono::steady_clock::time_point const startTime)
 {
     asyncWriteCounters_.registerFinished(1u);
+    auto const duration = durationInMillisecondsSince(startTime);
+    writeDurationHistogram_.get().observe(duration);
 }
 
 void
@@ -96,9 +134,12 @@ BackendCounters::registerReadStarted(std::uint64_t const count)
 }
 
 void
-BackendCounters::registerReadFinished(std::uint64_t const count)
+BackendCounters::registerReadFinished(std::chrono::steady_clock::time_point const startTime, std::uint64_t const count)
 {
     asyncReadCounters_.registerFinished(count);
+    auto const duration = durationInMillisecondsSince(startTime);
+    for (std::uint64_t i = 0; i < count; ++i)
+        readDurationHistogram_.get().observe(duration);
 }
 
 void
@@ -161,7 +202,10 @@ BackendCounters::AsyncOperationCounters::registerStarted(std::uint64_t const cou
 void
 BackendCounters::AsyncOperationCounters::registerFinished(std::uint64_t const count)
 {
-    assert(pendingCounter_.get().value() >= static_cast<std::int64_t>(count));
+    ASSERT(
+        pendingCounter_.get().value() >= static_cast<std::int64_t>(count),
+        "Finished operations can't be more than pending"
+    );
     pendingCounter_.get() -= count;
     completedCounter_.get() += count;
 }
@@ -175,7 +219,9 @@ BackendCounters::AsyncOperationCounters::registerRetry(std::uint64_t count)
 void
 BackendCounters::AsyncOperationCounters::registerError(std::uint64_t count)
 {
-    assert(pendingCounter_.get().value() >= static_cast<std::int64_t>(count));
+    ASSERT(
+        pendingCounter_.get().value() >= static_cast<std::int64_t>(count), "Error operations can't be more than pending"
+    );
     pendingCounter_.get() -= count;
     errorCounter_.get() += count;
 }
@@ -187,7 +233,8 @@ BackendCounters::AsyncOperationCounters::report() const
         {name_ + "_pending", pendingCounter_.get().value()},
         {name_ + "_completed", completedCounter_.get().value()},
         {name_ + "_retry", retryCounter_.get().value()},
-        {name_ + "_error", errorCounter_.get().value()}};
+        {name_ + "_error", errorCounter_.get().value()}
+    };
 }
 
 }  // namespace data

src/data/BackendCounters.hpp

@@ -19,37 +19,60 @@
 
 #pragma once
 
-#include <util/prometheus/Prometheus.h>
+#include "util/prometheus/Counter.hpp"
+#include "util/prometheus/Gauge.hpp"
+#include "util/prometheus/Histogram.hpp"
 
 #include <boost/json/object.hpp>
 
-#include <atomic>
+#include <chrono>
+#include <cstdint>
 #include <functional>
 #include <memory>
-#include <utility>
+#include <string>
 
 namespace data {
 
 /**
  * @brief A concept for a class that can be used to count backend operations.
  */
-// clang-format off
 template <typename T>
 concept SomeBackendCounters = requires(T a) {
     typename T::PtrType;
-    { a.registerTooBusy() } -> std::same_as<void>;
-    { a.registerWriteSync() } -> std::same_as<void>;
-    { a.registerWriteSyncRetry() } -> std::same_as<void>;
-    { a.registerWriteStarted() } -> std::same_as<void>;
-    { a.registerWriteFinished() } -> std::same_as<void>;
-    { a.registerWriteRetry() } -> std::same_as<void>;
-    { a.registerReadStarted(std::uint64_t{}) } -> std::same_as<void>;
-    { a.registerReadFinished(std::uint64_t{}) } -> std::same_as<void>;
-    { a.registerReadRetry(std::uint64_t{}) } -> std::same_as<void>;
-    { a.registerReadError(std::uint64_t{}) } -> std::same_as<void>;
-    { a.report() } -> std::same_as<boost::json::object>;
+    {
+        a.registerTooBusy()
+    } -> std::same_as<void>;
+    {
+        a.registerWriteSync(std::chrono::steady_clock::time_point{})
+    } -> std::same_as<void>;
+    {
+        a.registerWriteSyncRetry()
+    } -> std::same_as<void>;
+    {
+        a.registerWriteStarted()
+    } -> std::same_as<void>;
+    {
+        a.registerWriteFinished(std::chrono::steady_clock::time_point{})
+    } -> std::same_as<void>;
+    {
+        a.registerWriteRetry()
+    } -> std::same_as<void>;
+    {
+        a.registerReadStarted(std::uint64_t{})
+    } -> std::same_as<void>;
+    {
+        a.registerReadFinished(std::chrono::steady_clock::time_point{}, std::uint64_t{})
+    } -> std::same_as<void>;
+    {
+        a.registerReadRetry(std::uint64_t{})
+    } -> std::same_as<void>;
+    {
+        a.registerReadError(std::uint64_t{})
+    } -> std::same_as<void>;
+    {
+        a.report()
+    } -> std::same_as<boost::json::object>;
 };
-// clang-format on
 
 /**
  * @brief Holds statistics about the backend.
@@ -67,7 +90,7 @@ public:
     registerTooBusy();
 
     void
-    registerWriteSync();
+    registerWriteSync(std::chrono::steady_clock::time_point startTime);
 
     void
     registerWriteSyncRetry();
@@ -76,7 +99,7 @@ public:
     registerWriteStarted();
 
     void
-    registerWriteFinished();
+    registerWriteFinished(std::chrono::steady_clock::time_point startTime);
 
     void
     registerWriteRetry();
@@ -85,7 +108,7 @@ public:
     registerReadStarted(std::uint64_t count = 1u);
 
     void
-    registerReadFinished(std::uint64_t count = 1u);
+    registerReadFinished(std::chrono::steady_clock::time_point startTime, std::uint64_t count = 1u);
 
     void
     registerReadRetry(std::uint64_t count = 1u);
@@ -133,6 +156,9 @@ private:
 
     AsyncOperationCounters asyncWriteCounters_{"write_async"};
     AsyncOperationCounters asyncReadCounters_{"read_async"};
+
+    std::reference_wrapper<util::prometheus::HistogramInt> readDurationHistogram_;
+    std::reference_wrapper<util::prometheus::HistogramInt> writeDurationHistogram_;
 };
 
 }  // namespace data

src/data/BackendFactory.hpp

@@ -19,12 +19,18 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <data/CassandraBackend.h>
-#include <util/config/Config.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "data/CassandraBackend.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <memory>
+#include <stdexcept>
+#include <string>
 
 namespace data {
 
@@ -55,10 +61,8 @@ make_Backend(util::Config const& config)
         throw std::runtime_error("Invalid database type");
 
     auto const rng = backend->hardFetchLedgerRangeNoThrow();
-    if (rng) {
-        backend->updateRange(rng->minSequence);
-        backend->updateRange(rng->maxSequence);
-    }
+    if (rng)
+        backend->setRange(rng->minSequence, rng->maxSequence);
 
     LOG(log.info()) << "Constructed BackendInterface Successfully";
     return backend;

src/data/BackendInterface.cpp

@@ -17,11 +17,31 @@
 */
 //==============================================================================
 
-#include <data/BackendInterface.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
 
+#include "data/Types.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
+#include <ripple/protocol/Fees.h>
 #include <ripple/protocol/Indexes.h>
+#include <ripple/protocol/SField.h>
 #include <ripple/protocol/STLedgerEntry.h>
+#include <ripple/protocol/Serializer.h>
+
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <vector>
 
 // local to compilation unit loggers
 namespace {
@@ -43,7 +63,7 @@ BackendInterface::finishWrites(std::uint32_t const ledgerSequence)
 void
 BackendInterface::writeLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob)
 {
-    assert(key.size() == sizeof(ripple::uint256));
+    ASSERT(key.size() == sizeof(ripple::uint256), "Key must be 256 bits");
     doWriteLedgerObject(std::move(key), seq, std::move(blob));
 }
 
@@ -109,6 +129,7 @@ BackendInterface::fetchLedgerObjects(
 
     return results;
 }
+
 // Fetches the successor to key/index
 std::optional<ripple::uint256>
 BackendInterface::fetchSuccessorKey(
@@ -155,7 +176,7 @@ BackendInterface::fetchBookOffers(
     // TODO try to speed this up. This can take a few seconds. The goal is
     // to get it down to a few hundred milliseconds.
     BookOffersPage page;
-    const ripple::uint256 bookEnd = ripple::getQualityNext(book);
+    ripple::uint256 const bookEnd = ripple::getQualityNext(book);
     ripple::uint256 uTipIndex = book;
     std::vector<ripple::uint256> keys;
     auto getMillis = [](auto diff) { return std::chrono::duration_cast<std::chrono::milliseconds>(diff).count(); };
@@ -178,7 +199,8 @@ BackendInterface::fetchBookOffers(
         while (keys.size() < limit) {
             ++numPages;
             ripple::STLedgerEntry const sle{
-                ripple::SerialIter{offerDir->blob.data(), offerDir->blob.size()}, offerDir->key};
+                ripple::SerialIter{offerDir->blob.data(), offerDir->blob.size()}, offerDir->key
+            };
             auto indexes = sle.getFieldV256(ripple::sfIndexes);
             keys.insert(keys.end(), indexes.begin(), indexes.end());
             auto next = sle.getFieldU64(ripple::sfIndexNext);
@@ -188,7 +210,7 @@ BackendInterface::fetchBookOffers(
             }
             auto nextKey = ripple::keylet::page(uTipIndex, next);
             auto nextDir = fetchLedgerObject(nextKey.key, ledgerSequence, yield);
-            assert(nextDir);
+            ASSERT(nextDir.has_value(), "Next dir must exist");
             offerDir->blob = *nextDir;
             offerDir->key = nextKey.key;
         }
@@ -200,7 +222,7 @@ BackendInterface::fetchBookOffers(
     for (size_t i = 0; i < keys.size() && i < limit; ++i) {
         LOG(gLog.trace()) << "Key = " << ripple::strHex(keys[i]) << " blob = " << ripple::strHex(objs[i])
                           << " ledgerSequence = " << ledgerSequence;
-        assert(!objs[i].empty());
+        ASSERT(!objs[i].empty(), "Ledger object can't be empty");
         page.offers.push_back({keys[i], objs[i]});
     }
     auto end = std::chrono::system_clock::now();
@@ -234,7 +256,14 @@ void
 BackendInterface::updateRange(uint32_t newMax)
 {
     std::scoped_lock const lck(rngMtx_);
-    assert(!range || newMax >= range->maxSequence);
+
+    ASSERT(
+        !range || newMax >= range->maxSequence,
+        "Range shouldn't exist yet or newMax should be greater. newMax = {}, range->maxSequence = {}",
+        newMax,
+        range->maxSequence
+    );
+
     if (!range) {
         range = {newMax, newMax};
     } else {
@@ -242,6 +271,19 @@ BackendInterface::updateRange(uint32_t newMax)
     }
 }
 
+void
+BackendInterface::setRange(uint32_t min, uint32_t max, bool force)
+{
+    std::scoped_lock const lck(rngMtx_);
+
+    if (!force) {
+        ASSERT(min <= max, "Range min must be less than or equal to max");
+        ASSERT(not range.has_value(), "Range was already set");
+    }
+
+    range = {min, max};
+}
+
 LedgerPage
 BackendInterface::fetchLedgerPage(
     std::optional<ripple::uint256> const& cursor,
@@ -302,14 +344,42 @@ BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context
     ripple::SerialIter it(bytes->data(), bytes->size());
     ripple::SLE const sle{it, key};
 
-    if (sle.getFieldIndex(ripple::sfBaseFee) != -1)
-        fees.base = sle.getFieldU64(ripple::sfBaseFee);
+    // XRPFees amendment introduced new fields for fees calculations.
+    // New fields are set and the old fields are removed via `set_fees` tx.
+    // Fallback to old fields if `set_fees` was not yet used to update the fields on this tx.
+    auto hasNewFields = false;
+    {
+        auto const baseFeeXRP = sle.at(~ripple::sfBaseFeeDrops);
+        auto const reserveBaseXRP = sle.at(~ripple::sfReserveBaseDrops);
+        auto const reserveIncrementXRP = sle.at(~ripple::sfReserveIncrementDrops);
 
-    if (sle.getFieldIndex(ripple::sfReserveBase) != -1)
-        fees.reserve = sle.getFieldU32(ripple::sfReserveBase);
+        if (baseFeeXRP)
+            fees.base = baseFeeXRP->xrp();
 
-    if (sle.getFieldIndex(ripple::sfReserveIncrement) != -1)
-        fees.increment = sle.getFieldU32(ripple::sfReserveIncrement);
+        if (reserveBaseXRP)
+            fees.reserve = reserveBaseXRP->xrp();
+
+        if (reserveIncrementXRP)
+            fees.increment = reserveIncrementXRP->xrp();
+
+        hasNewFields = baseFeeXRP || reserveBaseXRP || reserveIncrementXRP;
+    }
+
+    if (not hasNewFields) {
+        // Fallback to old fields
+        auto const baseFee = sle.at(~ripple::sfBaseFee);
+        auto const reserveBase = sle.at(~ripple::sfReserveBase);
+        auto const reserveIncrement = sle.at(~ripple::sfReserveIncrement);
+
+        if (baseFee)
+            fees.base = baseFee.value();
+
+        if (reserveBase)
+            fees.reserve = reserveBase.value();
+
+        if (reserveIncrement)
+            fees.increment = reserveIncrement.value();
+    }
 
     return fees;
 }

src/data/BackendInterface.hpp

@@ -19,19 +19,32 @@
 
 #pragma once
 
-#include <data/DBHelpers.h>
-#include <data/LedgerCache.h>
-#include <data/Types.h>
-#include <util/config/Config.h>
-#include <util/log/Logger.h>
+#include "data/DBHelpers.hpp"
+#include "data/LedgerCache.hpp"
+#include "data/Types.hpp"
+#include "util/log/Logger.hpp"
 
-#include <ripple/protocol/Fees.h>
-#include <ripple/protocol/LedgerHeader.h>
+#include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/json.hpp>
+#include <boost/json/object.hpp>
+#include <boost/utility/result_of.hpp>
+#include <ripple/basics/base_uint.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/Fees.h>
+#include <ripple/protocol/LedgerHeader.h>
 
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <optional>
+#include <shared_mutex>
+#include <string>
 #include <thread>
 #include <type_traits>
+#include <vector>
 
 namespace data {
 
@@ -86,7 +99,7 @@ synchronous(FnType&& func)
     boost::asio::io_context ctx;
 
     using R = typename boost::result_of<FnType(boost::asio::yield_context)>::type;
-    if constexpr (!std::is_same<R, void>::value) {
+    if constexpr (!std::is_same_v<R, void>) {
         R res;
         boost::asio::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func, &res](auto yield) {
             res = func(yield);
@@ -191,6 +204,16 @@ public:
     void
     updateRange(uint32_t newMax);
 
+    /**
+     * @brief Sets the range of sequences that are stored in the DB.
+     *
+     * @param min The new minimum sequence available
+     * @param max The new maximum sequence available
+     * @param force If set to true, the range will be set even if it's already set
+     */
+    void
+    setRange(uint32_t min, uint32_t max, bool force = false);
+
     /**
      * @brief Fetch the fees from a specific ledger sequence.
      *
@@ -521,7 +544,7 @@ public:
      * @param data A vector of NFTsData objects representing the NFTs
      */
     virtual void
-    writeNFTs(std::vector<NFTsData>&& data) = 0;
+    writeNFTs(std::vector<NFTsData> const& data) = 0;
 
     /**
      * @brief Write a new set of account transactions.
@@ -529,7 +552,7 @@ public:
      * @param data A vector of AccountTransactionsData objects representing the account transactions
      */
     virtual void
-    writeAccountTransactions(std::vector<AccountTransactionsData>&& data) = 0;
+    writeAccountTransactions(std::vector<AccountTransactionsData> data) = 0;
 
     /**
      * @brief Write NFTs transactions.
@@ -537,7 +560,7 @@ public:
      * @param data A vector of NFTTransactionsData objects
      */
     virtual void
-    writeNFTTransactions(std::vector<NFTTransactionsData>&& data) = 0;
+    writeNFTTransactions(std::vector<NFTTransactionsData> const& data) = 0;
 
     /**
      * @brief Write a new successor.

src/data/CassandraBackend.hpp

@@ -19,19 +19,42 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <data/cassandra/Concepts.h>
-#include <data/cassandra/Handle.h>
-#include <data/cassandra/Schema.h>
-#include <data/cassandra/SettingsProvider.h>
-#include <data/cassandra/impl/ExecutionStrategy.h>
-#include <util/LedgerUtils.h>
-#include <util/Profiler.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "data/cassandra/Concepts.hpp"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Schema.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ExecutionStrategy.hpp"
+#include "util/Assert.hpp"
+#include "util/LedgerUtils.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
 
+#include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+#include <cassandra.h>
+#include <ripple/basics/Blob.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
+#include <ripple/protocol/AccountID.h>
 #include <ripple/protocol/LedgerHeader.h>
 #include <ripple/protocol/nft.h>
-#include <boost/asio/spawn.hpp>
+
+#include <atomic>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <limits>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
 
 namespace data::cassandra {
 
@@ -493,41 +516,39 @@ public:
         if (nftIDs.size() == limit)
             ret.cursor = nftIDs.back();
 
-        auto const nftQueryStatement = schema_->selectNFTBulk.bind(nftIDs);
-        nftQueryStatement.bindAt(1, ledgerSequence);
+        std::vector<Statement> selectNFTStatements;
+        selectNFTStatements.reserve(nftIDs.size());
 
-        // Fetch all the NFT data, meanwhile filtering out the NFTs that are not within the ledger range
-        auto const nftRes = executor_.read(yield, nftQueryStatement);
-        auto const& nftQueryResults = nftRes.value();
+        std::transform(
+            std::cbegin(nftIDs),
+            std::cend(nftIDs),
+            std::back_inserter(selectNFTStatements),
+            [&](auto const& nftID) { return schema_->selectNFT.bind(nftID, ledgerSequence); }
+        );
 
-        if (not nftQueryResults.hasRows()) {
-            LOG(log_.debug()) << "No rows returned";
-            return {};
+        auto const nftInfos = executor_.readEach(yield, selectNFTStatements);
+
+        std::vector<Statement> selectNFTURIStatements;
+        selectNFTURIStatements.reserve(nftIDs.size());
+
+        std::transform(
+            std::cbegin(nftIDs),
+            std::cend(nftIDs),
+            std::back_inserter(selectNFTURIStatements),
+            [&](auto const& nftID) { return schema_->selectNFTURI.bind(nftID, ledgerSequence); }
+        );
+
+        auto const nftUris = executor_.readEach(yield, selectNFTURIStatements);
+
+        for (auto i = 0u; i < nftIDs.size(); i++) {
+            if (auto const maybeRow = nftInfos[i].template get<uint32_t, ripple::AccountID, bool>(); maybeRow) {
+                auto [seq, owner, isBurned] = *maybeRow;
+                NFT nft(nftIDs[i], seq, owner, isBurned);
+                if (auto const maybeUri = nftUris[i].template get<ripple::Blob>(); maybeUri)
+                    nft.uri = *maybeUri;
+                ret.nfts.push_back(nft);
+            }
         }
-
-        auto const nftURIQueryStatement = schema_->selectNFTURIBulk.bind(nftIDs);
-        nftURIQueryStatement.bindAt(1, ledgerSequence);
-
-        // Get the URI for each NFT, but it's possible that URI doesn't exist
-        auto const uriRes = executor_.read(yield, nftURIQueryStatement);
-        auto const& nftURIQueryResults = uriRes.value();
-
-        std::unordered_map<std::string, Blob> nftURIMap;
-        for (auto const [nftID, uri] : extract<ripple::uint256, Blob>(nftURIQueryResults))
-            nftURIMap.insert({ripple::strHex(nftID), uri});
-
-        for (auto const [nftID, seq, owner, isBurned] :
-             extract<ripple::uint256, std::uint32_t, ripple::AccountID, bool>(nftQueryResults)) {
-            NFT nft;
-            nft.tokenID = nftID;
-            nft.ledgerSequence = seq;
-            nft.owner = owner;
-            nft.isBurned = isBurned;
-            if (nftURIMap.contains(ripple::strHex(nft.tokenID)))
-                nft.uri = nftURIMap.at(ripple::strHex(nft.tokenID));
-            ret.nfts.push_back(nft);
-        }
-
         return ret;
     }
 
@@ -622,7 +643,7 @@ public:
             );
         });
 
-        assert(numHashes == results.size());
+        ASSERT(numHashes == results.size(), "Number of hashes and results must match");
         LOG(log_.debug()) << "Fetched " << numHashes << " transactions from Cassandra in " << timeDiff
                           << " milliseconds";
         return results;
@@ -735,14 +756,14 @@ public:
     {
         LOG(log_.trace()) << "Writing successor. key = " << key.size() << " bytes. "
                           << " seq = " << std::to_string(seq) << " successor = " << successor.size() << " bytes.";
-        assert(!key.empty());
-        assert(!successor.empty());
+        ASSERT(!key.empty(), "Key must not be empty");
+        ASSERT(!successor.empty(), "Successor must not be empty");
 
         executor_.write(schema_->insertSuccessor, std::move(key), seq, std::move(successor));
     }
 
     void
-    writeAccountTransactions(std::vector<AccountTransactionsData>&& data) override
+    writeAccountTransactions(std::vector<AccountTransactionsData> data) override
     {
         std::vector<Statement> statements;
         statements.reserve(data.size() * 10);  // assume 10 transactions avg
@@ -766,7 +787,7 @@ public:
     }
 
     void
-    writeNFTTransactions(std::vector<NFTTransactionsData>&& data) override
+    writeNFTTransactions(std::vector<NFTTransactionsData> const& data) override
     {
         std::vector<Statement> statements;
         statements.reserve(data.size());
@@ -798,7 +819,7 @@ public:
     }
 
     void
-    writeNFTs(std::vector<NFTsData>&& data) override
+    writeNFTs(std::vector<NFTsData> const& data) override
     {
         std::vector<Statement> statements;
         statements.reserve(data.size() * 3);

src/data/DBHelpers.hpp

@@ -20,15 +20,24 @@
 /** @file */
 #pragma once
 
-#include <ripple/basics/Log.h>
-#include <ripple/basics/StringUtilities.h>
-#include <ripple/protocol/SField.h>
-#include <ripple/protocol/STAccount.h>
-#include <ripple/protocol/TxMeta.h>
+#include "util/Assert.hpp"
 
 #include <boost/container/flat_set.hpp>
+#include <ripple/basics/Blob.h>
+#include <ripple/basics/Log.h>
+#include <ripple/basics/StringUtilities.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/SField.h>
+#include <ripple/protocol/STAccount.h>
+#include <ripple/protocol/STLedgerEntry.h>
+#include <ripple/protocol/Serializer.h>
+#include <ripple/protocol/TxMeta.h>
 
-#include <data/Types.h>
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <string>
 
 /**
  * @brief Struct used to keep track of what to write to account_transactions/account_tx tables.
@@ -233,7 +242,7 @@ getBookBase(T const& key)
 {
     static constexpr size_t KEY_SIZE = 24;
 
-    assert(key.size() == ripple::uint256::size());
+    ASSERT(key.size() == ripple::uint256::size(), "Invalid key size {}", key.size());
 
     ripple::uint256 ret;
     for (size_t i = 0; i < KEY_SIZE; ++i)

src/data/LedgerCache.cpp

@@ -17,7 +17,19 @@
 */
 //==============================================================================
 
-#include <data/LedgerCache.h>
+#include "data/LedgerCache.hpp"
+
+#include "data/Types.hpp"
+#include "util/Assert.hpp"
+
+#include <ripple/basics/base_uint.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <vector>
 
 namespace data {
 
@@ -28,6 +40,16 @@ LedgerCache::latestLedgerSequence() const
     return latestSeq_;
 }
 
+void
+LedgerCache::waitUntilCacheContainsSeq(uint32_t seq)
+{
+    if (disabled_)
+        return;
+    std::unique_lock lock(mtx_);
+    cv_.wait(lock, [this, seq] { return latestSeq_ >= seq; });
+    return;
+}
+
 void
 LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground)
 {
@@ -37,7 +59,12 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
     {
         std::scoped_lock const lck{mtx_};
         if (seq > latestSeq_) {
-            assert(seq == latestSeq_ + 1 || latestSeq_ == 0);
+            ASSERT(
+                seq == latestSeq_ + 1 || latestSeq_ == 0,
+                "New sequense must be either next or first. seq = {}, latestSeq_ = {}",
+                seq,
+                latestSeq_
+            );
             latestSeq_ = seq;
         }
         for (auto const& obj : objs) {
@@ -55,6 +82,7 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
                     deletes_.insert(obj.key);
             }
         }
+        cv_.notify_all();
     }
 }
 

src/data/LedgerCache.hpp

@@ -19,14 +19,23 @@
 
 #pragma once
 
+#include "data/Types.hpp"
+#include "util/prometheus/Counter.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
+
 #include <ripple/basics/base_uint.h>
 #include <ripple/basics/hardened_hash.h>
-#include <data/Types.h>
+
+#include <atomic>
+#include <condition_variable>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
 #include <map>
-#include <mutex>
+#include <optional>
 #include <shared_mutex>
-#include <util/prometheus/Prometheus.h>
-#include <utility>
+#include <unordered_set>
 #include <vector>
 
 namespace data {
@@ -65,6 +74,7 @@ class LedgerCache {
     std::map<ripple::uint256, CacheEntry> map_;
 
     mutable std::shared_mutex mtx_;
+    std::condition_variable_any cv_;
     uint32_t latestSeq_ = 0;
     std::atomic_bool full_ = false;
     std::atomic_bool disabled_ = false;
@@ -162,6 +172,9 @@ public:
      */
     float
     getSuccessorHitRate() const;
+
+    void
+    waitUntilCacheContainsSeq(uint32_t seq);
 };
 
 }  // namespace data

src/data/Types.hpp

@@ -22,8 +22,10 @@
 #include <ripple/basics/base_uint.h>
 #include <ripple/protocol/AccountID.h>
 
+#include <cstdint>
 #include <optional>
-#include <string>
+#include <string_view>
+#include <tuple>
 #include <utility>
 #include <vector>
 

src/data/cassandra/Concepts.hpp

@@ -19,15 +19,20 @@
 
 #pragma once
 
-#include <data/cassandra/Types.h>
+#include "data/cassandra/Types.hpp"
 
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/json.hpp>
+#include <boost/json/object.hpp>
 
 #include <chrono>
 #include <concepts>
+#include <cstdint>
 #include <optional>
 #include <string>
+#include <utility>
+#include <vector>
 
 namespace data::cassandra {
 

src/data/cassandra/Error.hpp

@@ -21,6 +21,8 @@
 
 #include <cassandra.h>
 
+#include <cstdint>
+#include <ostream>
 #include <string>
 #include <utility>
 

src/data/cassandra/Handle.cpp

@@ -17,7 +17,18 @@
 */
 //==============================================================================
 
-#include <data/cassandra/Handle.h>
+#include "data/cassandra/Handle.hpp"
+
+#include "data/cassandra/Types.hpp"
+
+#include <cassandra.h>
+
+#include <functional>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
 
 namespace data::cassandra {
 

src/data/cassandra/Handle.hpp

@@ -19,22 +19,20 @@
 
 #pragma once
 
-#include <data/cassandra/Error.h>
-#include <data/cassandra/Types.h>
-#include <data/cassandra/impl/Batch.h>
-#include <data/cassandra/impl/Cluster.h>
-#include <data/cassandra/impl/Future.h>
-#include <data/cassandra/impl/ManagedObject.h>
-#include <data/cassandra/impl/Result.h>
-#include <data/cassandra/impl/Session.h>
-#include <data/cassandra/impl/Statement.h>
-#include <util/Expected.h>
+#include "data/cassandra/Error.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/Batch.hpp"
+#include "data/cassandra/impl/Cluster.hpp"
+#include "data/cassandra/impl/Future.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Result.hpp"
+#include "data/cassandra/impl/Session.hpp"
+#include "data/cassandra/impl/Statement.hpp"
 
 #include <cassandra.h>
 
-#include <chrono>
-#include <compare>
-#include <iterator>
+#include <functional>
+#include <string_view>
 #include <vector>
 
 namespace data::cassandra {

src/data/cassandra/Schema.hpp

@@ -19,16 +19,19 @@
 
 #pragma once
 
-#include <data/cassandra/Concepts.h>
-#include <data/cassandra/Handle.h>
-#include <data/cassandra/SettingsProvider.h>
-#include <data/cassandra/Types.h>
-#include <util/Expected.h>
-#include <util/config/Config.h>
-#include <util/log/Logger.h>
+#include "data/cassandra/Concepts.hpp"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "util/log/Logger.hpp"
 
 #include <fmt/compile.h>
 
+#include <functional>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <vector>
+
 namespace data::cassandra {
 
 template <SomeSettingsProvider SettingsProviderType>
@@ -592,20 +595,6 @@ public:
             ));
         }();
 
-        PreparedStatement selectNFTBulk = [this]() {
-            return handle_.get().prepare(fmt::format(
-                R"(
-                SELECT token_id, sequence, owner, is_burned
-                  FROM {}    
-                 WHERE token_id IN ?
-                   AND sequence <= ?
-              ORDER BY sequence DESC
-   PER PARTITION LIMIT 1
-                )",
-                qualifiedTableName(settingsProvider_.get(), "nf_tokens")
-            ));
-        }();
-
         PreparedStatement selectNFTURI = [this]() {
             return handle_.get().prepare(fmt::format(
                 R"(
@@ -620,20 +609,6 @@ public:
             ));
         }();
 
-        PreparedStatement selectNFTURIBulk = [this]() {
-            return handle_.get().prepare(fmt::format(
-                R"(
-                SELECT token_id, uri
-                  FROM {}    
-                 WHERE token_id IN ?
-                   AND sequence <= ?
-              ORDER BY sequence DESC
-   PER PARTITION LIMIT 1
-                )",
-                qualifiedTableName(settingsProvider_.get(), "nf_token_uris")
-            ));
-        }();
-
         PreparedStatement selectNFTTx = [this]() {
             return handle_.get().prepare(fmt::format(
                 R"(

src/data/cassandra/SettingsProvider.cpp

@@ -17,17 +17,28 @@
 */
 //==============================================================================
 
-#include <data/cassandra/SettingsProvider.h>
-#include <data/cassandra/impl/Cluster.h>
-#include <data/cassandra/impl/Statement.h>
-#include <util/Constants.h>
-#include <util/config/Config.h>
+#include "data/cassandra/SettingsProvider.hpp"
 
-#include <boost/json.hpp>
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/Cluster.hpp"
+#include "util/Constants.hpp"
+#include "util/config/Config.hpp"
 
+#include <boost/json/conversion.hpp>
+#include <boost/json/value.hpp>
+
+#include <cerrno>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
 #include <fstream>
+#include <ios>
+#include <iterator>
+#include <optional>
+#include <stdexcept>
 #include <string>
-#include <thread>
+#include <system_error>
 
 namespace data::cassandra {
 
@@ -112,8 +123,8 @@ SettingsProvider::parseSettings() const
         config_.valueOr<uint32_t>("max_read_requests_outstanding", settings.maxReadRequestsOutstanding);
     settings.coreConnectionsPerHost =
         config_.valueOr<uint32_t>("core_connections_per_host", settings.coreConnectionsPerHost);
-
     settings.queueSizeIO = config_.maybeValue<uint32_t>("queue_size_io");
+    settings.writeBatchSize = config_.valueOr<std::size_t>("write_batch_size", settings.writeBatchSize);
 
     auto const connectTimeoutSecond = config_.maybeValue<uint32_t>("connect_timeout");
     if (connectTimeoutSecond)

src/data/cassandra/SettingsProvider.hpp

@@ -19,11 +19,14 @@
 
 #pragma once
 
-#include <data/cassandra/Handle.h>
-#include <data/cassandra/Types.h>
-#include <util/Expected.h>
-#include <util/config/Config.h>
-#include <util/log/Logger.h>
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
+
+#include <cstdint>
+#include <optional>
+#include <string>
 
 namespace data::cassandra {
 

src/data/cassandra/Types.hpp

@@ -19,9 +19,9 @@
 
 #pragma once
 
-#include <util/Expected.h>
+#include "util/Expected.hpp"
 
-#include <string>
+#include <cstdint>
 
 namespace data::cassandra {
 

src/data/cassandra/impl/AsyncExecutor.hpp

@@ -19,17 +19,20 @@
 
 #pragma once
 
-#include <data/cassandra/Concepts.h>
-#include <data/cassandra/Handle.h>
-#include <data/cassandra/Types.h>
-#include <data/cassandra/impl/RetryPolicy.h>
-#include <util/Expected.h>
-#include <util/log/Logger.h>
+#include "data/cassandra/Concepts.hpp"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/RetryPolicy.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
 
 #include <functional>
 #include <memory>
+#include <mutex>
+#include <optional>
+#include <utility>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/Batch.cpp

@@ -17,12 +17,17 @@
 */
 //==============================================================================
 
-#include <data/cassandra/Error.h>
-#include <data/cassandra/impl/Batch.h>
-#include <data/cassandra/impl/Statement.h>
-#include <util/Expected.h>
+#include "data/cassandra/impl/Batch.hpp"
 
-#include <exception>
+#include "data/cassandra/Error.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Statement.hpp"
+#include "util/Expected.hpp"
+
+#include <cassandra.h>
+
+#include <stdexcept>
 #include <vector>
 
 namespace {

src/data/cassandra/impl/Batch.hpp

@@ -19,11 +19,13 @@
 
 #pragma once
 
-#include <data/cassandra/Types.h>
-#include <data/cassandra/impl/ManagedObject.h>
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 
+#include <vector>
+
 namespace data::cassandra::detail {
 
 struct Batch : public ManagedObject<CassBatch> {

src/data/cassandra/impl/Cluster.cpp

@@ -17,15 +17,18 @@
 */
 //==============================================================================
 
-#include <data/cassandra/impl/Cluster.h>
-#include <data/cassandra/impl/SslContext.h>
-#include <data/cassandra/impl/Statement.h>
-#include <util/Expected.h>
+#include "data/cassandra/impl/Cluster.hpp"
 
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/SslContext.hpp"
+#include "util/log/Logger.hpp"
+
+#include <cassandra.h>
 #include <fmt/core.h>
 
-#include <exception>
-#include <vector>
+#include <stdexcept>
+#include <string>
+#include <variant>
 
 namespace {
 constexpr auto clusterDeleter = [](CassCluster* ptr) { cass_cluster_free(ptr); };
@@ -80,6 +83,7 @@ Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), c
     LOG(log_.info()) << "Threads: " << settings.threads;
     LOG(log_.info()) << "Core connections per host: " << settings.coreConnectionsPerHost;
     LOG(log_.info()) << "IO queue size: " << queueSize;
+    LOG(log_.info()) << "Batched writes auto-chunk size: " << settings.writeBatchSize;
 }
 
 void
@@ -88,7 +92,8 @@ Cluster::setupConnection(Settings const& settings)
     std::visit(
         overloadSet{
             [this](Settings::ContactPoints const& points) { setupContactPoints(points); },
-            [this](Settings::SecureConnectionBundle const& bundle) { setupSecureBundle(bundle); }},
+            [this](Settings::SecureConnectionBundle const& bundle) { setupSecureBundle(bundle); }
+        },
         settings.connectionInfo
     );
 }

src/data/cassandra/impl/Cluster.hpp

@@ -19,8 +19,8 @@
 
 #pragma once
 
-#include <data/cassandra/impl/ManagedObject.h>
-#include <util/log/Logger.h>
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "util/log/Logger.hpp"
 
 #include <cassandra.h>
 
@@ -42,6 +42,8 @@ struct Settings {
     static constexpr std::size_t DEFAULT_CONNECTION_TIMEOUT = 10000;
     static constexpr uint32_t DEFAULT_MAX_WRITE_REQUESTS_OUTSTANDING = 10'000;
     static constexpr uint32_t DEFAULT_MAX_READ_REQUESTS_OUTSTANDING = 100'000;
+    static constexpr std::size_t DEFAULT_BATCH_SIZE = 20;
+
     /**
      * @brief Represents the configuration of contact points for cassandra.
      */
@@ -81,6 +83,9 @@ struct Settings {
     /** @brief The number of connection per host to always have active */
     uint32_t coreConnectionsPerHost = 1u;
 
+    /** @brief Size of batches when writing */
+    std::size_t writeBatchSize = DEFAULT_BATCH_SIZE;
+
     /** @brief Size of the IO queue */
     std::optional<uint32_t> queueSizeIO{};
 

src/data/cassandra/impl/Collection.hpp

@@ -19,13 +19,16 @@
 
 #pragma once
 
-#include <data/cassandra/impl/ManagedObject.h>
+#include "data/cassandra/impl/ManagedObject.hpp"
 
-#include <ripple/basics/base_uint.h>
 #include <cassandra.h>
+#include <ripple/basics/base_uint.h>
 
+#include <cstdint>
+#include <stdexcept>
 #include <string>
 #include <string_view>
+#include <vector>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/ExecutionStrategy.hpp

@@ -19,24 +19,35 @@
 
 #pragma once
 
-#include <data/BackendCounters.h>
-#include <data/BackendInterface.h>
-#include <data/cassandra/Handle.h>
-#include <data/cassandra/Types.h>
-#include <data/cassandra/impl/AsyncExecutor.h>
-#include <util/Expected.h>
-#include <util/log/Logger.h>
+#include "data/BackendCounters.hpp"
+#include "data/BackendInterface.hpp"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/AsyncExecutor.hpp"
+#include "util/Assert.hpp"
+#include "util/Batching.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
+#include <boost/asio/associated_executor.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/io_service.hpp>
 #include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
 
 #include <atomic>
+#include <chrono>
 #include <condition_variable>
+#include <cstddef>
+#include <cstdint>
 #include <functional>
 #include <memory>
 #include <mutex>
 #include <optional>
+#include <stdexcept>
 #include <thread>
+#include <type_traits>
+#include <vector>
 
 namespace data::cassandra::detail {
 
@@ -58,6 +69,8 @@ class DefaultExecutionStrategy {
     std::uint32_t maxReadRequestsOutstanding_;
     std::atomic_uint32_t numReadRequestsOutstanding_ = 0;
 
+    std::size_t writeBatchSize_;
+
     std::mutex throttleMutex_;
     std::condition_variable throttleCv_;
 
@@ -92,6 +105,7 @@ public:
     )
         : maxWriteRequestsOutstanding_{settings.maxWriteRequestsOutstanding}
         , maxReadRequestsOutstanding_{settings.maxReadRequestsOutstanding}
+        , writeBatchSize_{settings.writeBatchSize}
         , work_{ioc_}
         , handle_{std::cref(handle)}
         , thread_{[this]() { ioc_.run(); }}
@@ -140,10 +154,11 @@ public:
     ResultOrErrorType
     writeSync(StatementType const& statement)
     {
-        counters_->registerWriteSync();
+        auto const startTime = std::chrono::steady_clock::now();
         while (true) {
             auto res = handle_.get().execute(statement);
             if (res) {
+                counters_->registerWriteSync(startTime);
                 return res;
             }
 
@@ -178,6 +193,8 @@ public:
     void
     write(PreparedStatementType const& preparedStatement, Args&&... args)
     {
+        auto const startTime = std::chrono::steady_clock::now();
+
         auto statement = preparedStatement.bind(std::forward<Args>(args)...);
         incrementOutstandingRequestCount();
 
@@ -187,10 +204,10 @@ public:
             ioc_,
             handle_,
             std::move(statement),
-            [this](auto const&) {
+            [this, startTime](auto const&) {
                 decrementOutstandingRequestCount();
 
-                counters_->registerWriteFinished();
+                counters_->registerWriteFinished(startTime);
             },
             [this]() { counters_->registerWriteRetry(); }
         );
@@ -210,20 +227,28 @@ public:
         if (statements.empty())
             return;
 
-        incrementOutstandingRequestCount();
+        util::forEachBatch(std::move(statements), writeBatchSize_, [this](auto begin, auto end) {
+            auto const startTime = std::chrono::steady_clock::now();
+            auto chunk = std::vector<StatementType>{};
 
-        counters_->registerWriteStarted();
-        // Note: lifetime is controlled by std::shared_from_this internally
-        AsyncExecutor<std::decay_t<decltype(statements)>, HandleType>::run(
-            ioc_,
-            handle_,
-            std::move(statements),
-            [this](auto const&) {
-                decrementOutstandingRequestCount();
-                counters_->registerWriteFinished();
-            },
-            [this]() { counters_->registerWriteRetry(); }
-        );
+            chunk.reserve(std::distance(begin, end));
+            std::move(begin, end, std::back_inserter(chunk));
+
+            incrementOutstandingRequestCount();
+            counters_->registerWriteStarted();
+
+            // Note: lifetime is controlled by std::shared_from_this internally
+            AsyncExecutor<std::decay_t<decltype(chunk)>, HandleType>::run(
+                ioc_,
+                handle_,
+                std::move(chunk),
+                [this, startTime](auto const&) {
+                    decrementOutstandingRequestCount();
+                    counters_->registerWriteFinished(startTime);
+                },
+                [this]() { counters_->registerWriteRetry(); }
+            );
+        });
     }
 
     /**
@@ -257,6 +282,8 @@ public:
     [[maybe_unused]] ResultOrErrorType
     read(CompletionTokenType token, std::vector<StatementType> const& statements)
     {
+        auto const startTime = std::chrono::steady_clock::now();
+
         auto const numStatements = statements.size();
         std::optional<FutureWithCallbackType> future;
         counters_->registerReadStarted(numStatements);
@@ -282,7 +309,7 @@ public:
             numReadRequestsOutstanding_ -= numStatements;
 
             if (res) {
-                counters_->registerReadFinished(numStatements);
+                counters_->registerReadFinished(startTime, numStatements);
                 return res;
             }
 
@@ -310,6 +337,8 @@ public:
     [[maybe_unused]] ResultOrErrorType
     read(CompletionTokenType token, StatementType const& statement)
     {
+        auto const startTime = std::chrono::steady_clock::now();
+
         std::optional<FutureWithCallbackType> future;
         counters_->registerReadStarted();
 
@@ -333,7 +362,7 @@ public:
             --numReadRequestsOutstanding_;
 
             if (res) {
-                counters_->registerReadFinished();
+                counters_->registerReadFinished(startTime);
                 return res;
             }
 
@@ -362,6 +391,8 @@ public:
     std::vector<ResultType>
     readEach(CompletionTokenType token, std::vector<StatementType> const& statements)
     {
+        auto const startTime = std::chrono::steady_clock::now();
+
         std::atomic_uint64_t errorsCount = 0u;
         std::atomic_int numOutstanding = statements.size();
         numReadRequestsOutstanding_ += statements.size();
@@ -400,12 +431,12 @@ public:
         numReadRequestsOutstanding_ -= statements.size();
 
         if (errorsCount > 0) {
-            assert(errorsCount <= statements.size());
+            ASSERT(errorsCount <= statements.size(), "Errors number cannot exceed statements number");
             counters_->registerReadError(errorsCount);
-            counters_->registerReadFinished(statements.size() - errorsCount);
+            counters_->registerReadFinished(startTime, statements.size() - errorsCount);
             throw DatabaseTimeout{};
         }
-        counters_->registerReadFinished(statements.size());
+        counters_->registerReadFinished(startTime, statements.size());
 
         std::vector<ResultType> results;
         results.reserve(futures.size());
@@ -422,8 +453,18 @@ public:
             }
         );
 
-        assert(futures.size() == statements.size());
-        assert(results.size() == statements.size());
+        ASSERT(
+            futures.size() == statements.size(),
+            "Futures size must be equal to statements size. Got {} and {}",
+            futures.size(),
+            statements.size()
+        );
+        ASSERT(
+            results.size() == statements.size(),
+            "Results size must be equal to statements size. Got {} and {}",
+            results.size(),
+            statements.size()
+        );
         return results;
     }
 
@@ -455,10 +496,7 @@ private:
     decrementOutstandingRequestCount()
     {
         // sanity check
-        if (numWriteRequestsOutstanding_ == 0) {
-            assert(false);
-            throw std::runtime_error("decrementing num outstanding below 0");
-        }
+        ASSERT(numWriteRequestsOutstanding_ > 0, "Decrementing num outstanding below 0");
         size_t const cur = (--numWriteRequestsOutstanding_);
         {
             // mutex lock required to prevent race condition around spurious

src/data/cassandra/impl/Future.cpp

@@ -17,12 +17,19 @@
 */
 //==============================================================================
 
-#include <data/cassandra/Error.h>
-#include <data/cassandra/impl/Future.h>
-#include <data/cassandra/impl/Result.h>
+#include "data/cassandra/impl/Future.hpp"
 
-#include <exception>
-#include <vector>
+#include "data/cassandra/Error.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Result.hpp"
+
+#include <cassandra.h>
+
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <utility>
 
 namespace {
 constexpr auto futureDeleter = [](CassFuture* ptr) { cass_future_free(ptr); };

src/data/cassandra/impl/Future.hpp

@@ -19,11 +19,14 @@
 
 #pragma once
 
-#include <data/cassandra/Types.h>
-#include <data/cassandra/impl/ManagedObject.h>
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 
+#include <functional>
+#include <memory>
+
 namespace data::cassandra::detail {
 
 struct Future : public ManagedObject<CassFuture> {

src/data/cassandra/impl/ManagedObject.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include <memory>
+#include <stdexcept>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/Result.cpp

@@ -17,7 +17,13 @@
 */
 //==============================================================================
 
-#include <data/cassandra/impl/Result.h>
+#include "data/cassandra/impl/Result.hpp"
+
+#include "data/cassandra/impl/ManagedObject.hpp"
+
+#include <cassandra.h>
+
+#include <cstddef>
 
 namespace {
 constexpr auto resultDeleter = [](CassResult const* ptr) { cass_result_free(ptr); };

src/data/cassandra/impl/Result.hpp

@@ -19,17 +19,25 @@
 
 #pragma once
 
-#include <data/cassandra/impl/ManagedObject.h>
-#include <data/cassandra/impl/Tuple.h>
-#include <util/Expected.h>
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Tuple.hpp"
 
+#include <cassandra.h>
 #include <ripple/basics/base_uint.h>
 #include <ripple/protocol/AccountID.h>
-#include <cassandra.h>
 
-#include <compare>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
 #include <iterator>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <string_view>
 #include <tuple>
+#include <type_traits>
+#include <utility>
+#include <vector>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/RetryPolicy.hpp

@@ -19,10 +19,10 @@
 
 #pragma once
 
-#include <data/cassandra/Handle.h>
-#include <data/cassandra/Types.h>
-#include <util/Expected.h>
-#include <util/log/Logger.h>
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "util/Expected.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
 

src/data/cassandra/impl/Session.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include <data/cassandra/impl/ManagedObject.h>
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/SslContext.cpp

@@ -17,7 +17,14 @@
 */
 //==============================================================================
 
-#include <data/cassandra/impl/SslContext.h>
+#include "data/cassandra/impl/SslContext.hpp"
+
+#include "data/cassandra/impl/ManagedObject.hpp"
+
+#include <cassandra.h>
+
+#include <stdexcept>
+#include <string>
 
 namespace {
 constexpr auto contextDeleter = [](CassSsl* ptr) { cass_ssl_free(ptr); };

src/data/cassandra/impl/SslContext.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include <data/cassandra/impl/ManagedObject.h>
+#include "data/cassandra/impl/ManagedObject.hpp"
 
 #include <cassandra.h>
 

src/data/cassandra/impl/Statement.hpp

@@ -19,20 +19,25 @@
 
 #pragma once
 
-#include <data/cassandra/Types.h>
-#include <data/cassandra/impl/Collection.h>
-#include <data/cassandra/impl/ManagedObject.h>
-#include <data/cassandra/impl/Tuple.h>
-#include <util/Expected.h>
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/Collection.hpp"
+#include "data/cassandra/impl/ManagedObject.hpp"
+#include "data/cassandra/impl/Tuple.hpp"
 
-#include <ripple/basics/base_uint.h>
-#include <ripple/protocol/STAccount.h>
 #include <cassandra.h>
 #include <fmt/core.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/STAccount.h>
 
-#include <chrono>
-#include <compare>
-#include <iterator>
+#include <cstddef>
+#include <cstdint>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <type_traits>
+#include <vector>
 
 namespace data::cassandra::detail {
 

src/data/cassandra/impl/Tuple.cpp

@@ -17,7 +17,11 @@
 */
 //==============================================================================
 
-#include <data/cassandra/impl/Tuple.h>
+#include "data/cassandra/impl/Tuple.hpp"
+
+#include "data/cassandra/impl/ManagedObject.hpp"
+
+#include <cassandra.h>
 
 namespace {
 constexpr auto tupleDeleter = [](CassTuple* ptr) { cass_tuple_free(ptr); };

src/data/cassandra/impl/Tuple.hpp

@@ -19,15 +19,19 @@
 
 #pragma once
 
-#include <data/cassandra/impl/ManagedObject.h>
+#include "data/cassandra/impl/ManagedObject.hpp"
 
-#include <ripple/basics/base_uint.h>
 #include <cassandra.h>
+#include <ripple/basics/base_uint.h>
 
+#include <cstddef>
+#include <cstdint>
 #include <functional>
+#include <stdexcept>
 #include <string>
 #include <string_view>
 #include <tuple>
+#include <type_traits>
 
 namespace data::cassandra::detail {
 
@@ -75,7 +79,7 @@ public:
         }
         // clio only uses bigint (int64_t) so we convert any incoming type
         else if constexpr (std::is_convertible_v<DecayedType, int64_t>) {
-            auto const rc = cass_tuple_set_int64(*this, idx, value);
+            auto const rc = cass_tuple_set_int64(*this, idx, std::forward<Type>(value));
             throwErrorIfNeeded(rc, "Bind int64");
         } else if constexpr (std::is_same_v<DecayedType, ripple::uint256>) {
             auto const rc = cass_tuple_set_bytes(

src/etl/ETLHelpers.hpp

@@ -20,12 +20,19 @@
 /** @file */
 #pragma once
 
+#include "util/Assert.hpp"
+
 #include <ripple/basics/base_uint.h>
+
+#include <chrono>
 #include <condition_variable>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
 #include <mutex>
 #include <optional>
 #include <queue>
-#include <sstream>
+#include <vector>
 
 namespace etl {
 /**
@@ -209,7 +216,7 @@ public:
 inline std::vector<ripple::uint256>
 getMarkers(size_t numMarkers)
 {
-    assert(numMarkers <= 256);
+    ASSERT(numMarkers <= 256, "Number of markers must be <= 256. Got: {}", numMarkers);
 
     unsigned char const incr = 256 / numMarkers;
 
@@ -222,4 +229,4 @@ getMarkers(size_t numMarkers)
     }
     return markers;
 }
-}  // namespace etl
+}  // namespace etl

src/etl/ETLService.cpp

@@ -17,12 +17,27 @@
 */
 //==============================================================================
 
-#include <etl/ETLService.h>
-#include <util/Constants.h>
+#include "etl/ETLService.hpp"
 
+#include "data/BackendInterface.hpp"
+#include "util/Assert.hpp"
+#include "util/Constants.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
+
+#include <boost/asio/io_context.hpp>
+#include <ripple/beast/core/CurrentThreadName.h>
 #include <ripple/protocol/LedgerHeader.h>
 
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <stdexcept>
+#include <thread>
 #include <utility>
+#include <vector>
 
 namespace etl {
 // Database must be populated when this starts
@@ -32,14 +47,21 @@ ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
     if (finishSequence_ && startSequence > *finishSequence_)
         return {};
 
+    LOG(log_.debug()) << "Wait for cache containing seq " << startSequence - 1
+                      << " current cache last seq =" << backend_->cache().latestLedgerSequence();
+    backend_->cache().waitUntilCacheContainsSeq(startSequence - 1);
+
     LOG(log_.debug()) << "Starting etl pipeline";
     state_.isWriting = true;
 
-    auto rng = backend_->hardFetchLedgerRangeNoThrow();
-    if (!rng || rng->maxSequence < startSequence - 1) {
-        assert(false);
-        throw std::runtime_error("runETLPipeline: parent ledger is null");
-    }
+    auto const rng = backend_->hardFetchLedgerRangeNoThrow();
+    ASSERT(rng.has_value(), "Parent ledger range can't be null");
+    ASSERT(
+        rng->maxSequence >= startSequence - 1,
+        "Got not parent ledger. rnd->maxSequence = {}, startSequence = {}",
+        rng->maxSequence,
+        startSequence
+    );
 
     auto const begin = std::chrono::system_clock::now();
     auto extractors = std::vector<std::unique_ptr<ExtractorType>>{};
@@ -125,7 +147,7 @@ ETLService::monitor()
         cacheLoader_.load(rng->maxSequence);
     }
 
-    assert(rng);
+    ASSERT(rng.has_value(), "Ledger range can't be null");
     uint32_t nextSequence = rng->maxSequence + 1;
 
     LOG(log_.debug()) << "Database is populated. "

src/etl/ETLService.hpp

@@ -19,27 +19,37 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <data/LedgerCache.h>
-#include <etl/LoadBalancer.h>
-#include <etl/Source.h>
-#include <etl/SystemState.h>
-#include <etl/impl/AmendmentBlock.h>
-#include <etl/impl/CacheLoader.h>
-#include <etl/impl/ExtractionDataPipe.h>
-#include <etl/impl/Extractor.h>
-#include <etl/impl/LedgerFetcher.h>
-#include <etl/impl/LedgerLoader.h>
-#include <etl/impl/LedgerPublisher.h>
-#include <etl/impl/Transformer.h>
-#include <feed/SubscriptionManager.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "data/LedgerCache.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/ETLState.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/Source.hpp"
+#include "etl/SystemState.hpp"
+#include "etl/impl/AmendmentBlock.hpp"
+#include "etl/impl/CacheLoader.hpp"
+#include "etl/impl/ExtractionDataPipe.hpp"
+#include "etl/impl/Extractor.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "etl/impl/LedgerLoader.hpp"
+#include "etl/impl/LedgerPublisher.hpp"
+#include "etl/impl/Transformer.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "util/log/Logger.hpp"
 
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/steady_timer.hpp>
+#include <boost/json/object.hpp>
 #include <grpcpp/grpcpp.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
+#include <cstddef>
+#include <cstdint>
 #include <memory>
+#include <optional>
+#include <string>
+#include <thread>
 
 struct AccountTransactionsData;
 struct NFTTransactionsData;

src/etl/ETLState.cpp

@@ -17,8 +17,16 @@
 */
 //==============================================================================
 
-#include <etl/ETLState.h>
-#include <rpc/JS.h>
+#include "etl/ETLState.hpp"
+
+#include "rpc/JS.hpp"
+
+#include <boost/json/conversion.hpp>
+#include <boost/json/value.hpp>
+#include <boost/json/value_to.hpp>
+#include <ripple/protocol/jss.h>
+
+#include <cstdint>
 
 namespace etl {
 

src/etl/ETLState.hpp

@@ -19,9 +19,12 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
+#include "data/BackendInterface.hpp"
 
 #include <boost/json.hpp>
+#include <boost/json/conversion.hpp>
+#include <boost/json/value.hpp>
+#include <boost/json/value_to.hpp>
 
 #include <cstdint>
 #include <optional>

src/etl/LoadBalancer.cpp

@@ -17,25 +17,36 @@
 */
 //==============================================================================
 
-#include <data/DBHelpers.h>
-#include <etl/ETLService.h>
-#include <etl/NFTHelpers.h>
-#include <etl/ProbingSource.h>
-#include <etl/Source.h>
-#include <rpc/RPCHelpers.h>
-#include <util/Profiler.h>
-#include <util/Random.h>
-#include <util/log/Logger.h>
+#include "etl/LoadBalancer.hpp"
 
-#include <ripple/beast/net/IPEndpoint.h>
-#include <ripple/protocol/STLedgerEntry.h>
-#include <boost/asio/strand.hpp>
-#include <boost/beast/http.hpp>
-#include <boost/beast/ssl.hpp>
-#include <boost/json.hpp>
-#include <boost/json/src.hpp>
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/ETLService.hpp"
+#include "etl/ETLState.hpp"
+#include "etl/ProbingSource.hpp"
+#include "etl/Source.hpp"
+#include "util/Assert.hpp"
+#include "util/Random.hpp"
+#include "util/log/Logger.hpp"
 
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/json/array.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/value.hpp>
+#include <fmt/core.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <stdexcept>
+#include <string>
 #include <thread>
+#include <utility>
+#include <vector>
 
 using namespace util;
 
@@ -206,7 +217,7 @@ bool
 LoadBalancer::shouldPropagateTxnStream(Source* in) const
 {
     for (auto& src : sources_) {
-        assert(src);
+        ASSERT(src != nullptr, "Source is nullptr");
 
         // We pick the first Source encountered that is connected
         if (src->isConnected())

src/etl/LoadBalancer.hpp

@@ -19,16 +19,29 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <etl/ETLHelpers.h>
-#include <etl/ETLState.h>
-#include <feed/SubscriptionManager.h>
-#include <util/config/Config.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/ETLState.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 #include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/value.hpp>
 #include <grpcpp/grpcpp.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <org/xrpl/rpc/v1/ledger.pb.h>
+#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
 
 namespace etl {
 class Source;
@@ -99,26 +112,6 @@ public:
         std::shared_ptr<NetworkValidatedLedgers> validatedLedgers
     );
 
-    /**
-     * @brief A factory function for the ETL source.
-     *
-     * @param config The configuration to use
-     * @param ioc The io_context to run on
-     * @param backend BackendInterface implementation
-     * @param subscriptions Subscription manager
-     * @param validatedLedgers The network validated ledgers datastructure
-     * @param balancer The load balancer
-     */
-    static std::unique_ptr<Source>
-    make_Source(
-        util::Config const& config,
-        boost::asio::io_context& ioc,
-        std::shared_ptr<BackendInterface> backend,
-        std::shared_ptr<feed::SubscriptionManager> subscriptions,
-        std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
-        LoadBalancer& balancer
-    );
-
     ~LoadBalancer();
 
     /**
@@ -187,6 +180,26 @@ public:
     getETLState() noexcept;
 
 private:
+    /**
+     * @brief A factory function for the ETL source.
+     *
+     * @param config The configuration to use
+     * @param ioc The io_context to run on
+     * @param backend BackendInterface implementation
+     * @param subscriptions Subscription manager
+     * @param validatedLedgers The network validated ledgers datastructure
+     * @param balancer The load balancer
+     */
+    static std::unique_ptr<Source>
+    make_Source(
+        util::Config const& config,
+        boost::asio::io_context& ioc,
+        std::shared_ptr<BackendInterface> backend,
+        std::shared_ptr<feed::SubscriptionManager> subscriptions,
+        std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
+        LoadBalancer& balancer
+    );
+
     /**
      * @brief Execute a function on a randomly selected source.
      *

src/etl/NFTHelpers.cpp

@@ -17,15 +17,33 @@
 */
 //==============================================================================
 
-#include <ripple/protocol/STBase.h>
-#include <ripple/protocol/STTx.h>
-#include <ripple/protocol/TxMeta.h>
-#include <vector>
+#include "data/DBHelpers.hpp"
 
-#include <data/BackendInterface.h>
-#include <data/DBHelpers.h>
-#include <data/Types.h>
 #include <fmt/core.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/LedgerFormats.h>
+#include <ripple/protocol/SField.h>
+#include <ripple/protocol/STArray.h>
+#include <ripple/protocol/STBase.h>
+#include <ripple/protocol/STLedgerEntry.h>
+#include <ripple/protocol/STObject.h>
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/Serializer.h>
+#include <ripple/protocol/TER.h>
+#include <ripple/protocol/TxFormats.h>
+#include <ripple/protocol/TxMeta.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <iterator>
+#include <optional>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
 
 namespace etl {
 
@@ -114,7 +132,8 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
 
     return {
         {NFTTransactionsData(*diff.first, txMeta, sttx.getTransactionID())},
-        NFTsData(*diff.first, *owner, sttx.getFieldVL(ripple::sfURI), txMeta)};
+        NFTsData(*diff.first, *owner, sttx.getFieldVL(ripple::sfURI), txMeta)
+    };
 }
 
 std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
@@ -198,7 +217,8 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
                                             .downcast<ripple::STObject>()
                                             .getAccountID(ripple::sfOwner);
         return {
-            {NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())}, NFTsData(tokenID, owner, txMeta, false)};
+            {NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())}, NFTsData(tokenID, owner, txMeta, false)
+        };
     }
 
     // Otherwise we have to infer the new owner from the affected nodes.
@@ -247,7 +267,8 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
         if (nft != nfts.end()) {
             return {
                 {NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())},
-                NFTsData(tokenID, nodeOwner, txMeta, false)};
+                NFTsData(tokenID, nodeOwner, txMeta, false)
+            };
         }
     }
 
@@ -337,4 +358,4 @@ getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string c
 
     return nfts;
 }
-}  // namespace etl
+}  // namespace etl

src/etl/NFTHelpers.hpp

@@ -20,11 +20,17 @@
 /** @file */
 #pragma once
 
-#include <data/DBHelpers.h>
+#include "data/DBHelpers.hpp"
 
 #include <ripple/protocol/STTx.h>
 #include <ripple/protocol/TxMeta.h>
 
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
 namespace etl {
 
 /**
@@ -48,4 +54,4 @@ getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx);
 std::vector<NFTsData>
 getNFTDataFromObj(std::uint32_t seq, std::string const& key, std::string const& blob);
 
-}  // namespace etl
+}  // namespace etl

src/etl/ProbingSource.cpp

@@ -17,7 +17,32 @@
 */
 //==============================================================================
 
-#include <etl/ProbingSource.h>
+#include "etl/ProbingSource.hpp"
+
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/Source.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
+
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/ssl/context.hpp>
+#include <boost/json/object.hpp>
+#include <boost/uuid/nil_generator.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <grpcpp/support/status.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
 
 namespace etl {
 
@@ -173,7 +198,8 @@ ProbingSource::make_SSLHooks() noexcept
                     plainSrc_->resume();
                 }
                 return SourceHooks::Action::STOP;
-            }};
+            }
+    };
 }
 
 SourceHooks
@@ -200,6 +226,7 @@ ProbingSource::make_PlainHooks() noexcept
                     sslSrc_->resume();
                 }
                 return SourceHooks::Action::STOP;
-            }};
+            }
+    };
 };
 }  // namespace etl

src/etl/ProbingSource.hpp

@@ -19,17 +19,33 @@
 
 #pragma once
 
-#include <etl/Source.h>
-#include <util/config/Config.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/Source.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/ssl/context.hpp>
 #include <boost/beast/core.hpp>
 #include <boost/beast/core/string.hpp>
 #include <boost/beast/ssl.hpp>
 #include <boost/beast/websocket.hpp>
+#include <boost/json/object.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
 
+#include <cstdint>
+#include <memory>
 #include <mutex>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
 
 namespace etl {
 

src/etl/Source.cpp

@@ -17,22 +17,22 @@
 */
 //==============================================================================
 
-#include <data/DBHelpers.h>
-#include <etl/ETLService.h>
-#include <etl/LoadBalancer.h>
-#include <etl/ProbingSource.h>
-#include <etl/Source.h>
-#include <rpc/RPCHelpers.h>
-#include <util/Profiler.h>
+#include "etl/Source.hpp"
 
-#include <ripple/beast/net/IPEndpoint.h>
-#include <ripple/protocol/STLedgerEntry.h>
-#include <boost/asio/strand.hpp>
-#include <boost/beast/http.hpp>
-#include <boost/beast/ssl.hpp>
-#include <boost/json.hpp>
+#include "util/log/Logger.hpp"
 
-#include <thread>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/post.hpp>
+#include <boost/asio/ssl/stream_base.hpp>
+#include <boost/beast/core/error.hpp>
+#include <boost/beast/core/role.hpp>
+#include <boost/beast/core/stream_traits.hpp>
+#include <boost/beast/http/field.hpp>
+#include <boost/beast/websocket/rfc6455.hpp>
+#include <boost/beast/websocket/stream_base.hpp>
+
+#include <memory>
+#include <string>
 
 namespace etl {
 

src/etl/Source.hpp

@@ -19,26 +19,77 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <etl/ETLHelpers.h>
-#include <etl/LoadBalancer.h>
-#include <etl/impl/AsyncData.h>
-#include <etl/impl/ForwardCache.h>
-#include <feed/SubscriptionManager.h>
-#include <util/config/Config.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "etl/ETLHelpers.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/impl/AsyncData.hpp"
+#include "etl/impl/ForwardCache.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "util/Assert.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 #include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string/split.hpp>
 #include <boost/asio.hpp>
+#include <boost/asio/associated_executor.hpp>
+#include <boost/asio/buffer.hpp>
+#include <boost/asio/error.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/address.hpp>
+#include <boost/asio/ip/tcp.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/ssl/context.hpp>
+#include <boost/asio/ssl/error.hpp>
+#include <boost/asio/steady_timer.hpp>
+#include <boost/asio/strand.hpp>
 #include <boost/beast/core.hpp>
+#include <boost/beast/core/buffers_to_string.hpp>
+#include <boost/beast/core/error.hpp>
+#include <boost/beast/core/flat_buffer.hpp>
 #include <boost/beast/core/string.hpp>
+#include <boost/beast/core/tcp_stream.hpp>
+#include <boost/beast/http/field.hpp>
 #include <boost/beast/ssl.hpp>
+#include <boost/beast/ssl/ssl_stream.hpp>
+#include <boost/beast/version.hpp>
 #include <boost/beast/websocket.hpp>
+#include <boost/beast/websocket/rfc6455.hpp>
+#include <boost/beast/websocket/stream.hpp>
+#include <boost/beast/websocket/stream_base.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/parse.hpp>
+#include <boost/json/serialize.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/uuid/random_generator.hpp>
 #include <boost/uuid/uuid.hpp>
 #include <boost/uuid/uuid_generators.hpp>
+#include <grpcpp/client_context.h>
 #include <grpcpp/grpcpp.h>
+#include <grpcpp/security/credentials.h>
+#include <grpcpp/support/channel_arguments.h>
+#include <grpcpp/support/status.h>
+#include <openssl/err.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <sstream>
+#include <string>
 #include <utility>
+#include <vector>
 
 namespace feed {
 class SubscriptionManager;
@@ -312,7 +363,7 @@ public:
         boost::json::object response;
 
         namespace beast = boost::beast;
-        namespace http = boost::beast::http;
+        namespace http = beast::http;
         namespace websocket = beast::websocket;
         namespace net = boost::asio;
         using tcp = boost::asio::ip::tcp;
@@ -485,7 +536,7 @@ public:
         std::vector<std::string> edgeKeys;
 
         while (numFinished < calls.size() && cq.Next(&tag, &ok)) {
-            assert(tag);
+            ASSERT(tag != nullptr, "Tag can't be null.");
             auto ptr = static_cast<etl::detail::AsyncCallData*>(tag);
 
             if (!ok) {
@@ -794,7 +845,7 @@ private:
                 uint32_t const sequence = std::stoll(minAndMax[0]);
                 pairs.emplace_back(sequence, sequence);
             } else {
-                assert(minAndMax.size() == 2);
+                ASSERT(minAndMax.size() == 2, "Min and max should be of size 2. Got size = {}", minAndMax.size());
                 uint32_t const min = std::stoll(minAndMax[0]);
                 uint32_t const max = std::stoll(minAndMax[1]);
                 pairs.emplace_back(min, max);

src/etl/impl/AmendmentBlock.hpp

@@ -19,10 +19,11 @@
 
 #pragma once
 
-#include <etl/SystemState.h>
-#include <util/log/Logger.h>
+#include "etl/SystemState.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
+#include <boost/asio/post.hpp>
 #include <boost/asio/steady_timer.hpp>
 
 #include <chrono>

src/etl/impl/AsyncData.hpp

@@ -19,12 +19,26 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <etl/NFTHelpers.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "data/Types.hpp"
+#include "etl/NFTHelpers.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
 
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <grpcpp/client_context.h>
 #include <grpcpp/grpcpp.h>
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger_data.pb.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
+#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
 
 namespace etl::detail {
 
@@ -60,7 +74,12 @@ public:
                           << " . prefix = " << ripple::strHex(std::string(1, prefix))
                           << " . nextPrefix_ = " << ripple::strHex(std::string(1, nextPrefix_));
 
-        assert(nextPrefix_ > prefix || nextPrefix_ == 0x00);
+        ASSERT(
+            nextPrefix_ > prefix || nextPrefix_ == 0x00,
+            "Next prefix must be greater than current prefix. Got: nextPrefix_ = {}, prefix = {}",
+            nextPrefix_,
+            prefix
+        );
 
         cur_ = std::make_unique<org::xrpl::rpc::v1::GetLedgerDataResponse>();
         next_ = std::make_unique<org::xrpl::rpc::v1::GetLedgerDataResponse>();

src/etl/impl/CacheLoader.hpp

@@ -19,20 +19,48 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "data/Types.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
 
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+#include <boost/algorithm/hex.hpp>
 #include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/beast/core.hpp>
+#include <boost/beast/core/buffers_to_string.hpp>
+#include <boost/beast/core/error.hpp>
+#include <boost/beast/core/flat_buffer.hpp>
 #include <boost/beast/core/string.hpp>
+#include <boost/beast/core/tcp_stream.hpp>
 #include <boost/beast/websocket.hpp>
+#include <boost/beast/websocket/stream.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/parse.hpp>
+#include <boost/json/value.hpp>
 #include <grpcpp/grpcpp.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
+#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
+#include <atomic>
 #include <chrono>
-#include <mutex>
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <functional>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <random>
+#include <sstream>
+#include <string>
 #include <thread>
+#include <utility>
+#include <vector>
 
 namespace etl::detail {
 
@@ -134,10 +162,7 @@ public:
             return;
         }
 
-        if (cache_.get().isFull()) {
-            assert(false);
-            return;
-        }
+        ASSERT(!cache_.get().isFull(), "Cache must not be full. seq = {}", seq);
 
         if (!clioPeers_.empty()) {
             boost::asio::spawn(ioContext_.get(), [this, seq](boost::asio::yield_context yield) {
@@ -220,7 +245,8 @@ private:
                     {"ledger_index", ledgerIndex},
                     {"binary", true},
                     {"out_of_order", true},
-                    {"limit", LIMIT}};
+                    {"limit", LIMIT}
+                };
 
                 if (marker)
                     request["marker"] = *marker;

src/etl/impl/ExtractionDataPipe.hpp

@@ -19,10 +19,13 @@
 
 #pragma once
 
-#include <etl/ETLHelpers.h>
-#include <util/log/Logger.h>
+#include "etl/ETLHelpers.hpp"
+#include "util/log/Logger.hpp"
 
+#include <cstddef>
+#include <cstdint>
 #include <memory>
+#include <optional>
 #include <vector>
 
 namespace etl::detail {

src/etl/impl/Extractor.hpp

@@ -19,14 +19,18 @@
 
 #pragma once
 
-#include <etl/SystemState.h>
-#include <util/Profiler.h>
-#include <util/log/Logger.h>
+#include "etl/SystemState.hpp"
+#include "util/Assert.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
 
 #include <ripple/beast/core/CurrentThreadName.h>
 
 #include <chrono>
-#include <mutex>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <optional>
 #include <thread>
 #include <utility>
 
@@ -76,7 +80,7 @@ public:
     void
     waitTillFinished()
     {
-        assert(thread_.joinable());
+        ASSERT(thread_.joinable(), "Extractor thread must be joinable");
         thread_.join();
     }
 

src/etl/impl/ForwardCache.cpp

@@ -17,12 +17,21 @@
 */
 //==============================================================================
 
-#include <etl/Source.h>
-#include <etl/impl/ForwardCache.h>
-#include <rpc/RPCHelpers.h>
+#include "etl/impl/ForwardCache.hpp"
+
+#include "etl/Source.hpp"
+#include "rpc/RPCHelpers.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio/spawn.hpp>
-#include <boost/json.hpp>
+#include <boost/json/object.hpp>
+
+#include <atomic>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <string>
 
 namespace etl::detail {
 

src/etl/impl/ForwardCache.hpp

@@ -19,16 +19,19 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <etl/ETLHelpers.h>
-#include <util/config/Config.h>
-#include <util/log/Logger.h>
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
 
 #include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/strand.hpp>
 #include <boost/json.hpp>
+#include <boost/json/object.hpp>
 
-#include <atomic>
-#include <mutex>
+#include <cstdint>
+#include <optional>
+#include <shared_mutex>
+#include <string>
 #include <unordered_map>
 
 namespace etl {

src/etl/impl/LedgerFetcher.hpp

@@ -19,14 +19,14 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <etl/Source.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "util/log/Logger.hpp"
 
-#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 #include <grpcpp/grpcpp.h>
+#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
-#include <optional>
+#include <cstdint>
+#include <memory>
 #include <utility>
 
 namespace etl::detail {
@@ -89,9 +89,15 @@ public:
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
 
-        auto response = loadBalancer_->fetchLedger(
-            sequence, true, !backend_->cache().isFull() || backend_->cache().latestLedgerSequence() >= sequence
-        );
+        auto const isCacheFull = backend_->cache().isFull();
+        auto const isLedgerCached = backend_->cache().latestLedgerSequence() >= sequence;
+        if (isLedgerCached) {
+            LOG(log_.info()) << sequence << " is already cached, the current latest seq in cache is "
+                             << backend_->cache().latestLedgerSequence() << " and the cache is "
+                             << (isCacheFull ? "full" : "not full");
+        }
+
+        auto response = loadBalancer_->fetchLedger(sequence, true, !isCacheFull || isLedgerCached);
         if (response)
             LOG(log_.trace()) << "GetLedger reply = " << response->DebugString();
 

src/etl/impl/LedgerLoader.hpp

@@ -19,18 +19,34 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <etl/NFTHelpers.h>
-#include <etl/SystemState.h>
-#include <etl/impl/LedgerFetcher.h>
-#include <util/LedgerUtils.h>
-#include <util/Profiler.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "etl/NFTHelpers.hpp"
+#include "etl/SystemState.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "util/Assert.hpp"
+#include "util/LedgerUtils.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
 
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
 #include <ripple/beast/core/CurrentThreadName.h>
+#include <ripple/protocol/LedgerHeader.h>
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/Serializer.h>
+#include <ripple/protocol/TxMeta.h>
 
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
 #include <memory>
+#include <optional>
+#include <string>
 #include <utility>
+#include <vector>
 
 /**
  * @brief Account transactions, NFT transactions and NFT data bundled togeher.
@@ -152,8 +168,7 @@ public:
         // check that database is actually empty
         auto rng = backend_->hardFetchLedgerRangeNoThrow();
         if (rng) {
-            LOG(log_.fatal()) << "Database is not empty";
-            assert(false);
+            ASSERT(false, "Database is not empty");
             return {};
         }
 
@@ -189,50 +204,48 @@ public:
                 size_t numWrites = 0;
                 backend_->cache().setFull();
 
-                auto seconds =
-                    ::util::timed<std::chrono::seconds>([this, edgeKeys = &edgeKeys, sequence, &numWrites]() {
-                        for (auto& key : *edgeKeys) {
-                            LOG(log_.debug()) << "Writing edge key = " << ripple::strHex(key);
-                            auto succ =
-                                backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
-                            if (succ)
-                                backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
-                        }
+                auto seconds = ::util::timed<std::chrono::seconds>([this, edgeKeys = &edgeKeys, sequence, &numWrites] {
+                    for (auto& key : *edgeKeys) {
+                        LOG(log_.debug()) << "Writing edge key = " << ripple::strHex(key);
+                        auto succ = backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
+                        if (succ)
+                            backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
+                    }
 
-                        ripple::uint256 prev = data::firstKey;
-                        while (auto cur = backend_->cache().getSuccessor(prev, sequence)) {
-                            assert(cur);
-                            if (prev == data::firstKey)
-                                backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
+                    ripple::uint256 prev = data::firstKey;
+                    while (auto cur = backend_->cache().getSuccessor(prev, sequence)) {
+                        ASSERT(cur.has_value(), "Succesor for key {} must exist", ripple::strHex(prev));
+                        if (prev == data::firstKey)
+                            backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
 
-                            if (isBookDir(cur->key, cur->blob)) {
-                                auto base = getBookBase(cur->key);
-                                // make sure the base is not an actual object
-                                if (!backend_->cache().get(cur->key, sequence)) {
-                                    auto succ = backend_->cache().getSuccessor(base, sequence);
-                                    assert(succ);
-                                    if (succ->key == cur->key) {
-                                        LOG(log_.debug()) << "Writing book successor = " << ripple::strHex(base)
-                                                          << " - " << ripple::strHex(cur->key);
+                        if (isBookDir(cur->key, cur->blob)) {
+                            auto base = getBookBase(cur->key);
+                            // make sure the base is not an actual object
+                            if (!backend_->cache().get(base, sequence)) {
+                                auto succ = backend_->cache().getSuccessor(base, sequence);
+                                ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
+                                if (succ->key == cur->key) {
+                                    LOG(log_.debug()) << "Writing book successor = " << ripple::strHex(base) << " - "
+                                                      << ripple::strHex(cur->key);
 
-                                        backend_->writeSuccessor(
-                                            uint256ToString(base), sequence, uint256ToString(cur->key)
-                                        );
-                                    }
+                                    backend_->writeSuccessor(
+                                        uint256ToString(base), sequence, uint256ToString(cur->key)
+                                    );
                                 }
-
-                                ++numWrites;
                             }
 
-                            prev = cur->key;
-                            static constexpr std::size_t LOG_INTERVAL = 100000;
-                            if (numWrites % LOG_INTERVAL == 0 && numWrites != 0)
-                                LOG(log_.info()) << "Wrote " << numWrites << " book successors";
+                            ++numWrites;
                         }
 
-                        backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(data::lastKey));
-                        ++numWrites;
-                    });
+                        prev = cur->key;
+                        static constexpr std::size_t LOG_INTERVAL = 100000;
+                        if (numWrites % LOG_INTERVAL == 0 && numWrites != 0)
+                            LOG(log_.info()) << "Wrote " << numWrites << " book successors";
+                    }
+
+                    backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(data::lastKey));
+                    ++numWrites;
+                });
 
                 LOG(log_.info()) << "Looping through cache and submitting all writes took " << seconds
                                  << " seconds. numWrites = " << std::to_string(numWrites);
@@ -242,8 +255,8 @@ public:
 
             if (not state_.get().isStopping) {
                 backend_->writeAccountTransactions(std::move(insertTxResult.accountTxData));
-                backend_->writeNFTs(std::move(insertTxResult.nfTokensData));
-                backend_->writeNFTTransactions(std::move(insertTxResult.nfTokenTxData));
+                backend_->writeNFTs(insertTxResult.nfTokensData);
+                backend_->writeNFTTransactions(insertTxResult.nfTokenTxData);
             }
 
             backend_->finishWrites(sequence);

src/etl/impl/LedgerPublisher.hpp

@@ -19,16 +19,34 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <etl/SystemState.h>
-#include <feed/SubscriptionManager.h>
-#include <util/LedgerUtils.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "etl/SystemState.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
 
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/strand.hpp>
+#include <ripple/basics/chrono.h>
+#include <ripple/protocol/Fees.h>
 #include <ripple/protocol/LedgerHeader.h>
+#include <ripple/protocol/SField.h>
+#include <ripple/protocol/STObject.h>
+#include <ripple/protocol/Serializer.h>
 
 #include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <string>
+#include <thread>
 #include <utility>
+#include <vector>
 
 namespace etl::detail {
 
@@ -115,7 +133,7 @@ public:
                 return backend_->fetchLedgerBySequence(ledgerSequence, yield);
             });
 
-            assert(lgr);
+            ASSERT(lgr.has_value(), "Ledger must exist in database. Ledger sequence = {}", ledgerSequence);
             publish(*lgr);
 
             return true;
@@ -157,7 +175,7 @@ public:
                 std::optional<ripple::Fees> fees = data::synchronousAndRetryOnTimeout([&](auto yield) {
                     return backend_->fetchFees(lgrInfo.seq, yield);
                 });
-                assert(fees);
+                ASSERT(fees.has_value(), "Fees must exist for ledger {}", lgrInfo.seq);
 
                 std::vector<data::TransactionAndMetadata> transactions =
                     data::synchronousAndRetryOnTimeout([&](auto yield) {
@@ -165,7 +183,7 @@ public:
                     });
 
                 auto const ledgerRange = backend_->fetchLedgerRange();
-                assert(ledgerRange);
+                ASSERT(ledgerRange.has_value(), "Ledger range must exist");
 
                 std::string const range =
                     std::to_string(ledgerRange->minSequence) + "-" + std::to_string(ledgerRange->maxSequence);

src/etl/impl/Transformer.hpp

@@ -19,22 +19,35 @@
 
 #pragma once
 
-#include <data/BackendInterface.h>
-#include <etl/SystemState.h>
-#include <etl/impl/AmendmentBlock.h>
-#include <etl/impl/LedgerLoader.h>
-#include <util/LedgerUtils.h>
-#include <util/Profiler.h>
-#include <util/log/Logger.h>
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "etl/SystemState.hpp"
+#include "etl/impl/AmendmentBlock.hpp"
+#include "etl/impl/LedgerLoader.hpp"
+#include "util/Assert.hpp"
+#include "util/LedgerUtils.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
 
+#include <grpcpp/grpcpp.h>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
 #include <ripple/beast/core/CurrentThreadName.h>
 #include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
-#include <grpcpp/grpcpp.h>
+#include <ripple/protocol/LedgerHeader.h>
 
 #include <chrono>
+#include <cstdint>
+#include <functional>
 #include <memory>
+#include <optional>
+#include <set>
+#include <stdexcept>
+#include <string>
 #include <thread>
 #include <utility>
+#include <vector>
 
 namespace etl::detail {
 
@@ -113,7 +126,7 @@ public:
     void
     waitTillFinished()
     {
-        assert(thread_.joinable());
+        ASSERT(thread_.joinable(), "Transformer thread must be joinable");
         thread_.join();
     }
 
@@ -198,8 +211,8 @@ private:
                           << rawData.transactions_list().transactions_size();
 
         backend_->writeAccountTransactions(std::move(insertTxResultOp->accountTxData));
-        backend_->writeNFTs(std::move(insertTxResultOp->nfTokensData));
-        backend_->writeNFTTransactions(std::move(insertTxResultOp->nfTokenTxData));
+        backend_->writeNFTs(insertTxResultOp->nfTokensData);
+        backend_->writeNFTTransactions(insertTxResultOp->nfTokenTxData);
 
         auto [success, duration] =
             ::util::timed<std::chrono::duration<double>>([&]() { return backend_->finishWrites(lgrInfo.seq); });
@@ -228,7 +241,7 @@ private:
 
         for (auto& obj : *(rawData.mutable_ledger_objects()->mutable_objects())) {
             auto key = ripple::uint256::fromVoidChecked(obj.key());
-            assert(key);
+            ASSERT(key.has_value(), "Failed to deserialize key from void");
 
             cacheUpdates.push_back({*key, {obj.mutable_data()->begin(), obj.mutable_data()->end()}});
             LOG(log_.debug()) << "key = " << ripple::strHex(*key) << " - mod type = " << obj.mod_type();
@@ -245,7 +258,7 @@ private:
 
                 if (isDeleted) {
                     auto const old = backend_->cache().get(*key, lgrInfo.seq - 1);
-                    assert(old);
+                    ASSERT(old.has_value(), "Deleted object must be in cache");
                     checkBookBase = isBookDir(*key, *old);
                 } else {
                     checkBookBase = isBookDir(*key, *blob);
@@ -256,7 +269,11 @@ private:
 
                     auto const bookBase = getBookBase(*key);
                     auto const oldFirstDir = backend_->cache().getSuccessor(bookBase, lgrInfo.seq - 1);
-                    assert(oldFirstDir);
+                    ASSERT(
+                        oldFirstDir.has_value(),
+                        "Book base must have a successor for lgrInfo.seq - 1 = {}",
+                        lgrInfo.seq - 1
+                    );
 
                     // We deleted the first directory, or we added a directory prior to the old first
                     // directory