chore: Commits for 2.4.0 rc1 (#1959)

Commits from `develop` to include into `2.4.0` release.

.clang-tidy

@@ -4,13 +4,19 @@ Checks: '-*,
   bugprone-assert-side-effect,
   bugprone-bad-signal-to-kill-thread,
   bugprone-bool-pointer-implicit-conversion,
+  bugprone-casting-through-void,
+  bugprone-chained-comparison,
+  bugprone-compare-pointer-to-member-virtual-function,
   bugprone-copy-constructor-init,
+  bugprone-crtp-constructor-accessibility,
   bugprone-dangling-handle,
   bugprone-dynamic-static-initializers,
   bugprone-empty-catch,
   bugprone-fold-init-type,
   bugprone-forward-declaration-namespace,
   bugprone-inaccurate-erase,
+  bugprone-inc-dec-in-conditions,
+  bugprone-incorrect-enable-if,
   bugprone-incorrect-roundings,
   bugprone-infinite-loop,
   bugprone-integer-division,
@@ -21,15 +27,18 @@ Checks: '-*,
   bugprone-misplaced-pointer-arithmetic-in-alloc,
   bugprone-misplaced-widening-cast,
   bugprone-move-forwarding-reference,
+  bugprone-multi-level-implicit-pointer-conversion,
   bugprone-multiple-new-in-one-expression,
   bugprone-multiple-statement-macro,
   bugprone-no-escape,
   bugprone-non-zero-enum-to-bool-conversion,
+  bugprone-optional-value-conversion,
   bugprone-parent-virtual-call,
+  bugprone-pointer-arithmetic-on-polymorphic-object,
   bugprone-posix-return,
   bugprone-redundant-branch-condition,
   bugprone-reserved-identifier,
-  bugprone-unused-return-value,
+  bugprone-return-const-ref-from-parameter,
   bugprone-shared-ptr-array-mismatch,
   bugprone-signal-handler,
   bugprone-signed-char-misuse,
@@ -49,6 +58,7 @@ Checks: '-*,
   bugprone-suspicious-realloc-usage,
   bugprone-suspicious-semicolon,
   bugprone-suspicious-string-compare,
+  bugprone-suspicious-stringview-data-usage,
   bugprone-swapped-arguments,
   bugprone-switch-missing-default-case,
   bugprone-terminating-continue,
@@ -60,16 +70,20 @@ Checks: '-*,
   bugprone-unhandled-self-assignment,
   bugprone-unique-ptr-array-mismatch,
   bugprone-unsafe-functions,
+  bugprone-unused-local-non-trivial-variable,
   bugprone-unused-raii,
+  bugprone-unused-return-value,
   bugprone-use-after-move,
   bugprone-virtual-near-miss,
   cppcoreguidelines-init-variables,
   cppcoreguidelines-misleading-capture-default-by-value,
+  cppcoreguidelines-no-suspend-with-lock,
   cppcoreguidelines-pro-type-member-init,
   cppcoreguidelines-pro-type-static-cast-downcast,
   cppcoreguidelines-rvalue-reference-param-not-moved,
   cppcoreguidelines-use-default-member-init,
   cppcoreguidelines-virtual-class-destructor,
+  hicpp-ignored-remove-result,
   llvm-namespace-comment,
   misc-const-correctness,
   misc-definitions-in-headers,
@@ -87,10 +101,14 @@ Checks: '-*,
   modernize-make-unique,
   modernize-pass-by-value,
   modernize-type-traits,
+  modernize-use-designated-initializers,
   modernize-use-emplace,
   modernize-use-equals-default,
   modernize-use-equals-delete,
   modernize-use-override,
+  modernize-use-ranges,
+  modernize-use-starts-ends-with,
+  modernize-use-std-numbers,
   modernize-use-using,
   performance-faster-string-find,
   performance-for-range-copy,
@@ -100,6 +118,8 @@ Checks: '-*,
   performance-move-constructor-init,
   performance-no-automatic-move,
   performance-trivially-destructible,
+  readability-avoid-nested-conditional-operator,
+  readability-avoid-return-with-void-value,
   readability-braces-around-statements,
   readability-const-return-type,
   readability-container-contains,
@@ -107,26 +127,65 @@ Checks: '-*,
   readability-convert-member-functions-to-static,
   readability-duplicate-include,
   readability-else-after-return,
+  readability-enum-initial-value,
   readability-implicit-bool-conversion,
   readability-inconsistent-declaration-parameter-name,
+  readability-identifier-naming,
   readability-make-member-function-const,
+  readability-math-missing-parentheses,
   readability-misleading-indentation,
   readability-non-const-parameter,
+  readability-redundant-casting,
   readability-redundant-declaration,
+  readability-redundant-inline-specifier,
   readability-redundant-member-init,
   readability-redundant-string-init,
+  readability-reference-to-constructed-temporary,
   readability-simplify-boolean-expr,
   readability-static-accessed-through-instance,
   readability-static-definition-in-anonymous-namespace,
-  readability-suspicious-call-argument
+  readability-suspicious-call-argument,
+  readability-use-std-min-max
   '
 
 CheckOptions:
   readability-braces-around-statements.ShortStatementLines: 2
+  readability-identifier-naming.MacroDefinitionCase: UPPER_CASE
+  readability-identifier-naming.ClassCase: CamelCase
+  readability-identifier-naming.StructCase: CamelCase
+  readability-identifier-naming.UnionCase: CamelCase
+  readability-identifier-naming.EnumCase: CamelCase
+  readability-identifier-naming.EnumConstantCase: CamelCase
+  readability-identifier-naming.ScopedEnumConstantCase: CamelCase
+  readability-identifier-naming.GlobalConstantCase: UPPER_CASE
+  readability-identifier-naming.GlobalConstantPrefix: 'k'
+  readability-identifier-naming.GlobalVariableCase: CamelCase
+  readability-identifier-naming.GlobalVariablePrefix: 'g'
+  readability-identifier-naming.ConstexprFunctionCase: camelBack
+  readability-identifier-naming.ConstexprMethodCase: camelBack
+  readability-identifier-naming.ClassMethodCase: camelBack
+  readability-identifier-naming.ClassMemberCase: camelBack
+  readability-identifier-naming.ClassConstantCase: UPPER_CASE
+  readability-identifier-naming.ClassConstantPrefix: 'k'
+  readability-identifier-naming.StaticConstantCase: UPPER_CASE
+  readability-identifier-naming.StaticConstantPrefix: 'k'
+  readability-identifier-naming.StaticVariableCase: UPPER_CASE
+  readability-identifier-naming.StaticVariablePrefix: 'k'
+  readability-identifier-naming.ConstexprVariableCase: UPPER_CASE
+  readability-identifier-naming.ConstexprVariablePrefix: 'k'
+  readability-identifier-naming.LocalConstantCase: camelBack
+  readability-identifier-naming.LocalVariableCase: camelBack
+  readability-identifier-naming.TemplateParameterCase: CamelCase
+  readability-identifier-naming.ParameterCase: camelBack
+  readability-identifier-naming.FunctionCase: camelBack
+  readability-identifier-naming.MemberCase: camelBack
+  readability-identifier-naming.PrivateMemberSuffix: _
+  readability-identifier-naming.ProtectedMemberSuffix: _
+  readability-identifier-naming.PublicMemberSuffix: ''
+  readability-identifier-naming.FunctionIgnoredRegexp: '.*tag_invoke.*'
   bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true
-  bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc;::std::expected
-  misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*'
+  bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc
+  misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*;.*ranges_lower_bound\.h;time.h;stdlib.h'
 
-HeaderFilterRegex: '^.*/(src|unittests)/.*\.(h|hpp)$'
+HeaderFilterRegex: '^.*/(src|tests)/.*\.(h|hpp)$'
 WarningsAsErrors: '*'
-

.clangd

@@ -1,5 +1,13 @@
+CompileFlags:
+  Add: [-D__cpp_concepts=202002]
+
 Diagnostics:
   UnusedIncludes: Strict
   MissingIncludes: Strict
   Includes:
-    IgnoreHeader: ".*/(detail|impl)/.*"
+    IgnoreHeader:
+      - ".*/(detail|impl)/.*"
+      - ".*expected.*"
+      - ".*ranges_lower_bound.h"
+      - "time.h"
+      - "stdlib.h"

.gitattributes

@@ -0,0 +1,2 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.svg filter=lfs diff=lfs merge=lfs -text

.githooks/check-docs

@@ -13,14 +13,36 @@ TMPDIR=${ROOT}/.cache/doxygen
 TMPFILE=${TMPDIR}/docs.log
 DOCDIR=${TMPDIR}/out
 
+# Check doxygen is at all installed
 if [ -z "$DOXYGEN" ]; then
     # No hard error if doxygen is not installed yet
     cat <<EOF
 
                                    WARNING
 -----------------------------------------------------------------------------
-        'doxygen' is required to check documentation.
-        Please install it for next time. For the time being it's on CI.
+        'doxygen' is required to check documentation. 
+        Please install it for next time.
+
+        Your changes may fail to pass CI once pushed.
+-----------------------------------------------------------------------------
+
+EOF
+    exit 0
+fi
+
+# Check version of doxygen is at least 1.12
+version=$($DOXYGEN --version | grep -o '[0-9\.]*')
+
+if [[ "1.12.0" > "$version" ]]; then
+    # No hard error if doxygen version is not the one we want - let CI deal with it
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+        A minimum of version 1.12 of `which doxygen` is required.
+        Your version is $version. Please upgrade it for next time.
+
+        Your changes may fail to pass CI once pushed.
 -----------------------------------------------------------------------------
 
 EOF

.githooks/check-format

@@ -5,19 +5,33 @@
 # This script checks the format of the code and cmake files.
 # In many cases it will automatically fix the issues and abort the commit.
 
+no_formatted_directories_staged() {
+    staged_directories=$(git diff-index --cached --name-only HEAD | awk -F/ '{print $1}')
+    for sd in $staged_directories; do
+        if [[ "$sd" =~ ^(benchmark|cmake|src|tests)$ ]]; then
+            return 1
+        fi
+    done
+    return 0
+}
+
+if no_formatted_directories_staged ; then
+    exit 0
+fi
+
 echo "+ Checking code format..."
 
 # paths to check and re-format
-sources="src unittests"
+sources="src tests"
 formatter="clang-format -i"
 version=$($formatter --version | grep -o '[0-9\.]*')
 
-if [[ "17.0.0" > "$version" ]]; then
+if [[ "19.0.0" > "$version" ]]; then
     cat <<EOF
 
                                     ERROR
 -----------------------------------------------------------------------------
-            A minimum of version 17 of `which clang-format` is required.
+            A minimum of version 19 of `which clang-format` is required.
             Your version is $version.
             Please fix paths and run again.
 -----------------------------------------------------------------------------
@@ -59,7 +73,9 @@ function grep_code {
     grep -l "${1}" ${sources} -r --include \*.hpp --include \*.cpp
 }
 
-if [[ "$OSTYPE" == "darwin"* ]]; then
+GNU_SED=$(sed --version 2>&1 | grep -q 'GNU' && echo true || echo false)
+
+if [[ "$GNU_SED" == "false" ]]; then # macOS sed
     # make all includes to be <...> style
     grep_code '#include ".*"' | xargs sed -i '' -E 's|#include "(.*)"|#include <\1>|g'
 
@@ -71,7 +87,7 @@ else
     grep_code '#include ".*"' | xargs sed -i -E 's|#include "(.*)"|#include <\1>|g'
 
     # make local includes to be "..." style
-    main_src_dirs=$(find ./src -type d -maxdepth 1 -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g')
+    main_src_dirs=$(find ./src -maxdepth 1 -type d  -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g')
     grep_code "#include <\($main_src_dirs\)/.*>" | xargs sed -i -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g"
 fi
 
@@ -83,9 +99,10 @@ first=$(git diff $sources $cmake_files)
 find $sources -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.ipp' \) -print0 | xargs -0 $formatter
 cmake-format -i $cmake_files
 second=$(git diff $sources $cmake_files)
-changes=$(diff <(echo "$first") <(echo "$second") | wc -l | sed -e 's/^[[:space:]]*//')
+changes=$(diff <(echo "$first") <(echo "$second"))
+changes_number=$(echo -n "$changes" | wc -l | sed -e 's/^[[:space:]]*//')
 
-if [ "$changes" != "0" ]; then
+if [ "$changes_number" != "0" ]; then
     cat <<\EOF
 
                                    WARNING
@@ -95,5 +112,8 @@ if [ "$changes" != "0" ]; then
 -----------------------------------------------------------------------------
 
 EOF
+    if [[ "$1" == "--diff" ]]; then
+        echo "$changes"
+    fi
     exit 1
 fi

.githooks/post-checkout

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-checkout' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-checkout "$@"

.githooks/post-commit

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-commit' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-commit "$@"

.githooks/post-merge

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-merge' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-merge "$@"

.githooks/pre-commit

@@ -4,3 +4,4 @@
 
 source .githooks/check-format
 source .githooks/check-docs
+

.githooks/pre-push

@@ -0,0 +1,58 @@
+#!/bin/sh
+
+
+# git for-each-ref refs/tags  # see which tags are annotated and which are lightweight. Annotated tags are "tag" objects.
+# # Set these so your commits and tags are always signed
+# git config commit.gpgsign true
+# git config tag.gpgsign true
+
+verify_commit_signed() {
+    if git verify-commit HEAD &> /dev/null; then
+        :
+        # echo "HEAD commit seems signed..."
+    else
+        echo "HEAD commit isn't signed!"
+        exit 1
+    fi
+}
+
+verify_tag() {
+    if git describe --exact-match --tags HEAD &> /dev/null; then
+        : # You might be ok to push
+        # echo "Tag is annotated."
+        return 0
+    else
+        echo "Tag for [$version] not an annotated tag."
+        exit 1
+    fi
+}
+
+verify_tag_signed() {
+    if git verify-tag "$version" &> /dev/null ; then
+        : # ok, I guess we'll let you push
+        # echo "Tag appears signed"
+        return 0
+    else
+        echo "$version tag isn't signed"
+        echo "Sign it with [git tag -ams\"$version\" $version]"
+        exit 1
+    fi
+}
+
+while read local_ref local_oid remote_ref remote_oid; do
+    # Check some things if we're pushing a branch called "release/"
+    if echo "$remote_ref" | grep ^refs\/heads\/release\/ &> /dev/null ; then
+        version=$(git tag --points-at HEAD)
+        echo "Looks like you're trying to push a $version release..."
+        echo "Making sure you've signed and tagged it."
+        if verify_commit_signed && verify_tag && verify_tag_signed ; then
+            : # Ok, I guess you can push
+        else
+            exit 1
+        fi
+    fi
+done
+
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'pre-push' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+
+git lfs pre-push "$@"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,6 +8,7 @@ assignees: ''
 ---
 
 <!-- Please search existing issues to avoid creating duplicates. -->
+<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
 
 ## Issue Description
 <!-- Provide a summary for your issue/bug. -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -8,6 +8,7 @@ assignees: ''
 ---
 
 <!-- Please search existing issues to avoid creating duplicates. -->
+<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
 
 ## Summary
 <!-- Provide a summary to the feature request -->

.github/ISSUE_TEMPLATE/question.md

@@ -9,6 +9,7 @@ assignees: ''
 
 <!-- Please search existing issues to avoid creating duplicates. -->
 <!-- Consider starting a [discussion](https://github.com/XRPLF/clio/discussions) instead. -->
+<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
 
 ## Question
 <!-- Your question -->

.github/actions/build_clio/action.yml

@@ -4,12 +4,18 @@ inputs:
   target:
     description: Build target name
     default: all
+  substract_threads:
+    description: An option for the action get_number_of_threads. See get_number_of_threads 
+    required: true
+    default: '0'
 runs:
   using: composite
   steps:
     - name: Get number of threads
       uses: ./.github/actions/get_number_of_threads
       id: number_of_threads
+      with:
+        substract_threads: ${{ inputs.substract_threads }}
 
     - name: Build Clio
       shell: bash

.github/actions/build_docker_image/action.yml

@@ -0,0 +1,66 @@
+name: Build and push Docker image
+description: Build and push Docker image to DockerHub and GitHub Container Registry
+inputs:
+  image_name:
+    description: Name of the image to build
+    required: true
+  push_image:
+    description: Whether to push the image to the registry (true/false)
+    required: true
+  directory:
+    description: The directory containing the Dockerfile
+    required: true
+  tags:
+    description: Comma separated tags to apply to the image
+    required: true
+  platforms:
+    description: Platforms to build the image for (e.g. linux/amd64,linux/arm64)
+    required: true
+  description:
+    description: Short description of the image
+    required: true
+runs:
+  using: composite
+  steps:
+      - name: Login to DockerHub
+        if: ${{ inputs.push_image == 'true' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_PW }}
+
+      - name: Login to GitHub Container Registry
+        if: ${{ inputs.push_image == 'true' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ env.GITHUB_TOKEN }}
+
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/metadata-action@v5
+        id: meta
+        with:
+          images: ${{ inputs.image_name }}
+          tags: ${{ inputs.tags }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ inputs.directory }}
+          platforms: ${{ inputs.platforms }}
+          push: ${{ inputs.push_image == 'true' }}
+          tags: ${{ steps.meta.outputs.tags }}
+
+      - name: Update DockerHub description
+        if: ${{ inputs.push_image == 'true' }}
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_PW }}
+          repository: ${{ inputs.image_name }}
+          short-description: ${{ inputs.description }}
+          readme-filepath: ${{ inputs.directory }}/README.md
+

.github/actions/code_coverage/action.yml

@@ -6,15 +6,19 @@ runs:
     - name: Run tests
       shell: bash
       run: |
-        build/clio_tests --backend_host=scylladb
+        build/clio_tests
 
     - name: Run gcovr
       shell: bash
       run: |
-        gcovr -e unittests --xml build/coverage_report.xml -j8 --exclude-throw-branches
+        gcovr -e tests \
+          -e src/data/cassandra \
+          -e src/data/CassandraBackend.hpp \
+          -e 'src/data/BackendFactory.*' \
+          --xml build/coverage_report.xml -j8 --exclude-throw-branches
 
     - name: Archive coverage report
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: coverage-report.xml
         path: build/coverage_report.xml

.github/actions/create_issue/action.yml

@@ -0,0 +1,35 @@
+name: Create an issue
+description: Create an issue
+inputs:
+  title:
+    description: Issue title
+    required: true
+  body:
+    description: Issue body
+    required: true
+  labels:
+    description: Comma-separated list of labels
+    required: true
+    default: 'bug'
+  assignees:
+    description: Comma-separated list of assignees
+    required: true
+    default: 'cindyyan317,godexsoft,kuznetsss'
+outputs:
+  created_issue_id:
+    description: Created issue id
+    value: ${{ steps.create_issue.outputs.created_issue }}
+runs:
+  using: composite
+  steps:
+      - name: Create an issue
+        id: create_issue
+        shell: bash
+        run: |
+          echo -e '${{ inputs.body }}' > issue.md
+          gh issue create --assignee '${{ inputs.assignees }}' --label '${{ inputs.labels }}' --title '${{ inputs.title }}' --body-file ./issue.md > create_issue.log
+          created_issue=$(cat create_issue.log | sed 's|.*/||')
+          echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
+          rm create_issue.log issue.md
+
+

.github/actions/generate/action.yml

@@ -12,10 +12,22 @@ inputs:
     description: Build type for third-party libraries and clio. Could be 'Release', 'Debug'
     required: true
     default: 'Release'
+  build_integration_tests:
+    description: Whether to build integration tests
+    required: true
+    default: 'true'
   code_coverage:
     description: Whether conan's coverage option should be on or not
     required: true
     default: 'false'
+  static:
+    description: Whether Clio is to be statically linked
+    required: true
+    default: 'false'
+  sanitizer:
+    description: Sanitizer to use
+    required: true
+    default: 'false' # false, tsan, asan or ubsan
 runs:
   using: composite
   steps:
@@ -28,14 +40,21 @@ runs:
       env:
         BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '' }}"
         CODE_COVERAGE: "${{ inputs.code_coverage == 'true' && 'True' || 'False' }}"
+        STATIC_OPTION: "${{ inputs.static == 'true' && 'True' || 'False' }}"
+        INTEGRATION_TESTS_OPTION: "${{ inputs.build_integration_tests == 'true' && 'True' || 'False' }}"
       run: |
         cd build
-        conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:tests=True -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
-
+        conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:static="${STATIC_OPTION}" -o clio:tests=True -o clio:integration_tests="${INTEGRATION_TESTS_OPTION}" -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
+    
     - name: Run cmake
       shell: bash
       env:
         BUILD_TYPE: "${{ inputs.build_type }}"
+        SANITIZER_OPTION: |
+          ${{ inputs.sanitizer == 'tsan' && '-Dsan=thread' ||
+              inputs.sanitizer == 'ubsan' && '-Dsan=undefined' ||
+              inputs.sanitizer == 'asan' && '-Dsan=address' ||
+              '' }}
       run: |
         cd build
-        cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} ${{ inputs.extra_cmake_args }} .. -G Ninja
+        cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" ${SANITIZER_OPTION} .. -G Ninja

.github/actions/get_number_of_threads/action.yml

@@ -1,5 +1,10 @@
 name: Get number of threads
 description: Determines number of threads to use on macOS and Linux
+inputs:
+  substract_threads:
+    description: How many threads to substract from the calculated number 
+    required: true
+    default: '0'
 outputs:
   threads_number:
     description: Number of threads to use
@@ -19,8 +24,11 @@ runs:
       shell: bash
       run: echo "num=$(($(nproc) - 2))" >> $GITHUB_OUTPUT
 
-    - name: Export output variable
-      shell: bash
+    - name: Shift and export number of threads
       id: number_of_threads_export
+      shell: bash
       run: |
-        echo "num=${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }}" >> $GITHUB_OUTPUT
+        num_of_threads=${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }}
+        shift_by=${{ inputs.substract_threads }}
+        shifted=$((num_of_threads - shift_by))
+        echo "num=$(( shifted > 1 ? shifted : 1 ))" >> $GITHUB_OUTPUT

.github/actions/prepare_runner/action.yml

@@ -11,10 +11,8 @@ runs:
       if: ${{ runner.os == 'macOS' }}
       shell: bash
       run: |
-          brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1
-          if ! command -v conan &> /dev/null; then
-            echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
-          fi
+          brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1 ca-certificates
+          echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
 
     - name: Fix git permissions on Linux
       if: ${{ runner.os == 'Linux' }}

.github/actions/restore_cache/action.yml

@@ -4,6 +4,9 @@ inputs:
   conan_dir:
     description: Path to .conan directory
     required: true
+  conan_profile:
+    description: Conan profile name
+    required: true
   ccache_dir:
     description: Path to .ccache directory
     required: true
@@ -44,16 +47,16 @@ runs:
         echo "hash=$hash" >> $GITHUB_OUTPUT
 
     - name: Restore conan cache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       id: conan_cache
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ steps.conan_hash.outputs.hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ steps.conan_hash.outputs.hash }}
 
     - name: Restore ccache cache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       id: ccache_cache
       if: ${{ env.CCACHE_DISABLE != '1' }}
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-${{ inputs.conan_profile }}-develop-${{ steps.git_common_ancestor.outputs.commit }}

.github/actions/save_cache/action.yml

@@ -4,6 +4,9 @@ inputs:
   conan_dir:
     description: Path to .conan directory
     required: true
+  conan_profile:
+    description: Conan profile name
+    required: true
   conan_hash:
     description: Hash to use as a part of conan cache key
     required: true
@@ -41,16 +44,16 @@ runs:
 
     - name: Save conan cache
       if: ${{ inputs.conan_cache_hit != 'true' }}
-      uses: actions/cache/save@v3
+      uses: actions/cache/save@v4
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ inputs.conan_hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ inputs.conan_hash }}
 
     - name: Save ccache cache
       if: ${{ inputs.ccache_cache_hit != 'true' || inputs.ccache_cache_miss_rate == '100.0' }}
-      uses: actions/cache/save@v3
+      uses: actions/cache/save@v4
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-${{ inputs.conan_profile }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
 
 

.github/actions/setup_conan/action.yml

@@ -1,5 +1,9 @@
 name: Setup conan
 description: Setup conan profile and artifactory
+inputs:
+  conan_profile:
+    description: Conan profile name
+    required: true
 outputs:
   conan_profile:
     description: Created conan profile name
@@ -11,10 +15,10 @@ runs:
       if: ${{ runner.os == 'macOS' }}
       shell: bash
       env:
-        CONAN_PROFILE: clio_apple_clang_15
+        CONAN_PROFILE: apple_clang_16
       id: conan_setup_mac
       run: |
-        echo "Creating $CONAN_PROFILE conan profile";
+        echo "Creating $CONAN_PROFILE conan profile"
         conan profile new $CONAN_PROFILE --detect --force
         conan profile update settings.compiler.libcxx=libc++ $CONAN_PROFILE
         conan profile update settings.compiler.cppstd=20 $CONAN_PROFILE
@@ -27,7 +31,7 @@ runs:
       shell: bash
       id: conan_setup_linux
       run: |
-        echo "created_conan_profile=default" >> $GITHUB_OUTPUT
+        echo "created_conan_profile=${{ inputs.conan_profile }}" >> $GITHUB_OUTPUT
 
     - name: Export output variable
       shell: bash

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "04:00"
+      timezone: "Etc/GMT"
+    reviewers:
+      - "cindyyan317"
+      - "godexsoft"
+      - "kuznetsss"
+    commit-message:
+      prefix: "[CI] "
+    target-branch: "develop"

.github/scripts/execute-tests-under-sanitizer

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -o pipefail
+
+# Note: This script is intended to be run from the root of the repository.
+#
+# This script runs each unit-test separately and generates reports from the currently active sanitizer.
+# Output is saved in ./.sanitizer-report in the root of the repository
+
+if [[ -z "$1" ]]; then
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+     Path to clio_tests should be passed as first argument to the script.
+-----------------------------------------------------------------------------
+
+EOF
+    exit 1
+fi
+
+TEST_BINARY=$1
+
+if [[ ! -f "$TEST_BINARY" ]]; then
+  echo "Test binary not found: $TEST_BINARY"
+  exit 1
+fi
+
+TESTS=$($TEST_BINARY --gtest_list_tests | awk '/^  / {print suite $1} !/^  / {suite=$1}')
+
+OUTPUT_DIR="./.sanitizer-report"
+mkdir -p "$OUTPUT_DIR"
+
+for TEST in $TESTS; do
+  OUTPUT_FILE="$OUTPUT_DIR/${TEST//\//_}"
+  export TSAN_OPTIONS="log_path=\"$OUTPUT_FILE\" die_after_fork=0"
+  export ASAN_OPTIONS="log_path=\"$OUTPUT_FILE\""
+  export UBSAN_OPTIONS="log_path=\"$OUTPUT_FILE\""
+  export MallocNanoZone='0' # for MacOSX
+  $TEST_BINARY --gtest_filter="$TEST" > /dev/null 2>&1
+
+  if [ $? -ne 0 ]; then
+    echo "'$TEST' failed a sanitizer check."
+  fi
+done

.github/scripts/update-libxrpl-version

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Note: This script is intended to be run from the root of the repository.
+#
+# This script modifies conanfile.py such that the specified version of libXRPL is used.
+
+if [[ -z "$1" ]]; then
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+            Version should be passed as first argument to the script.
+-----------------------------------------------------------------------------
+
+EOF
+    exit 1
+fi
+
+VERSION=$1
+GNU_SED=$(sed --version 2>&1 | grep -q 'GNU' && echo true || echo false)
+
+echo "+ Updating required libXRPL version to $VERSION"
+
+if [[ "$GNU_SED" == "false" ]]; then
+    sed -i '' -E "s|'xrpl/[a-zA-Z0-9\\.\\-]+'|'xrpl/$VERSION'|g" conanfile.py
+else
+    sed -i -E "s|'xrpl/[a-zA-Z0-9\\.\\-]+'|'xrpl/$VERSION'|g" conanfile.py
+fi

.github/workflows/build.yml

@@ -9,20 +9,24 @@ on:
 jobs:
   check_format:
     name: Check format
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     container:
       image: rippleci/clio_ci:latest
     steps:
+      - name: Fix git permissions on Linux
+        shell: bash
+        run: git config --global --add safe.directory $PWD
+
       - uses: actions/checkout@v4
       - name: Run formatters
         id: run_formatters
         run: |
-          ./.githooks/check-format
+          ./.githooks/check-format --diff
         shell: bash
-  
+ 
   check_docs:
     name: Check documentation
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     container:
       image: rippleci/clio_ci:latest
     steps:
@@ -35,126 +39,52 @@ jobs:
 
   build:
     name: Build
-    needs: 
-      - check_format 
+    needs:
+      - check_format
       - check_docs
     strategy:
       fail-fast: false
       matrix:
         include:
           - os: heavy
-            container:
-              image: rippleci/clio_ci:latest
+            conan_profile: gcc
             build_type: Release
+            container: '{ "image": "rippleci/clio_ci:latest" }'
             code_coverage: false
+            static: true
           - os: heavy
-            container:
-              image: rippleci/clio_ci:latest
+            conan_profile: gcc
             build_type: Debug
+            container: '{ "image": "rippleci/clio_ci:latest" }'
             code_coverage: true
-          - os: macos14
+            static: true
+          - os: heavy
+            conan_profile: clang
+            build_type: Release
+            container: '{ "image": "rippleci/clio_ci:latest" }'
+            code_coverage: false
+            static: true
+          - os: heavy
+            conan_profile: clang
+            build_type: Debug
+            container: '{ "image": "rippleci/clio_ci:latest" }'
+            code_coverage: false
+            static: true
+          - os: macos15
             build_type: Release
             code_coverage: false
-    runs-on: [self-hosted, "${{ matrix.os }}"]
-    container: ${{ matrix.container }}
-
-    services:
-      scylladb:
-        image: ${{ (matrix.code_coverage) && 'scylladb/scylla' || '' }}
-        options: >-
-          --health-cmd "cqlsh -e 'describe cluster'" 
-          --health-interval 10s 
-          --health-timeout 5s 
-          --health-retries 5
-
-    steps:
-      - name: Clean workdir
-        if: ${{ runner.os == 'macOS' }}
-        uses: kuznetsss/workspace-cleanup@1.0
-
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Prepare runner
-        uses: ./.github/actions/prepare_runner
-        with:
-          disable_ccache: false
-
-      - name: Setup conan
-        uses: ./.github/actions/setup_conan
-        id: conan
-
-      - name: Restore cache
-        uses: ./.github/actions/restore_cache
-        id: restore_cache
-        with:
-          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
-          ccache_dir: ${{ env.CCACHE_DIR }}
-          build_type: ${{ matrix.build_type }}
-          code_coverage: ${{ matrix.code_coverage }}
-
-      - name: Run conan and cmake
-        uses: ./.github/actions/generate
-        with:
-          conan_profile: ${{ steps.conan.outputs.conan_profile }}
-          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
-          build_type: ${{ matrix.build_type }}
-          code_coverage: ${{ matrix.code_coverage }}
-
-      - name: Build Clio
-        uses: ./.github/actions/build_clio
-
-      - name: Show ccache's statistics
-        shell: bash
-        id: ccache_stats
-        run: |
-          ccache -s > /tmp/ccache.stats
-          miss_rate=$(cat /tmp/ccache.stats | grep 'Misses' | head -n1 | sed 's/.*(\(.*\)%).*/\1/')
-          echo "miss_rate=${miss_rate}" >> $GITHUB_OUTPUT
-          cat /tmp/ccache.stats
-
-      - name: Strip tests
-        if: ${{ !matrix.code_coverage }}
-        run: strip build/clio_tests
-
-      - name: Upload clio_server
-        uses: actions/upload-artifact@v3
-        with:
-          name: clio_server_${{ runner.os }}_${{ matrix.build_type }}
-          path: build/clio_server
-
-      - name: Upload clio_tests
-        if: ${{ !matrix.code_coverage }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: clio_tests_${{ runner.os }}
-          path: build/clio_tests
-
-      - name: Save cache
-        uses: ./.github/actions/save_cache
-        with:
-          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
-          conan_hash: ${{ steps.restore_cache.outputs.conan_hash }}
-          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
-          ccache_dir: ${{ env.CCACHE_DIR }}
-          ccache_cache_hit: ${{ steps.restore_cache.outputs.ccache_cache_hit }}
-          ccache_cache_miss_rate: ${{ steps.ccache_stats.outputs.miss_rate }}
-          build_type: ${{ matrix.build_type }}
-          code_coverage: ${{ matrix.code_coverage }}
-
-      # TODO: This is not a part of build process but it is the easiest way to do it here.
-      # It will be refactored in https://github.com/XRPLF/clio/issues/1075
-      - name: Run code coverage
-        if: ${{ matrix.code_coverage }}
-        uses: ./.github/actions/code_coverage
-
-  upload_coverage_report:
-    name: Codecov
-    needs: build
-    uses: ./.github/workflows/upload_coverage_report.yml
-    secrets:
-      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+            static: false
+    uses: ./.github/workflows/build_impl.yml
+    with:
+      runs_on: ${{ matrix.os }}
+      container: ${{ matrix.container }}
+      conan_profile: ${{ matrix.conan_profile }}
+      build_type: ${{ matrix.build_type }}
+      code_coverage: ${{ matrix.code_coverage }}
+      static: ${{ matrix.static }}
+      unit_tests: true
+      integration_tests: true
+      clio_server: true
 
   test:
     name: Run Tests
@@ -164,10 +94,24 @@ jobs:
       matrix:
         include:
           - os: heavy
+            conan_profile: gcc
+            build_type: Release
             container:
               image: rippleci/clio_ci:latest
-          - os: macos14
-    runs-on: [self-hosted, "${{ matrix.os }}"]
+          - os: heavy
+            conan_profile: clang
+            build_type: Release
+            container:
+              image: rippleci/clio_ci:latest
+          - os: heavy
+            conan_profile: clang
+            build_type: Debug
+            container:
+              image: rippleci/clio_ci:latest
+          - os: macos15
+            conan_profile: apple_clang_16
+            build_type: Release
+    runs-on: ${{ matrix.os }}
     container: ${{ matrix.container }}
 
     steps:
@@ -175,10 +119,52 @@ jobs:
         if: ${{ runner.os == 'macOS' }}
         uses: kuznetsss/workspace-cleanup@1.0
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: clio_tests_${{ runner.os }}
+          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
+
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests
-          ./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
+          ./clio_tests
+
+  check_config:
+    name: Check Config Description
+    needs: build
+    runs-on: heavy
+    container: 
+      image: rippleci/clio_ci:latest
+    steps:
+      - uses: actions/checkout@v4 
+      - uses: actions/download-artifact@v4
+        with:
+          name: clio_server_Linux_Release_gcc
+      - name: Compare Config Description
+        shell: bash
+        run: |
+          repoConfigFile=docs/config-description.md
+          if ! [ -f ${repoConfigFile} ]; then
+            echo "Config Description markdown file is missing in docs folder"
+            exit 1
+          fi
+
+          chmod +x ./clio_server
+          configDescriptionFile=config_description_new.md
+          ./clio_server -d ${configDescriptionFile}
+
+          configDescriptionHash=$(sha256sum ${configDescriptionFile} | cut -d' ' -f1)
+          repoConfigHash=$(sha256sum ${repoConfigFile} | cut -d' ' -f1)
+
+          if [ ${configDescriptionHash} != ${repoConfigHash} ]; then
+            echo "Markdown file is not up to date"
+            diff -u "${repoConfigFile}" "${configDescriptionFile}"
+            rm -f ${configDescriptionFile}
+            exit 1
+          fi
+          rm -f ${configDescriptionFile}
+          exit 0
+                    
+
+
+
+

.github/workflows/build_clio_docker_image.yml

@@ -0,0 +1,95 @@
+name: Build and publish Clio docker image
+on:
+  workflow_call:
+    inputs:
+      tags:
+        required: true
+        type: string
+        description: Comma separated tags for docker image
+      artifact_name:
+        type: string
+        description: Name of Github artifact to put into docker image
+      strip_binary:
+        type: boolean
+        description: Whether to strip clio binary
+        default: true
+      publish_image:
+        type: boolean
+        description: Whether to publish docker image
+        required: true
+
+  workflow_dispatch:
+    inputs:
+      tags:
+        required: true
+        type: string
+        description: Comma separated tags for docker image
+      clio_server_binary_url:
+        required: true
+        type: string
+        description: Url to download clio_server binary from
+      binary_sha256:
+        required: true
+        type: string
+        description: sha256 hash of the binary
+      strip_binary:
+        type: boolean
+        description: Whether to strip clio binary
+        default: true
+
+jobs:
+  build_and_publish_image:
+    name: Build and publish image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download Clio binary from artifact
+        if: ${{ inputs.artifact_name != null }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.artifact_name }}
+          path: ./docker/clio/artifact/
+
+      - name: Download Clio binary from url
+        if: ${{ inputs.clio_server_binary_url != null }}
+        shell: bash
+        run: |
+          wget ${{inputs.clio_server_binary_url}} -P ./docker/clio/artifact/
+          if [ "$(sha256sum ./docker/clio/clio_server | awk '{print $1}')" != "${{inputs.binary_sha256}}" ]; then
+            echo "Binary sha256 sum doesn't match"
+            exit 1
+          fi
+      - name: Unpack binary
+        shell: bash
+        run: |
+          sudo apt update && sudo apt install -y tar unzip
+          cd docker/clio/artifact
+          artifact=$(find . -type f)
+          if [[ $artifact == *.zip ]]; then
+            unzip $artifact
+          elif [[ $artifact == *.tar.gz ]]; then
+            tar -xvf $artifact
+          fi
+          mv clio_server ../
+          cd ../
+          rm -rf ./artifact
+
+      - name: Strip binary
+        if: ${{ inputs.strip_binary }}
+        shell: bash
+        run: strip ./docker/clio/clio_server
+
+      - name: Build Docker image
+        uses: ./.github/actions/build_docker_image
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          image_name: rippleci/clio
+          push_image: ${{ inputs.publish_image }}
+          directory: docker/clio
+          tags: ${{ inputs.tags }}
+          platforms: linux/amd64
+          description: Clio is an XRP Ledger API server.

.github/workflows/build_impl.yml

@@ -0,0 +1,192 @@
+name: Reusable build
+on:
+  workflow_call:
+    inputs:
+      runs_on:
+        description: Runner to run the job on
+        required: true
+        type: string
+        default: heavy
+
+      container:
+        description: "The container object as a JSON string (leave empty to run natively)"
+        required: true
+        type: string
+        default: ""
+
+      conan_profile:
+        description: Conan profile to use
+        required: true
+        type: string
+
+      build_type:
+        description: Build type
+        required: true
+        type: string
+
+      disable_cache:
+        description: Whether ccache and conan cache should be disabled
+        required: false
+        type: boolean
+        default: false
+
+      code_coverage:
+        description: Whether to enable code coverage
+        required: true
+        type: boolean
+        default: false
+
+      static:
+        description: Whether to build static binaries
+        required: true
+        type: boolean
+        default: true
+
+      unit_tests:
+        description: Whether to run unit tests
+        required: true
+        type: boolean
+        default: false
+
+      integration_tests:
+        description: Whether to run integration tests
+        required: true
+        type: boolean
+        default: false
+
+      clio_server:
+        description: Whether to build clio_server
+        required: true
+        type: boolean
+        default: true
+
+      target:
+        description: Build target name
+        required: false
+        type: string
+        default: all
+
+      sanitizer:
+        description: Sanitizer to use
+        required: false
+        type: string
+        default: 'false'
+
+jobs:
+  build:
+    name: Build ${{ inputs.container != '' && 'in container' || 'natively' }}
+    runs-on: ${{ inputs.runs_on }} 
+    container: ${{ inputs.container != '' && fromJson(inputs.container) || null }}
+
+    steps:
+      - name: Clean workdir
+        if: ${{ runner.os == 'macOS' }}
+        uses: kuznetsss/workspace-cleanup@1.0
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+          disable_ccache: ${{ inputs.disable_cache }}
+
+      - name: Setup conan
+        uses: ./.github/actions/setup_conan
+        id: conan
+        with:
+          conan_profile: ${{ inputs.conan_profile }}
+
+      - name: Restore cache
+        if: ${{ !inputs.disable_cache }}
+        uses: ./.github/actions/restore_cache
+        id: restore_cache
+        with:
+          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+          ccache_dir: ${{ env.CCACHE_DIR }}
+          build_type: ${{ inputs.build_type }}
+          code_coverage: ${{ inputs.code_coverage }}
+
+      - name: Run conan and cmake
+        uses: ./.github/actions/generate
+        with:
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+          conan_cache_hit: ${{ !inputs.disable_cache && steps.restore_cache.outputs.conan_cache_hit }}
+          build_type: ${{ inputs.build_type }}
+          code_coverage: ${{ inputs.code_coverage }}
+          static: ${{ inputs.static }}
+          sanitizer: ${{ inputs.sanitizer }}
+
+      - name: Build Clio
+        uses: ./.github/actions/build_clio
+        with:
+          target: ${{ inputs.target }}
+
+      - name: Show ccache's statistics
+        if: ${{ !inputs.disable_cache }}
+        shell: bash
+        id: ccache_stats
+        run: |
+          ccache -s > /tmp/ccache.stats
+          miss_rate=$(cat /tmp/ccache.stats | grep 'Misses' | head -n1 | sed 's/.*(\(.*\)%).*/\1/')
+          echo "miss_rate=${miss_rate}" >> $GITHUB_OUTPUT
+          cat /tmp/ccache.stats
+
+      - name: Strip unit_tests
+        if: ${{ inputs.unit_tests && !inputs.code_coverage && inputs.sanitizer == 'false' }}
+        run: strip build/clio_tests
+      
+      - name: Strip integration_tests
+        if: ${{ inputs.integration_tests && !inputs.code_coverage }}
+        run: strip build/clio_integration_tests
+
+      - name: Upload clio_server
+        if: ${{ inputs.clio_server }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: clio_server_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }}
+          path: build/clio_server
+     
+      - name: Upload clio_tests
+        if: ${{ inputs.unit_tests && !inputs.code_coverage }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }}
+          path: build/clio_tests
+      
+      - name: Upload clio_integration_tests
+        if: ${{ inputs.integration_tests && !inputs.code_coverage }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }}
+          path: build/clio_integration_tests
+
+      - name: Save cache
+        if: ${{ !inputs.disable_cache && github.ref == 'refs/heads/develop' }}
+        uses: ./.github/actions/save_cache
+        with:
+          conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          conan_hash: ${{ steps.restore_cache.outputs.conan_hash }}
+          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
+          ccache_dir: ${{ env.CCACHE_DIR }}
+          ccache_cache_hit: ${{ steps.restore_cache.outputs.ccache_cache_hit }}
+          ccache_cache_miss_rate: ${{ steps.ccache_stats.outputs.miss_rate }}
+          build_type: ${{ inputs.build_type }}
+          code_coverage: ${{ inputs.code_coverage }}
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+
+      # TODO: This is not a part of build process but it is the easiest way to do it here.
+      # It will be refactored in https://github.com/XRPLF/clio/issues/1075
+      - name: Run code coverage
+        if: ${{ inputs.code_coverage }}
+        uses: ./.github/actions/code_coverage
+
+  upload_coverage_report:
+    if: ${{ inputs.code_coverage }} 
+    name: Codecov
+    needs: build
+    uses: ./.github/workflows/upload_coverage_report.yml
+    secrets:
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/check_libxrpl.yml

@@ -0,0 +1,91 @@
+name: Check new libXRPL
+on:
+  repository_dispatch:
+    types: [check_libxrpl]
+
+jobs:
+  build:
+    name: Build Clio / `libXRPL ${{ github.event.client_payload.version }}`
+    runs-on: [self-hosted, heavy]
+    container:
+      image: rippleci/clio_ci:latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            fetch-depth: 0
+
+      - name: Update libXRPL version requirement
+        shell: bash
+        run: |
+          ./.github/scripts/update-libxrpl-version ${{ github.event.client_payload.version }}
+
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+            disable_ccache: true
+
+      - name: Setup conan
+        uses: ./.github/actions/setup_conan
+        id: conan
+        with:
+          conan_profile: gcc
+
+      - name: Run conan and cmake
+        uses: ./.github/actions/generate
+        with:
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
+          build_type: Release
+
+      - name: Build Clio
+        uses: ./.github/actions/build_clio
+
+      - name: Strip tests
+        run: strip build/clio_tests
+
+      - name: Upload clio_tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: clio_tests_check_libxrpl
+          path: build/clio_tests
+
+  run_tests:
+    name: Run tests
+    needs: build
+    runs-on: [self-hosted, heavy]
+    container:
+      image: rippleci/clio_ci:latest
+
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: clio_tests_check_libxrpl
+
+      - name: Run clio_tests
+        run: |
+          chmod +x ./clio_tests
+          ./clio_tests
+
+  create_issue_on_failure:
+    name: Create an issue on failure
+    needs: [build, run_tests]
+    if: ${{ always() && contains(needs.*.result, 'failure') }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create an issue
+        uses: ./.github/actions/create_issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          labels: 'compatibility,bug'
+          title: 'Proposed libXRPL check failed'
+          body: >
+            Clio build or tests failed against `libXRPL ${{ github.event.client_payload.version }}`.
+
+            Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/

.github/workflows/check_pr_title.yml

@@ -0,0 +1,18 @@
+name: Check PR title
+on:
+  pull_request:
+    types: [opened, edited, reopened, synchronize]
+    branches: [develop]
+
+jobs:
+  check_title:
+    runs-on: ubuntu-latest
+    # permissions:
+    #   pull-requests: write
+    steps:
+      - uses: ytanikin/PRConventionalCommits@1.3.0
+        with:
+          task_types: '["build","feat","fix","docs","test","ci","style","refactor","perf","chore"]'
+          add_label: false
+          # Turned off labelling because it leads to an error, see https://github.com/ytanikin/PRConventionalCommits/issues/19
+          # custom_labels: '{"build":"build", "feat":"enhancement", "fix":"bug", "docs":"documentation", "test":"testability", "ci":"ci", "style":"refactoring", "refactor":"refactoring", "perf":"performance", "chore":"tooling"}'

.github/workflows/clang-tidy.yml

@@ -1,7 +1,7 @@
 name: Clang-tidy check
 on:
   schedule:
-    - cron: "0 6 * * 1-5"
+    - cron: "0 9 * * 1-5"
   workflow_dispatch:
   pull_request:
     branches: [develop]
@@ -12,7 +12,7 @@ on:
 
 jobs:
   clang_tidy:
-    runs-on: [self-hosted, Linux]
+    runs-on: heavy
     container:
       image: rippleci/clio_ci:latest
     permissions:
@@ -33,6 +33,8 @@ jobs:
       - name: Setup conan
         uses: ./.github/actions/setup_conan
         id: conan
+        with:
+          conan_profile: clang
 
       - name: Restore cache
         uses: ./.github/actions/restore_cache
@@ -40,6 +42,7 @@ jobs:
         with:
           conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
           ccache_dir: ${{ env.CCACHE_DIR }}
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
 
       - name: Run conan and cmake
         uses: ./.github/actions/generate
@@ -57,7 +60,7 @@ jobs:
         shell: bash
         id: run_clang_tidy
         run: |
-          run-clang-tidy-17 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
+          run-clang-tidy-19 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
 
       - name: Check format
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
@@ -76,18 +79,17 @@ jobs:
       - name: Create an issue
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
         id: create_issue
-        shell: bash
+        uses: ./.github/actions/create_issue
         env:
           GH_TOKEN: ${{ github.token }}
-        run: |
-          echo -e 'Clang-tidy found issues in the code:\n' > issue.md
-          echo -e "List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/" >> issue.md
-          gh issue create --assignee 'cindyyan317,godexsoft,kuznetsss' --label bug --title 'Clang-tidy found bugs in code🐛' --body-file ./issue.md > create_issue.log
-          created_issue=$(cat create_issue.log | sed 's|.*/||')
-          echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
-          rm create_issue.log issue.md
+        with:
+          title: 'Clang-tidy found bugs in code 🐛'
+          body: >
+            Clang-tidy found issues in the code:
 
-      - uses: crazy-max/ghaction-import-gpg@v5
+            List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/
+
+      - uses: crazy-max/ghaction-import-gpg@v6
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
         with:
           gpg_private_key: ${{ secrets.ACTIONS_GPG_PRIVATE_KEY }}
@@ -97,7 +99,7 @@ jobs:
 
       - name: Create PR with fixes
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v7
         env:
           GH_REPO: ${{ github.repository }}
           GH_TOKEN: ${{ github.token }}
@@ -107,8 +109,8 @@ jobs:
           branch: "clang_tidy/autofix"
           branch-suffix: timestamp
           delete-branch: true
-          title: "[CI] clang-tidy auto fixes"
-          body: "Fixes #${{ steps.create_issue.outputs.created_issue }}. Please review and commit clang-tidy fixes."
+          title: "style: clang-tidy auto fixes"
+          body: "Fixes #${{ steps.create_issue.outputs.created_issue_id }}. Please review and commit clang-tidy fixes."
           reviewers: "cindyyan317,godexsoft,kuznetsss"
 
       - name: Fail the job

.github/workflows/clang-tidy_on_fix_merged.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   restart_clang_tidy:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     permissions:
       actions: write
@@ -17,7 +17,7 @@ jobs:
         id: check
         shell: bash
         run: |
-          passed=$(if [[ $(git log -1 --pretty=format:%s | grep '\[CI\] clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi)
+          passed=$(if [[ $(git log -1 --pretty=format:%s | grep 'style: clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi)
           echo "passed=$passed" >> $GITHUB_OUTPUT
 
       - name: Run clang-tidy workflow

.github/workflows/docs.yml

@@ -1,7 +1,7 @@
 name: Documentation
 on:
   push:
-    branches: [release/*, develop]
+    branches: [develop]
   workflow_dispatch:
 
 permissions:
@@ -18,13 +18,15 @@ jobs:
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     continue-on-error: true
     container:
       image: rippleci/clio_ci:latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with: 
+          lfs: true
 
       - name: Build docs
         run: |
@@ -32,13 +34,13 @@ jobs:
           cmake ../docs && cmake --build . --target docs
       
       - name: Setup Pages
-        uses: actions/configure-pages@v3
+        uses: actions/configure-pages@v5
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
           path: build_docs/html
-          name: docs-develop # TODO: use x.y.z for `release/x.y.z` branches and `develop` for latest dev docs
+          name: docs-develop
       
       - name: Deploy to GitHub Pages
         id: deployment

.github/workflows/nightly.yml

@@ -1,8 +1,12 @@
 name: Nightly release
 on:
   schedule:
-    - cron:  '0 5 * * 1-5'
+    - cron:  '0 8 * * 1-5'
   workflow_dispatch:
+  pull_request:
+    paths:
+      - '.github/workflows/nightly.yml'
+      - '.github/workflows/build_clio_docker_image.yml'
 
 jobs:
   build:
@@ -11,68 +15,29 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macos14
+          - os: macos15
             build_type: Release
+            static: false
           - os: heavy
             build_type: Release
-            container:
-              image: rippleci/clio_ci:latest
+            static: true
+            container: '{ "image": "rippleci/clio_ci:latest" }'
           - os: heavy
             build_type: Debug
-            container:
-              image: rippleci/clio_ci:latest
-    runs-on: [self-hosted, "${{ matrix.os }}"]
-    container: ${{ matrix.container }}
-
-    steps:
-      - name: Clean workdir
-        if: ${{ runner.os == 'macOS' }}
-        uses: kuznetsss/workspace-cleanup@1.0
-
-      - uses: actions/checkout@v4
-        with:
-            fetch-depth: 0
-
-      - name: Prepare runner
-        uses: ./.github/actions/prepare_runner
-        with:
-            disable_ccache: true
-
-      - name: Setup conan
-        uses: ./.github/actions/setup_conan
-        id: conan
-
-      - name: Run conan and cmake
-        uses: ./.github/actions/generate
-        with:
-          conan_profile: ${{ steps.conan.outputs.conan_profile }}
-          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
-          build_type: ${{ matrix.build_type }}
-
-      - name: Build Clio
-        uses: ./.github/actions/build_clio
-
-      - name: Strip tests
-        run: strip build/clio_tests
-
-      - name: Upload clio_tests
-        uses: actions/upload-artifact@v3
-        with:
-          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
-          path: build/clio_tests
-
-      - name: Compress clio_server
-        shell: bash
-        run: |
-            cd build
-            tar czf ./clio_server_${{ runner.os }}_${{ matrix.build_type }}.tar.gz ./clio_server
-
-      - name: Upload clio_server
-        uses: actions/upload-artifact@v3
-        with:
-          name: clio_server_${{ runner.os }}_${{ matrix.build_type }}
-          path: build/clio_server_${{ runner.os }}_${{ matrix.build_type }}.tar.gz
-
+            static: true
+            container: '{ "image": "rippleci/clio_ci:latest" }'
+    uses: ./.github/workflows/build_impl.yml
+    with:
+      runs_on: ${{ matrix.os }}
+      container: ${{ matrix.container }}
+      conan_profile: gcc
+      build_type: ${{ matrix.build_type }}
+      code_coverage: false
+      static: ${{ matrix.static }}
+      unit_tests: true
+      integration_tests: true
+      clio_server: true
+      disable_cache: true
 
   run_tests:
     needs: build
@@ -80,31 +45,64 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macos14
+          - os: macos15
+            conan_profile: apple_clang_16
             build_type: Release
+            integration_tests: false
           - os: heavy
+            conan_profile: gcc
             build_type: Release
+            container:
+              image: rippleci/clio_ci:latest
+            integration_tests: true  
           - os: heavy
+            conan_profile: gcc
             build_type: Debug
+            container:
+              image: rippleci/clio_ci:latest
+            integration_tests: true  
     runs-on: [self-hosted, "${{ matrix.os }}"]
+    container: ${{ matrix.container }}
+
+    services:
+      scylladb:
+        image: ${{ (matrix.integration_tests) && 'scylladb/scylla' || '' }}
+        options: >-
+          --health-cmd "cqlsh -e 'describe cluster'" 
+          --health-interval 10s 
+          --health-timeout 5s 
+          --health-retries 5
 
     steps:
       - name: Clean workdir
         if: ${{ runner.os == 'macOS' }}
         uses: kuznetsss/workspace-cleanup@1.0
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
+          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
 
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests
-          ./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
+          ./clio_tests
+  
+      - uses: actions/download-artifact@v4
+        with:
+          name: clio_integration_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
+
+  # To be enabled back once docker in mac runner arrives
+  # https://github.com/XRPLF/clio/issues/1400
+      - name: Run clio_integration_tests
+        if: matrix.integration_tests 
+        run: |
+          chmod +x ./clio_integration_tests
+          ./clio_integration_tests --backend_host=scylladb
 
   nightly_release:
+    if: ${{ github.event_name != 'pull_request' }}
     needs: run_tests
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       GH_REPO: ${{ github.repository }}
       GH_TOKEN: ${{ github.token }}
@@ -113,16 +111,16 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           path: nightly_release
+          pattern: clio_server_*
 
       - name: Prepare files
         shell: bash
         run: |
           cp ${{ github.workspace }}/.github/workflows/nightly_notes.md "${RUNNER_TEMP}/nightly_notes.md"
           cd nightly_release
-          rm -r clio_tests*
           for d in $(ls); do
             archive_name=$(ls $d)
             mv ${d}/${archive_name} ./
@@ -144,3 +142,36 @@ jobs:
           gh release create nightly --prerelease --title "Clio development (nightly) build" \
             --target $GITHUB_SHA --notes-file "${RUNNER_TEMP}/nightly_notes.md" \
             ./nightly_release/clio_server*
+
+  build_and_publish_docker_image:
+    uses: ./.github/workflows/build_clio_docker_image.yml
+    needs: run_tests
+    secrets: inherit
+    with:
+      tags: |
+          type=raw,value=nightly
+          type=raw,value=${{ github.sha }}
+      artifact_name: clio_server_Linux_Release_gcc
+      strip_binary: true
+      publish_image: ${{ github.event_name != 'pull_request' }}
+
+  create_issue_on_failure:
+    needs: [build, run_tests, nightly_release, build_and_publish_docker_image]
+    if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name != 'pull_request' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create an issue
+        uses: ./.github/actions/create_issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          title: 'Nightly release failed 🌙'
+          body: >
+            Nightly release failed:
+
+            Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/

.github/workflows/sanitizers.yml

@@ -0,0 +1,106 @@
+name: Run tests with sanitizers
+on:
+  schedule:
+    - cron: "0 4 * * 1-5"
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - '.github/workflows/sanitizers.yml'
+
+jobs:
+  build:
+    name: Build clio tests
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sanitizer: tsan
+            compiler: gcc
+          - sanitizer: asan
+            compiler: gcc
+          # - sanitizer: ubsan # todo: enable when heavy runners are available
+          #   compiler: gcc
+    uses: ./.github/workflows/build_impl.yml
+    with:
+      runs_on: ubuntu-latest # todo: change to heavy
+      container: '{ "image": "rippleci/clio_ci:latest" }'
+      disable_cache: true
+      conan_profile: ${{ matrix.compiler }}.${{ matrix.sanitizer }}
+      build_type: Release
+      code_coverage: false
+      static: false
+      unit_tests: true
+      integration_tests: false
+      clio_server: false
+      target: clio_tests
+      sanitizer: ${{ matrix.sanitizer }}
+
+  # consider combining this with the previous matrix instead
+  run_tests:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sanitizer: tsan
+            compiler: gcc
+          - sanitizer: asan
+            compiler: gcc
+          # - sanitizer: ubsan # todo: enable when heavy runners are available
+          #   compiler: gcc
+    runs-on: ubuntu-latest # todo: change to heavy
+    container:
+      image: rippleci/clio_ci:latest
+    permissions:
+      contents: write
+      issues: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: clio_tests_${{ runner.os }}_Release_${{ matrix.compiler }}.${{ matrix.sanitizer }}
+
+      - name: Run clio_tests [${{ matrix.compiler }} / ${{ matrix.sanitizer }}]
+        shell: bash
+        run: |
+          chmod +x ./clio_tests
+          ./.github/scripts/execute-tests-under-sanitizer ./clio_tests
+
+      - name: Check for sanitizer report
+        shell: bash
+        id: check_report
+        run: |
+          if ls .sanitizer-report/* 1> /dev/null 2>&1; then
+            echo "found_report=true" >> $GITHUB_OUTPUT
+          else
+            echo "found_report=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Upload report
+        if: ${{ steps.check_report.outputs.found_report == 'true' }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.compiler }}_${{ matrix.sanitizer }}_report
+          path: .sanitizer-report/*
+          include-hidden-files: true
+
+      #
+      # todo: enable when we have fixed all currently existing issues from sanitizers
+      #
+      # - name: Create an issue
+      #   if: ${{ steps.check_report.outputs.found_report == 'true' }}
+      #   uses: ./.github/actions/create_issue
+      #   env:
+      #     GH_TOKEN: ${{ github.token }}
+      #   with:
+      #     labels: 'bug'
+      #     title: '[${{ matrix.sanitizer }}/${{ matrix.compiler }}] reported issues'
+      #     body: >
+      #       Clio tests failed one or more sanitizer checks when built with ${{ matrix.compiler }}`.
+
+      #       Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/
+      #       Reports are available as artifacts.

.github/workflows/update_docker_ci.yml

@@ -1,40 +1,36 @@
 name: Update CI docker image
 on:
+  pull_request:
+    paths:
+      - 'docker/ci/**'
+      - 'docker/compilers/**'
+      - .github/workflows/update_docker_ci.yml
   push:
     branches: [develop]
     paths:
-      - 'docker/ci/**'
+      - 'docker/ci/**'        # CI image must update when either its dockerfile changes
+      - 'docker/compilers/**' # or any compilers changed and were pushed by hand
       - .github/workflows/update_docker_ci.yml
   workflow_dispatch:
 
 jobs:
   build_and_push:
     name: Build and push docker image
-    runs-on: ubuntu-20.04
+    runs-on: [self-hosted, heavy]
     steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PW }}
-
       - uses: actions/checkout@v4
-      - uses: docker/setup-qemu-action@v3
-      - uses: docker/setup-buildx-action@v3
-      - uses: docker/metadata-action@v5
-        id: meta
+      - uses: ./.github/actions/build_docker_image
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          images: rippleci/clio_ci
+          image_name: rippleci/clio_ci
+          push_image: ${{ github.event_name != 'pull_request' }}
+          directory: docker/ci
           tags: |
             type=raw,value=latest
-            type=raw,value=gcc_11
-            type=raw,value=${{ env.GITHUB_SHA }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: ${{ github.workspace }}/docker/ci
+            type=raw,value=gcc_12_clang_16
+            type=raw,value=${{ github.sha }}
           platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-
+          description: CI image for XRPLF/clio.

.github/workflows/upload_coverage_report.yml

@@ -9,23 +9,23 @@ on:
 jobs:
   upload_report:
     name: Upload report
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Download report artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: coverage-report.xml
           path: build
 
       - name: Upload coverage report
         if: ${{ hashFiles('build/coverage_report.xml') != '' }}
-        uses: wandalen/wretry.action@v1.3.0
+        uses: wandalen/wretry.action@v3.7.3
         with:
-          action: codecov/codecov-action@v3
+          action: codecov/codecov-action@v4
           with: |
             files: build/coverage_report.xml 
             fail_ci_if_error: false

.gitignore

@@ -6,6 +6,7 @@
 .vscode
 .python-version
 .DS_Store
+.sanitizer-report
 CMakeUserPresets.json
 config.json
-src/main/impl/Build.cpp
+src/util/build/Build.cpp

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.16.3)
+cmake_minimum_required(VERSION 3.20)
 
 set(CMAKE_PROJECT_INCLUDE_BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ClioVersion.cmake)
 
@@ -8,12 +8,16 @@ project(clio VERSION ${CLIO_VERSION} HOMEPAGE_URL "https://github.com/XRPLF/clio
 
 # =========================== Options ====================================== #
 option(verbose "Verbose build" FALSE)
-option(tests "Build tests" FALSE)
+option(tests "Build unit tests" FALSE)
+option(integration_tests "Build integration tests" FALSE)
 option(benchmark "Build benchmarks" FALSE)
 option(docs "Generate doxygen docs" FALSE)
 option(coverage "Build test coverage report" FALSE)
 option(packaging "Create distribution packages" FALSE)
 option(lint "Run clang-tidy checks during compilation" FALSE)
+option(static "Statically linked Clio" FALSE)
+option(snapshot "Build snapshot tool" FALSE)
+
 # ========================================================================== #
 set(san "" CACHE STRING "Add sanitizer instrumentation")
 set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
@@ -28,17 +32,9 @@ include(CheckCXXCompilerFlag)
 include(ClangTidy)
 
 add_library(clio_options INTERFACE)
+target_compile_features(clio_options INTERFACE cxx_std_23) # Clio needs c++23 but deps can remain c++20 for now
 target_include_directories(clio_options INTERFACE ${CMAKE_SOURCE_DIR}/src)
 
-# Set coverage build options
-if (coverage)
-  if (NOT tests)
-    message(FATAL_ERROR "Coverage requires tests to be enabled")
-  endif ()
-  include(CodeCoverage)
-  append_coverage_compiler_flags_to_target(clio_options INTERFACE)
-endif ()
-
 if (verbose)
   set(CMAKE_VERBOSE_MAKEFILE TRUE)
 endif ()
@@ -63,10 +59,7 @@ include(deps/cassandra)
 include(deps/libbacktrace)
 
 add_subdirectory(src)
-
-if (tests)
-  add_subdirectory(unittests)
-endif ()
+add_subdirectory(tests)
 
 if (benchmark)
   add_subdirectory(benchmarks)
@@ -74,15 +67,21 @@ endif ()
 
 # Enable selected sanitizer if enabled via `san`
 if (san)
+  set(SUPPORTED_SANITIZERS "address" "thread" "memory" "undefined")
+  list(FIND SUPPORTED_SANITIZERS "${san}" INDEX)
+  if (INDEX EQUAL -1)
+    message(FATAL_ERROR "Error: Unsupported sanitizer '${san}'. Supported values are: ${SUPPORTED_SANITIZERS}.")
+  endif ()
+
   target_compile_options(
-    clio PUBLIC # Sanitizers recommend minimum of -O1 for reasonable performance
-                $<$<CONFIG:Debug>:-O1> ${SAN_FLAG} -fno-omit-frame-pointer
+    clio_options INTERFACE # Sanitizers recommend minimum of -O1 for reasonable performance
+                           $<$<CONFIG:Debug>:-O1> ${SAN_FLAG} -fno-omit-frame-pointer
   )
   target_compile_definitions(
-    clio PUBLIC $<$<STREQUAL:${san},address>:SANITIZER=ASAN> $<$<STREQUAL:${san},thread>:SANITIZER=TSAN>
-                $<$<STREQUAL:${san},memory>:SANITIZER=MSAN> $<$<STREQUAL:${san},undefined>:SANITIZER=UBSAN>
+    clio_options INTERFACE $<$<STREQUAL:${san},address>:SANITIZER=ASAN> $<$<STREQUAL:${san},thread>:SANITIZER=TSAN>
+                           $<$<STREQUAL:${san},memory>:SANITIZER=MSAN> $<$<STREQUAL:${san},undefined>:SANITIZER=UBSAN>
   )
-  target_link_libraries(clio INTERFACE ${SAN_FLAG} ${SAN_LIB})
+  target_link_libraries(clio_options INTERFACE ${SAN_FLAG} ${SAN_LIB})
 endif ()
 
 # Generate `docs` target for doxygen documentation if enabled Note: use `make docs` to generate the documentation
@@ -94,3 +93,7 @@ include(install/install)
 if (packaging)
   include(cmake/packaging.cmake) # This file exists only in build runner
 endif ()
+
+if (snapshot)
+  add_subdirectory(tools/snapshot)
+endif ()

CONTRIBUTING.md

@@ -6,7 +6,7 @@ To contribute, please:
 2. Create a new branch on which to commit/push your changes.
 3. Write and test your code.
 4. Ensure that your code compiles with the provided build engine and update the provided build engine as part of your PR where needed and where appropriate.
-5. Where applicable, write test cases for your code and include those in `unittests`.
+5. Where applicable, write test cases for your code and include those in the relevant subfolder under `tests`.
 6. Ensure your code passes automated checks (e.g. clang-format)
 7. Squash your commits (i.e. rebase) into as few commits as is reasonable to describe your changes at a high level (typically a single commit for a small change). See below for more details.
 8. Open a PR to the main repository onto the _develop_ branch, and follow the provided template.
@@ -21,7 +21,7 @@ git config --local core.hooksPath .githooks
 ```
 
 ## Git hooks dependencies
-The pre-commit hook requires `clang-format >= 17.0.0` and `cmake-format` to be installed on your machine.
+The pre-commit hook requires `clang-format >= 19.0.0` and `cmake-format` to be installed on your machine.
 `clang-format` can be installed using `brew` on macOS and default package manager on Linux.
 `cmake-format` can be installed using `pip`.
 The hook will also attempt to automatically use `doxygen` to verify that everything public in the codebase is covered by doc comments. If `doxygen` is not installed, the hook will raise a warning suggesting to install `doxygen` for future commits.
@@ -72,6 +72,9 @@ git push --force
 Clio uses `ccache` to speed up compilation. If you want to use it, please make sure it is installed on your machine.
 CMake will automatically detect it and use it if it is available.
 
+## Opening a pull request
+When a pull request is open CI will perform checks on the new code.
+Title of the pull request and squashed commit should follow [conventional commits specification](https://www.conventionalcommits.org/en/v1.0.0/).
 
 ## Fixing issues found during code review
 While your code is in review, it's possible that some changes will be requested by reviewer(s).
@@ -102,7 +105,7 @@ The button for that is near the bottom of the PR's page on GitHub.
 This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent.
 
 ## Formatting
-Code must conform to `clang-format` version 17, unless the result would be unreasonably difficult to read or maintain.
+Code must conform to `clang-format` version 19, unless the result would be unreasonably difficult to read or maintain.
 In most cases the pre-commit hook will take care of formatting and will fix any issues automatically.
 To manually format your code, use `clang-format -i <your changed files>` for C++ files and `cmake-format -i <your changed files>` for CMake files.
 

README.md

@@ -28,7 +28,6 @@ Below are some useful docs to learn more about Clio.
 **For Developers**:
 
 - [How to build Clio](./docs/build-clio.md)
-- [Metrics and static analysis](./docs/metrics-and-static-analysis.md)
 - [Coverage report](./docs/coverage-report.md)
 
 **For Operators**:
@@ -36,10 +35,12 @@ Below are some useful docs to learn more about Clio.
 - [How to configure Clio and rippled](./docs/configure-clio.md)
 - [How to run Clio](./docs/run-clio.md)
 - [Logging](./docs/logging.md)
+- [Troubleshooting guide](./docs/trouble_shooting.md)
 
 **General reference material:**
 
 - [API reference](https://xrpl.org/http-websocket-apis.html)
+- [Developer docs](https://xrplf.github.io/clio)
 - [Clio documentation](https://xrpl.org/the-clio-server.html#the-clio-server)
 
 ## 🆘 Help

benchmarks/CMakeLists.txt

@@ -12,5 +12,5 @@ target_sources(
 include(deps/gbench)
 
 target_include_directories(clio_benchmark PRIVATE .)
-target_link_libraries(clio_benchmark PUBLIC clio benchmark::benchmark_main)
+target_link_libraries(clio_benchmark PUBLIC clio_etl benchmark::benchmark_main)
 set_target_properties(clio_benchmark PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

benchmarks/util/async/ExecutionContextBenchmarks.cpp

@@ -31,7 +31,6 @@
 #include <cstdint>
 #include <latch>
 #include <optional>
-#include <stdexcept>
 #include <thread>
 #include <vector>
 
@@ -189,10 +188,10 @@ public:
 static auto
 generateData()
 {
-    constexpr auto TOTAL = 10'000;
+    constexpr auto kTOTAL = 10'000;
     std::vector<uint64_t> data;
-    data.reserve(TOTAL);
-    for (auto i = 0; i < TOTAL; ++i)
+    data.reserve(kTOTAL);
+    for (auto i = 0; i < kTOTAL; ++i)
         data.push_back(util::Random::uniform(1, 100'000'000));
 
     return data;
@@ -209,7 +208,7 @@ benchmarkThreads(benchmark::State& state)
 }
 
 template <typename CtxType>
-void
+static void
 benchmarkExecutionContextBatched(benchmark::State& state)
 {
     auto data = generateData();
@@ -220,7 +219,7 @@ benchmarkExecutionContextBatched(benchmark::State& state)
 }
 
 template <typename CtxType>
-void
+static void
 benchmarkAnyExecutionContextBatched(benchmark::State& state)
 {
     auto data = generateData();

cmake/Build.cpp.in

@@ -17,25 +17,26 @@
 */
 //==============================================================================
 
-#include "main/Build.hpp"
+#include "util/build/Build.hpp"
 
 #include <string>
 
-namespace Build {
-static constexpr char versionString[] = "@CLIO_VERSION@";
+namespace util::build {
+
+static constexpr char versionString[] = "@CLIO_VERSION@"; // NOLINT(readability-identifier-naming)
 
 std::string const&
 getClioVersionString()
 {
-    static std::string const value = versionString;
+    static std::string const value = versionString; // NOLINT(readability-identifier-naming)
     return value;
 }
 
 std::string const&
 getClioFullVersionString()
 {
-    static std::string const value = "clio-" + getClioVersionString();
+    static std::string const value = "clio-" + getClioVersionString(); // NOLINT(readability-identifier-naming)
     return value;
 }
 
-}  // namespace Build
+}  // namespace util::build

cmake/CheckCompiler.cmake

@@ -1,20 +1,20 @@
 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
-    message(FATAL_ERROR "Clang 14+ required for building clio")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16)
+    message(FATAL_ERROR "Clang 16+ required for building clio")
   endif ()
   set(is_clang TRUE)
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
-    message(FATAL_ERROR "AppleClang 14+ required for building clio")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15)
+    message(FATAL_ERROR "AppleClang 15+ required for building clio")
   endif ()
   set(is_appleclang TRUE)
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11)
-    message(FATAL_ERROR "GCC 11+ required for building clio")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12)
+    message(FATAL_ERROR "GCC 12+ required for building clio")
   endif ()
   set(is_gcc TRUE)
 else ()
-  message(FATAL_ERROR "Supported compilers: AppleClang 14+, Clang 14+, GCC 11+")
+  message(FATAL_ERROR "Supported compilers: AppleClang 15+, Clang 16+, GCC 12+")
 endif ()
 
 if (san)

cmake/ClangTidy.cmake

@@ -8,7 +8,7 @@ if (lint)
     endif ()
     message(STATUS "Using clang-tidy from CLIO_CLANG_TIDY_BIN")
   else ()
-    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-17" "clang-tidy" REQUIRED)
+    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-19" "clang-tidy" REQUIRED)
   endif ()
 
   if (NOT _CLANG_TIDY_BIN)

cmake/ClioVersion.cmake

@@ -22,7 +22,11 @@ endif ()
 
 if (NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name
                                                              # YYYYMMDDHMS-<branch>-<git-rev>
-  execute_process(COMMAND date +%Y%m%d%H%M%S OUTPUT_VARIABLE DATE OUTPUT_STRIP_TRAILING_WHITESPACE)
+  set(GIT_COMMAND show -s --date=format:%Y%m%d%H%M%S --format=%cd)
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE DATE
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
   set(CLIO_VERSION "${DATE}-${BRANCH}-${REV}")
   set(DOC_CLIO_VERSION "develop")
 else ()
@@ -31,7 +35,7 @@ else ()
     COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE CLIO_TAG_VERSION
     OUTPUT_STRIP_TRAILING_WHITESPACE
   )
-  set(CLIO_VERSION "${CLIO_TAG_VERSION}-${REV}")
+  set(CLIO_VERSION "${CLIO_TAG_VERSION}")
   set(DOC_CLIO_VERSION "${CLIO_TAG_VERSION}")
 endif ()
 
@@ -41,4 +45,4 @@ endif ()
 
 message(STATUS "Build version: ${CLIO_VERSION}")
 
-configure_file(${CMAKE_CURRENT_LIST_DIR}/Build.cpp.in ${CMAKE_CURRENT_LIST_DIR}/../src/main/impl/Build.cpp)
+configure_file(${CMAKE_CURRENT_LIST_DIR}/Build.cpp.in ${CMAKE_CURRENT_LIST_DIR}/../src/util/build/Build.cpp)

cmake/Settings.cmake

@@ -17,6 +17,13 @@ set(COMPILER_FLAGS
     -pedantic
     -Wpedantic
     -Wunused
+    # FIXME: The following bunch are needed for gcc12 atm.
+    -Wno-missing-requires
+    -Wno-restrict
+    -Wno-null-dereference
+    -Wno-maybe-uninitialized
+    -Wno-unknown-warning-option # and this to work with clang
+    # TODO: Address these and others in https://github.com/XRPLF/clio/issues/1273
 )
 
 # TODO: reenable when we change CI #884 if (is_gcc AND NOT lint) list(APPEND COMPILER_FLAGS -Wduplicated-branches
@@ -32,6 +39,34 @@ if (is_appleclang)
   list(APPEND COMPILER_FLAGS -Wreorder-init-list)
 endif ()
 
+if (san)
+  # When building with sanitizers some compilers will actually produce extra warnings/errors. We don't want this yet, at
+  # least not until we have fixed all runtime issues reported by the sanitizers. Once that is done we can start removing
+  # some of these and trying to fix it in our codebase. We can never remove all of below because most of them are
+  # reported from deep inside libraries like boost or libxrpl.
+  #
+  # TODO: Address in https://github.com/XRPLF/clio/issues/1885
+  list(
+    APPEND
+    COMPILER_FLAGS
+    -Wno-error=tsan # Disables treating TSAN warnings as errors
+    -Wno-tsan # Disables TSAN warnings (thread-safety analysis)
+    -Wno-uninitialized # Disables warnings about uninitialized variables (AddressSanitizer, UndefinedBehaviorSanitizer,
+                       # etc.)
+    -Wno-stringop-overflow # Disables warnings about potential string operation overflows (AddressSanitizer)
+    -Wno-unsafe-buffer-usage # Disables warnings about unsafe memory operations (AddressSanitizer)
+    -Wno-frame-larger-than # Disables warnings about stack frame size being too large (AddressSanitizer)
+    -Wno-unused-function # Disables warnings about unused functions (LeakSanitizer, memory-related issues)
+    -Wno-unused-but-set-variable # Disables warnings about unused variables (MemorySanitizer)
+    -Wno-thread-safety-analysis # Disables warnings related to thread safety usage (ThreadSanitizer)
+    -Wno-thread-safety # Disables warnings related to thread safety usage (ThreadSanitizer)
+    -Wno-sign-compare # Disables warnings about signed/unsigned comparison (UndefinedBehaviorSanitizer)
+    -Wno-nonnull # Disables warnings related to null pointer dereferencing (UndefinedBehaviorSanitizer)
+    -Wno-address # Disables warnings about address-related issues (UndefinedBehaviorSanitizer)
+    -Wno-array-bounds # Disables array bounds checks (UndefinedBehaviorSanitizer)
+  )
+endif ()
+
 # See https://github.com/cpp-best-practices/cppbestpractices/blob/master/02-Use_the_Tools_Available.md#gcc--clang for
 # the flags description
 

cmake/deps/libbacktrace.cmake

@@ -1,3 +1,11 @@
-target_compile_definitions(clio_options INTERFACE BOOST_STACKTRACE_LINK)
-target_compile_definitions(clio_options INTERFACE BOOST_STACKTRACE_USE_BACKTRACE)
-find_package(libbacktrace REQUIRED CONFIG)
+if ("${san}" STREQUAL "")
+  target_compile_definitions(clio_options INTERFACE BOOST_STACKTRACE_LINK)
+  target_compile_definitions(clio_options INTERFACE BOOST_STACKTRACE_USE_BACKTRACE)
+  find_package(libbacktrace REQUIRED CONFIG)
+else ()
+  # Some sanitizers (TSAN and ASAN for sure) can't be used with libbacktrace because they have their own backtracing
+  # capabilities and there are conflicts. In any case, this makes sure Clio code knows that backtrace is not available.
+  # See relevant conan profiles for sanitizers where we disable stacktrace in Boost explicitly.
+  target_compile_definitions(clio_options INTERFACE CLIO_WITHOUT_STACKTRACE)
+  message(STATUS "Sanitizer enabled, disabling stacktrace")
+endif ()

conanfile.py

@@ -1,7 +1,6 @@
 from conan import ConanFile
 from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
 
-
 class Clio(ConanFile):
     name = 'clio'
     license = 'ISC'
@@ -10,38 +9,46 @@ class Clio(ConanFile):
     description = 'Clio RPC server'
     settings = 'os', 'compiler', 'build_type', 'arch'
     options = {
-        'fPIC': [True, False],
+        'static': [True, False],              # static linkage
+        'fPIC': [True, False],                # unused?
         'verbose': [True, False],
-        'tests': [True, False],     # build unit tests; create `clio_tests` binary
-        'benchmark': [True, False], # build benchmarks; create `clio_benchmarks` binary
-        'docs': [True, False],      # doxygen API docs; create custom target 'docs'
-        'packaging': [True, False], # create distribution packages
-        'coverage': [True, False],  # build for test coverage report; create custom target `clio_tests-ccov`
-        'lint': [True, False],      # run clang-tidy checks during compilation
+        'tests': [True, False],               # build unit tests; create `clio_tests` binary
+        'integration_tests': [True, False],   # build integration tests; create `clio_integration_tests` binary
+        'benchmark': [True, False],           # build benchmarks; create `clio_benchmarks` binary
+        'docs': [True, False],                # doxygen API docs; create custom target 'docs'
+        'packaging': [True, False],           # create distribution packages
+        'coverage': [True, False],            # build for test coverage report; create custom target `clio_tests-ccov`
+        'lint': [True, False],                # run clang-tidy checks during compilation
+        'snapshot': [True, False],            # build export/import snapshot tool
     }
 
     requires = [
         'boost/1.82.0',
         'cassandra-cpp-driver/2.17.0',
         'fmt/10.1.1',
-        'protobuf/3.21.12',
+        'protobuf/3.21.9',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.2.0-b1',
+        'xrpl/2.4.0-rc4',
+        'zlib/1.3.1',
         'libbacktrace/cci.20210118'
     ]
 
     default_options = {
+        'static': False,
         'fPIC': True,
         'verbose': False,
         'tests': False,
+        'integration_tests': False,
         'benchmark': False,
         'packaging': False,
         'coverage': False,
         'lint': False,
         'docs': False,
+        'snapshot': False,
         
         'xrpl/*:tests': False,
+        'xrpl/*:rocksdb': False,
         'cassandra-cpp-driver/*:shared': False,
         'date/*:header_only': True,
         'grpc/*:shared': False,
@@ -60,7 +67,7 @@ class Clio(ConanFile):
     )
 
     def requirements(self):
-        if self.options.tests:
+        if self.options.tests or self.options.integration_tests:
             self.requires('gtest/1.14.0')
         if self.options.benchmark:
             self.requires('benchmark/1.8.3')
@@ -79,12 +86,15 @@ class Clio(ConanFile):
     def generate(self):
         tc = CMakeToolchain(self)
         tc.variables['verbose'] = self.options.verbose
+        tc.variables['static'] = self.options.static
         tc.variables['tests'] = self.options.tests
+        tc.variables['integration_tests'] = self.options.integration_tests
         tc.variables['coverage'] = self.options.coverage
         tc.variables['lint'] = self.options.lint
         tc.variables['docs'] = self.options.docs
         tc.variables['packaging'] = self.options.packaging
         tc.variables['benchmark'] = self.options.benchmark
+        tc.variables['snapshot'] = self.options.snapshot
         tc.generate()
 
     def build(self):

docker/centos/Dockerfile

@@ -1,49 +0,0 @@
-# FROM centos:7 as deps
-FROM centos:7 as build
-
-ENV CLIO_DIR=/opt/clio/
-# ENV OPENSSL_DIR=/opt/openssl
-
-RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl
-RUN yum install -y devtoolset-11
-ENV version=3.16
-ENV build=3
-# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
-COPY docker/shared/install_cmake.sh /install_cmake.sh
-RUN /install_cmake.sh 3.16.3 /usr/local
-RUN source /opt/rh/devtoolset-11/enable
-WORKDIR /tmp
-# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR
-COPY docker/centos/build_git_centos7.sh build_git_centos7.sh
-
-RUN ./build_git_centos7.sh
-RUN git clone https://github.com/openssl/openssl
-WORKDIR /tmp/openssl
-RUN git checkout OpenSSL_1_1_1q
-#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic
-RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \
-    make -j $(nproc) && \
-    make install_sw
-WORKDIR /tmp
-# FROM centos:7 as build
-
-RUN git clone https://github.com/xrplf/clio.git
-COPY docker/shared/build_boost.sh build_boost.sh
-ENV OPENSSL_ROOT=/opt/local/openssl
-ENV BOOST_ROOT=/boost
-RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0
-RUN yum install -y bison flex
-RUN yum install -y rpmdevtools rpmlint
-RUN source /opt/rh/devtoolset-11/enable && cd /tmp/clio && \
-    cmake -B build -DBUILD_TESTS=1 && \
-    cmake --build build --parallel $(nproc)
-RUN mkdir output
-RUN strip clio/build/clio_server && strip clio/build/clio_tests
-RUN cp clio/build/clio_tests output/ && cp clio/build/clio_server output/
-RUN cp clio/docs/examples/config/example-config.json output/example-config.json
-
-FROM centos:7
-COPY --from=build /tmp/output /clio
-RUN mkdir -p /opt/clio/etc && mv /clio/docs/examples/config/example-config.json /opt/clio/etc/config.json
-
-CMD ["/clio/clio_server", "/opt/clio/etc/config.json"]

docker/centos/build_git_centos7.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-GIT_VERSION="2.37.1"
-curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz
-tar zxvf git-${GIT_VERSION}.tar.gz
-cd git-${GIT_VERSION}
-
-yum install -y centos-release-scl epel-release
-yum update -y
-yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel  libcurl-devel
-
-source /opt/rh/devtoolset-11/enable
-make configure
-./configure
-make git -j$(nproc)
-make install git
-git  --version | cut -d ' ' -f3

docker/centos/install_cmake.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-CMAKE_VERSION=${1:-"3.16.3"}
-cd /tmp
-URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
-curl -OJLs $URL
-tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
-ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake

docker/ci/README.md

@@ -0,0 +1,16 @@
+# CI image for XRPLF/clio
+
+This image contains an environment to build [Clio](https://github.com/XRPLF/clio), check code and documentation.
+It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but can also be used to compile Clio locally.
+
+The image is based on Ubuntu 20.04 and contains:
+- clang 16.0.6
+- gcc 12.3
+- doxygen 1.12
+- gh 2.40
+- ccache 4.10.2
+- conan 1.62
+- and some other useful tools
+
+Conan is set up to build Clio without any additional steps. There are two preset conan profiles: `clang` and `gcc` to use corresponding compiler. By default conan is setup to use `gcc`.
+Sanitizer builds for `ASAN`, `TSAN` and `UBSAN` are enabled via conan profiles for each of the supported compilers. These can be selected using the following pattern (all lowercase): `[compiler].[sanitizer]` (e.g. `--profile gcc.tsan`). 

docker/ci/conan/clang.asan

@@ -0,0 +1,9 @@
+include(clang)
+
+[options]
+boost:extra_b2_flags="cxxflags=\"-fsanitize=address\" linkflags=\"-fsanitize=address\""
+boost:without_stacktrace=True
+[env]
+CFLAGS="-fsanitize=address"
+CXXFLAGS="-fsanitize=address"
+LDFLAGS="-fsanitize=address"

docker/ci/conan/clang.tsan

@@ -0,0 +1,9 @@
+include(clang)
+
+[options]
+boost:extra_b2_flags="cxxflags=\"-fsanitize=thread\" linkflags=\"-fsanitize=thread\""
+boost:without_stacktrace=True
+[env]
+CFLAGS="-fsanitize=thread"
+CXXFLAGS="-fsanitize=thread"
+LDFLAGS="-fsanitize=thread"

docker/ci/conan/clang.ubsan

@@ -0,0 +1,9 @@
+include(clang)
+
+[options]
+boost:extra_b2_flags="cxxflags=\"-fsanitize=undefined\" linkflags=\"-fsanitize=undefined\""
+boost:without_stacktrace=True
+[env]
+CFLAGS="-fsanitize=undefined"
+CXXFLAGS="-fsanitize=undefined"
+LDFLAGS="-fsanitize=undefined"

docker/ci/conan/gcc.asan

@@ -0,0 +1,9 @@
+include(gcc)
+
+[options]
+boost:extra_b2_flags="cxxflags=\"-fsanitize=address\" linkflags=\"-fsanitize=address\""
+boost:without_stacktrace=True
+[env]
+CFLAGS="-fsanitize=address"
+CXXFLAGS="-fsanitize=address"
+LDFLAGS="-fsanitize=address"

docker/ci/conan/gcc.tsan

@@ -0,0 +1,9 @@
+include(gcc)
+
+[options]
+boost:extra_b2_flags="cxxflags=\"-fsanitize=thread\" linkflags=\"-fsanitize=thread\""
+boost:without_stacktrace=True
+[env]
+CFLAGS="-fsanitize=thread"
+CXXFLAGS="-fsanitize=thread"
+LDFLAGS="-fsanitize=thread"

docker/ci/conan/gcc.ubsan

@@ -0,0 +1,9 @@
+include(gcc)
+
+[options]
+boost:extra_b2_flags="cxxflags=\"-fsanitize=undefined\" linkflags=\"-fsanitize=undefined\""
+boost:without_stacktrace=True
+[env]
+CFLAGS="-fsanitize=undefined"
+CXXFLAGS="-fsanitize=undefined"
+LDFLAGS="-fsanitize=undefined"

docker/ci/dockerfile

@@ -1,42 +1,46 @@
-FROM ubuntu:focal
+FROM rippleci/clio_clang:16
 ARG DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
 
 SHELL ["/bin/bash", "-c"]
 USER root
-WORKDIR /root/
+WORKDIR /root
 
-ENV GCC_VERSION=11 \
-    CCACHE_VERSION=4.8.3 \
-    LLVM_TOOLS_VERSION=17 \
+ENV CCACHE_VERSION=4.10.2 \
+    LLVM_TOOLS_VERSION=19 \
     GH_VERSION=2.40.0 \
-    DOXYGEN_VERSION=1.10.0
+    DOXYGEN_VERSION=1.12.0
  
 # Add repositories
 RUN apt-get -qq update \
     && apt-get -qq install -y --no-install-recommends --no-install-suggests gnupg wget curl software-properties-common \
-    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
-    && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - \
-    && apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' \
     && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM_TOOLS_VERSION} main" >> /etc/apt/sources.list \
     && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 
 # Install packages
 RUN apt update -qq \
-    && apt install -y --no-install-recommends --no-install-suggests cmake python3 python3-pip sudo git \
-    ninja-build make pkg-config libzstd-dev libzstd1 g++-${GCC_VERSION} flex bison jq graphviz \
+    && apt install -y --no-install-recommends --no-install-suggests python3 python3-pip git git-lfs make ninja-build flex bison jq graphviz \
     clang-format-${LLVM_TOOLS_VERSION} clang-tidy-${LLVM_TOOLS_VERSION} clang-tools-${LLVM_TOOLS_VERSION} \
-    && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-${GCC_VERSION} 100 \
     && update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-${LLVM_TOOLS_VERSION} 100 \
-    && apt-get clean && apt remove -y software-properties-common \
-    && pip3 install  -q --upgrade --no-cache-dir pip \
-    && pip3 install  -q --no-cache-dir conan==1.62 gcovr cmake-format
+    && pip3 install -q --upgrade --no-cache-dir pip && pip3 install -q --no-cache-dir conan==1.62 gcovr cmake cmake-format \
+    && apt-get clean && apt remove -y software-properties-common
+
+# Install gcc-12 and make ldconfig aware of the new libstdc++ location (for gcc)
+# Note: Clang is using libc++ instead
+COPY --from=rippleci/clio_gcc:12.3.0 /gcc12.deb /
+RUN apt update && apt-get install -y binutils libc6-dev \
+    && dpkg -i /gcc12.deb \
+    && rm -rf /gcc12.deb \
+    && ldconfig
+
+# Rewire to use gcc-12 as default compiler
+RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 100 \
+    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
+    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100
 
 WORKDIR /tmp
 
@@ -46,34 +50,58 @@ RUN wget "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/
     && cd "ccache-${CCACHE_VERSION}" \
     && mkdir build && cd build \
     && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install
+    && cmake --build . --target install \
+    && rm -rf /tmp/* /var/tmp/*
 
-# Install doxygen from sounce
+# Install doxygen from source
 RUN wget "https://github.com/doxygen/doxygen/releases/download/Release_${DOXYGEN_VERSION//./_}/doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
     && tar xf "doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
     && cd "doxygen-${DOXYGEN_VERSION}" \
     && mkdir build && cd build \
     && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install
+    && cmake --build . --target install \
+    && rm -rf /tmp/* /var/tmp/*
 
 # Install gh
 RUN wget https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
     && tar xf gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
-    && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh
+    && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh \
+    && rm -rf /tmp/* /var/tmp/*
 
-# Clean up
-RUN rm -rf  /tmp/* /var/tmp/*
-
-WORKDIR /root/
+WORKDIR /root
 # Using root by default is not very secure but github checkout action doesn't work with any other user
 # https://github.com/actions/checkout/issues/956
 # And Github Actions doc recommends using root
 # https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
 
 # Setup conan
-RUN conan profile new default --detect \
-    && conan profile update settings.compiler.cppstd=20 default \
-    && conan profile update settings.compiler.libcxx=libstdc++11 default \
-    && conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+RUN conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
 
+# Note: intentionally leaving cppstd=20
+RUN conan profile new gcc --detect \
+    && conan profile update settings.compiler=gcc gcc \
+    && conan profile update settings.compiler.version=12 gcc \
+    && conan profile update settings.compiler.cppstd=20 gcc \
+    && conan profile update settings.compiler.libcxx=libstdc++11 gcc \
+    && conan profile update env.CC=/usr/bin/gcc-12 gcc \
+    && conan profile update env.CXX=/usr/bin/g++-12 gcc \
+    && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/gcc-12\", \"cpp\": \"/usr/bin/g++-12\"}" gcc
 
+RUN conan profile new clang --detect \
+    && conan profile update settings.compiler=clang clang \
+    && conan profile update settings.compiler.version=16 clang \
+    && conan profile update settings.compiler.cppstd=20 clang \
+    && conan profile update settings.compiler.libcxx=libc++ clang \
+    && conan profile update env.CC=/usr/bin/clang-16 clang \
+    && conan profile update env.CXX=/usr/bin/clang++-16 clang \
+    && conan profile update env.CXXFLAGS="-DBOOST_ASIO_DISABLE_CONCEPTS" clang \
+    && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/clang-16\", \"cpp\": \"/usr/bin/clang++-16\"}" clang 
+
+RUN echo "include(gcc)" >> .conan/profiles/default
+
+COPY conan/gcc.asan /root/.conan/profiles
+COPY conan/gcc.tsan /root/.conan/profiles
+COPY conan/gcc.ubsan /root/.conan/profiles
+COPY conan/clang.asan /root/.conan/profiles
+COPY conan/clang.tsan /root/.conan/profiles
+COPY conan/clang.ubsan /root/.conan/profiles

docker/clio/README.md

@@ -0,0 +1,23 @@
+# Clio official docker image
+
+[Clio](https://github.com/XRPLF/clio) is an XRP Ledger API server optimized for RPC calls over WebSocket or JSON-RPC.
+It stores validated historical ledger and transaction data in a space efficient format.
+
+This image contains `clio_server` binary allowing users to run Clio easily.
+
+## Clio configuration file
+
+Please note that while Clio requires a configuration file, this image doesn't include any default config.
+Your configuration file should be mounted under the path `/opt/clio/etc/config.json`.
+Clio repository provides an [example](https://github.com/XRPLF/clio/blob/develop/docs/examples/config/example-config.json) of the configuration file.
+
+Config file recommendations:
+- Set `log_to_console` to `false` if you want to avoid logs being written to `stdout`.
+- Set `log_directory` to `/opt/clio/log` to store logs in a volume.
+
+## Usage
+
+The following command can be used to run Clio in docker (assuming server's port is `51233` in your config):
+```bash
+docker run -d -v <path to your config.json>:/opt/clio/etc/config.json -v <path to store logs>:/opt/clio/log -p 51233:51233 rippleci/clio
+```

docker/clio/dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:22.04
+
+COPY ./clio_server /opt/clio/bin/clio_server
+
+RUN ln -s /opt/clio/bin/clio_server /usr/local/bin/clio_server && \
+    mkdir -p /opt/clio/etc/ && \
+    mkdir -p /opt/clio/log/ && \
+    groupadd -g 10001 clio && \
+    useradd -u 10000 -g 10001 -s /bin/bash clio && \
+    chown clio:clio /opt/clio/log && \
+    apt update && \
+    apt install -y libatomic1
+
+USER clio
+ENTRYPOINT ["/opt/clio/bin/clio_server"]
+CMD ["--conf", "/opt/clio/etc/config.json"]

docker/clio_docker/centos/build_boost.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -exu
-
-#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static
-# it's either those or link=static that halves the failures. probably link=static
-BOOST_VERSION=$1
-BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _)
-echo "BOOST_VERSION: ${BOOST_VERSION}"
-echo "BOOST_VERSION_: ${BOOST_VERSION_}"
-curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz"
-tar zxf "boost_${BOOST_VERSION_}.tar.gz"
-cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc)
-mkdir -p /boost && mv boost /boost && mv stage /boost

docker/clio_docker/centos/build_git_centos7.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-GIT_VERSION="2.37.1"
-curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz
-tar zxvf git-${GIT_VERSION}.tar.gz
-cd git-${GIT_VERSION}
-
-yum install -y centos-release-scl epel-release
-yum update -y
-yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel  libcurl-devel
-
-source /opt/rh/devtoolset-11/enable
-make configure
-./configure
-make git -j$(nproc)
-make install git
-git  --version | cut -d ' ' -f3

docker/clio_docker/centos/dockerfile

@@ -1,34 +0,0 @@
-FROM centos:7
-
-ENV CLIO_DIR=/opt/clio/
-# ENV OPENSSL_DIR=/opt/openssl
-
-RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl
-RUN yum install -y devtoolset-11
-ENV version=3.16
-ENV build=3
-# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
-COPY install_cmake.sh /install_cmake.sh
-RUN /install_cmake.sh 3.16.3 /usr/local
-RUN source /opt/rh/devtoolset-11/enable
-WORKDIR /tmp
-# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR
-COPY build_git_centos7.sh build_git_centos7.sh
-
-RUN ./build_git_centos7.sh
-RUN git clone https://github.com/openssl/openssl
-WORKDIR /tmp/openssl
-RUN git checkout OpenSSL_1_1_1q
-#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic
-RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \
-    make -j $(nproc) && \
-    make install_sw
-WORKDIR /tmp
-RUN git clone https://github.com/xrplf/clio.git
-COPY build_boost.sh build_boost.sh
-ENV OPENSSL_ROOT=/opt/local/openssl
-ENV BOOST_ROOT=/boost
-RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0
-RUN yum install -y bison flex
-RUN source /opt/rh/devtoolset-11/enable && \
-    cd /tmp/clio && cmake -B build -Dtests=0 -Dlocal_libarchive=1 -Dunity=0 -DBUILD_TESTS=0 && cmake --build build --parallel $(nproc)

docker/clio_docker/centos/install_cmake.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-CMAKE_VERSION=${1:-"3.16.3"}
-cd /tmp
-URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
-curl -OJLs $URL
-tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
-ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake

docker/compilers/clang-16/dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:focal 
+ARG DEBIAN_FRONTEND=noninteractive
+ARG TARGETARCH
+
+SHELL ["/bin/bash", "-c"]
+USER root
+WORKDIR /root
+
+ENV CLANG_VERSION=16
+
+RUN apt update -qq \
+    && apt install -qq -y --no-install-recommends --no-install-suggests \
+    wget software-properties-common gnupg
+
+RUN wget https://apt.llvm.org/llvm.sh \
+    && chmod +x llvm.sh \
+    && ./llvm.sh ${CLANG_VERSION} \
+    && rm -rf llvm.sh \
+    && apt-get install -y libc++-16-dev libc++abi-16-dev

docker/compilers/gcc-12/control.m4

@@ -0,0 +1,6 @@
+Package: gcc-12-ubuntu-UBUNTUVERSION
+Version: VERSION
+Architecture: TARGETARCH
+Maintainer: Alex Kremer <akremer@ripple.com>
+Description: Gcc VERSION build for ubuntu UBUNTUVERSION
+Depends: binutils, libc6-dev

docker/compilers/gcc-12/dockerfile

@@ -0,0 +1,74 @@
+FROM ubuntu:focal as build
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG TARGETARCH
+ARG UBUNTU_VERSION=20.04
+ARG GCC_VERSION=12.3.0
+ARG BUILD_VERSION=1
+
+RUN apt update && apt install -y wget build-essential file flex libz-dev libzstd-dev
+RUN wget https://gcc.gnu.org/pub/gcc/releases/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz \
+    && tar xf gcc-$GCC_VERSION.tar.gz \
+    && cd /gcc-$GCC_VERSION && ./contrib/download_prerequisites
+
+RUN mkdir /${TARGETARCH}-gcc-12
+WORKDIR /${TARGETARCH}-gcc-12
+RUN /gcc-$GCC_VERSION/configure \
+    --with-pkgversion="clio-build-$BUILD_VERSION https://github.com/XRPLF/clio" \
+    --enable-languages=c,c++ \
+    --prefix=/usr \
+    --with-gcc-major-version-only \
+    --program-suffix=-12 \
+    --enable-shared \
+    --enable-linker-build-id \
+    --libexecdir=/usr/lib \
+    --without-included-gettext \
+    --enable-threads=posix \
+    --libdir=/usr/lib \
+    --disable-nls \
+    --enable-clocale=gnu \
+    --enable-libstdcxx-backtrace=yes \
+    --enable-libstdcxx-debug \
+    --enable-libstdcxx-time=yes \
+    --with-default-libstdcxx-abi=new \
+    --enable-gnu-unique-object \
+    --disable-vtable-verify \
+    --enable-plugin \
+    --enable-default-pie \
+    --with-system-zlib \
+    --enable-libphobos-checking=release \
+    --with-target-system-zlib=auto \
+    --disable-werror \
+    --enable-cet \
+    --disable-multilib \
+    --without-cuda-driver \
+    --enable-checking=release \
+    && make -j`nproc` \
+    && make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION \
+    && mkdir -p /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64 \
+    && mv /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/lib64/libstdc++.so.6.0.30-gdb.py /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64/libstdc++.so.6.0.30-gdb.py
+
+# Generate deb
+WORKDIR /
+COPY control.m4 /
+COPY ld.so.conf /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/etc/ld.so.conf.d/1-gcc-12.conf
+
+RUN mkdir /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/DEBIAN \
+    && m4 -P -DUBUNTU_VERSION=$UBUNTU_VERSION -DVERSION=$GCC_VERSION-$BUILD_VERSION -DTARGETARCH=$TARGETARCH control.m4 > /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/DEBIAN/control \
+    && dpkg-deb --build --root-owner-group /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION /gcc12.deb
+
+# Create final image
+FROM ubuntu:focal as gcc
+COPY --from=build /gcc12.deb /
+
+# Make gcc-12 available but also leave gcc12.deb for others to copy if needed
+RUN apt update && apt-get install -y binutils libc6-dev \
+    && dpkg -i /gcc12.deb 
+
+RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 100 \
+    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
+    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100

docker/compilers/gcc-12/ld.so.conf

@@ -0,0 +1,2 @@
+# Path to the directory containing libstdc++.so.6
+/usr/lib64

docker/develop/compose.yaml

@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   clio_develop:
     image: rippleci/clio_ci:latest

docker/shared/build_boost.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -exu
-
-#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static
-# it's either those or link=static that halves the failures. probably link=static
-BOOST_VERSION=$1
-BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _)
-echo "BOOST_VERSION: ${BOOST_VERSION}"
-echo "BOOST_VERSION_: ${BOOST_VERSION_}"
-curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz"
-tar zxf "boost_${BOOST_VERSION_}.tar.gz"
-cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc)
-mkdir -p /boost && mv boost /boost && mv stage /boost

docker/shared/install_cmake.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-CMAKE_VERSION=${1:-"3.16.3"}
-cd /tmp
-URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
-curl -OJLs $URL
-tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
-ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake

docker/shared/install_openssl.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-set -e

docker/ubuntu/Dockerfile

@@ -1,24 +0,0 @@
-FROM ubuntu:20.04 AS boost
-
-RUN apt-get update && apt-get install -y build-essential
-ARG BOOST_VERSION_=1_75_0
-ARG BOOST_VERSION=1.75.0
-COPY docker/shared/build_boost.sh .
-RUN apt install -y curl
-RUN ./build_boost.sh ${BOOST_VERSION}
-ENV BOOST_ROOT=/boost
-
-FROM ubuntu:20.04 AS build
-ENV BOOST_ROOT=/boost
-COPY --from=boost /boost /boost
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install --no-install-recommends -y build-essential software-properties-common pkg-config libssl-dev wget curl gpg git zlib1g-dev bison flex autoconf lsb-release
-RUN apt install -y gpg-agent
-RUN wget https://apt.llvm.org/llvm.sh
-RUN chmod +x llvm.sh && ./llvm.sh 14 && ./llvm.sh 15
-# COPY . /clio
-## Install cmake
-ARG CMAKE_VERSION=3.16.3
-COPY docker/shared/install_cmake.sh .
-RUN ./install_cmake.sh ${CMAKE_VERSION}
-ENV PATH="/opt/local/cmake/bin:$PATH"

docs/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.16.3)
+cmake_minimum_required(VERSION 3.20)
 project(docs)
 
 include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/ClioVersion.cmake)

docs/build-clio.md

@@ -6,19 +6,22 @@ Clio is built with [CMake](https://cmake.org/) and uses [Conan](https://conan.io
 
 - [Python 3.7](https://www.python.org/downloads/)
 - [Conan 1.55](https://conan.io/downloads.html)
-- [CMake 3.16](https://cmake.org/download/)
+- [CMake 3.20](https://cmake.org/download/)
 - [**Optional**] [GCovr](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html): needed for code coverage generation
 - [**Optional**] [CCache](https://ccache.dev/): speeds up compilation if you are going to compile Clio often
 
 | Compiler    | Version |
 |-------------|---------|
-| GCC         | 11      |
-| Clang       | 14      |
-| Apple Clang | 14.0.3  |
+| GCC         | 12.3    |
+| Clang       | 16      |
+| Apple Clang | 15      |
 
 ### Conan Configuration
 
-Clio does not require anything but default settings in your (`~/.conan/profiles/default`) Conan profile. It's best to have no extra flags specified.
+Clio does not require anything other than `compiler.cppstd=20` in your (`~/.conan/profiles/default`) Conan profile.
+
+> [!NOTE]
+> Although Clio is built using C++23, it's required to set `compiler.cppstd=20` for the time being as some of Clio's dependencies are not yet capable of building under C++23.
 
 > Mac example:
 
@@ -29,10 +32,12 @@ os_build=Macos
 arch=armv8
 arch_build=armv8
 compiler=apple-clang
-compiler.version=14
+compiler.version=15
 compiler.libcxx=libc++
 build_type=Release
 compiler.cppstd=20
+[conf]
+tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]
 ```
 
 > Linux example:
@@ -44,7 +49,7 @@ os_build=Linux
 arch=x86_64
 arch_build=x86_64
 compiler=gcc
-compiler.version=11
+compiler.version=12
 compiler.libcxx=libstdc++11
 build_type=Release
 compiler.cppstd=20
@@ -88,6 +93,35 @@ If successful, `conan install` will find the required packages and `cmake` will
 > [!TIP]
 > To generate a Code Coverage report, include `-o coverage=True` in the `conan install` command above, along with `-o tests=True` to enable tests. After running the `cmake` commands, execute `make clio_tests-ccov`. The coverage report will be found at `clio_tests-llvm-cov/index.html`.
 
+> [!NOTE]
+> If you've built Clio before and the build is now failing, it's likely due to updated dependencies. Try deleting the build folder and then rerunning the Conan and CMake commands mentioned above.
+
+### Generating API docs for Clio
+
+The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen on your system.
+
+To generate the API docs:
+
+1. First, include `-o docs=True` in the conan install command. For example:
+
+    ```sh
+    mkdir build && cd build
+    conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False -o docs=True
+    ```
+
+2. Once that has completed successfully, run the `cmake` command and add the `--target docs` option:
+
+   ```sh
+   cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
+   cmake --build . --parallel 8 --target docs
+   ```
+
+3. Go to `build/docs/html` to view the generated files.
+
+    Open the `index.html` file in your browser to see the documentation pages.
+
+    ![API index page](./img/doxygen-docs-output.png "API index page")
+
 ## Building Clio with Docker
 
 It is also possible to build Clio using [Docker](https://www.docker.com/) if you don't want to install all the dependencies on your machine.
@@ -107,3 +141,60 @@ If you wish to develop against a `rippled` instance running in standalone mode t
 
 1. Advance the `rippled` ledger to at least ledger 256.
 2. Wait 10 minutes before first starting Clio against this standalone node.
+
+## Building with a Custom `libxrpl`
+
+Sometimes, during development, you need to build against a custom version of `libxrpl`. (For example, you may be developing compatibility for a proposed amendment that is not yet merged to the main `rippled` codebase.) To build Clio with compatibility for a custom fork or branch of `rippled`, follow these steps:
+
+1. First, pull/clone the appropriate `rippled` fork and switch to the branch you want to build. For example, the following example uses an in-development build with [XLS-33d Multi-Purpose Tokens](https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens):
+
+    ```sh
+    git clone https://github.com/shawnxie999/rippled/
+    cd rippled
+    git switch mpt-1.1
+    ```
+
+2. Export a custom package to your local Conan store using a user/channel:
+
+    ```sh
+    conan export . my/feature
+    ```
+
+3. Patch your local Clio build to use the right package.
+
+    Edit `conanfile.py` (from the Clio repository root). Replace the `xrpl` requirement with the custom package version from the previous step. This must also include the current version number from your `rippled` branch. For example:
+
+    ```py
+    # ... (excerpt from conanfile.py)
+        requires = [
+        'boost/1.82.0',
+        'cassandra-cpp-driver/2.17.0',
+        'fmt/10.1.1',
+        'protobuf/3.21.9',
+        'grpc/1.50.1',
+        'openssl/1.1.1u',
+        'xrpl/2.3.0-b1@my/feature', # Update this line
+        'libbacktrace/cci.20210118'
+    ]
+    ```
+
+4. Build Clio as you would have before.
+
+    See [Building Clio](#building-clio) for details.
+
+## Using `clang-tidy` for static analysis
+
+The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 19.0.
+
+Clang-tidy can be run by Cmake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command:
+
+```sh
+conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=True
+```
+
+By default Cmake will try to find `clang-tidy` automatically in your system.
+To force Cmake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example:
+
+```sh
+export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@19/bin/clang-tidy
+```

docs/config-description.md

@@ -0,0 +1,452 @@
+# Clio Config Description
+This file lists all Clio Configuration definitions in detail.
+
+## Configuration Details
+
+### Key: database.type
+- **Required**: True
+- **Type**: string
+- **Default value**: cassandra
+- **Constraints**: The value must be one of the following: `cassandra`
+ -   **Description**: Type of database to use. We currently support Cassandra and Scylladb. We default to Scylladb.
+### Key: database.cassandra.contact_points
+- **Required**: True
+- **Type**: string
+- **Default value**: localhost
+- **Constraints**: None
+ -   **Description**: A list of IP addresses or hostnames of the initial nodes (Cassandra/Scylladb cluster nodes) that the client will connect to when establishing a connection with the database. If you're running locally, it should be 'localhost' or 127.0.0.1
+### Key: database.cassandra.secure_connect_bundle
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: Configuration file that contains the necessary security credentials and connection details for securely connecting to a Cassandra database cluster.
+### Key: database.cassandra.port
+- **Required**: False
+- **Type**: int
+- **Default value**: None
+- **Constraints**: The minimum value is `1`. The maximum value is `65535
+ -   **Description**: Port number to connect to the database.
+### Key: database.cassandra.keyspace
+- **Required**: True
+- **Type**: string
+- **Default value**: clio
+- **Constraints**: None
+ -   **Description**: Keyspace to use for the database.
+### Key: database.cassandra.replication_factor
+- **Required**: True
+- **Type**: int
+- **Default value**: 3
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Number of replicated nodes for Scylladb. Visit this link for more details : https://university.scylladb.com/courses/scylla-essentials-overview/lessons/high-availability/topic/fault-tolerance-replication-factor/ 
+### Key: database.cassandra.table_prefix
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: Prefix for Database table names.
+### Key: database.cassandra.max_write_requests_outstanding
+- **Required**: True
+- **Type**: int
+- **Default value**: 10000
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Maximum number of outstanding write requests. Write requests are api calls that write to database 
+### Key: database.cassandra.max_read_requests_outstanding
+- **Required**: True
+- **Type**: int
+- **Default value**: 100000
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Maximum number of outstanding read requests, which reads from database
+### Key: database.cassandra.threads
+- **Required**: True
+- **Type**: int
+- **Default value**: The number of available CPU cores.
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Number of threads that will be used for database operations.
+### Key: database.cassandra.core_connections_per_host
+- **Required**: True
+- **Type**: int
+- **Default value**: 1
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Number of core connections per host for Cassandra.
+### Key: database.cassandra.queue_size_io
+- **Required**: False
+- **Type**: int
+- **Default value**: None
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Queue size for I/O operations in Cassandra.
+### Key: database.cassandra.write_batch_size
+- **Required**: True
+- **Type**: int
+- **Default value**: 20
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Batch size for write operations in Cassandra.
+### Key: database.cassandra.connect_timeout
+- **Required**: False
+- **Type**: int
+- **Default value**: None
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: The maximum amount of time in seconds the system will wait for a connection to be successfully established with the database.
+### Key: database.cassandra.request_timeout
+- **Required**: False
+- **Type**: int
+- **Default value**: None
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: The maximum amount of time in seconds the system will wait for a request to be fetched from database.
+### Key: database.cassandra.username
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: The username used for authenticating with the database.
+### Key: database.cassandra.password
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: The password used for authenticating with the database.
+### Key: database.cassandra.certfile
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: The path to the SSL/TLS certificate file used to establish a secure connection between the client and the Cassandra database.
+### Key: allow_no_etl
+- **Required**: True
+- **Type**: boolean
+- **Default value**: True
+- **Constraints**: None
+ -   **Description**: If True, no ETL nodes will run with Clio.
+### Key: etl_sources.[].ip
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: The value must be a valid IP address
+ -   **Description**: IP address of the ETL source.
+### Key: etl_sources.[].ws_port
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: The minimum value is `1`. The maximum value is `65535
+ -   **Description**: WebSocket port of the ETL source.
+### Key: etl_sources.[].grpc_port
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: The minimum value is `1`. The maximum value is `65535
+ -   **Description**: gRPC port of the ETL source.
+### Key: forwarding.cache_timeout
+- **Required**: True
+- **Type**: double
+- **Default value**: 0
+- **Constraints**: The value must be a positive double number
+ -   **Description**: Timeout duration for the forwarding cache used in Rippled communication.
+### Key: forwarding.request_timeout
+- **Required**: True
+- **Type**: double
+- **Default value**: 10
+- **Constraints**: The value must be a positive double number
+ -   **Description**: Timeout duration for the forwarding request used in Rippled communication.
+### Key: rpc.cache_timeout
+- **Required**: True
+- **Type**: double
+- **Default value**: 0
+- **Constraints**: The value must be a positive double number
+ -   **Description**: Timeout duration for RPC requests.
+### Key: num_markers
+- **Required**: False
+- **Type**: int
+- **Default value**: None
+- **Constraints**: The minimum value is `1`. The maximum value is `256`
+ -   **Description**: The number of markers is the number of coroutines to download the initial ledger
+### Key: dos_guard.whitelist.[]
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: List of IP addresses to whitelist for DOS protection.
+### Key: dos_guard.max_fetches
+- **Required**: True
+- **Type**: int
+- **Default value**: 1000000
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Maximum number of fetch operations allowed by DOS guard.
+### Key: dos_guard.max_connections
+- **Required**: True
+- **Type**: int
+- **Default value**: 20
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Maximum number of concurrent connections allowed by DOS guard.
+### Key: dos_guard.max_requests
+- **Required**: True
+- **Type**: int
+- **Default value**: 20
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Maximum number of requests allowed by DOS guard.
+### Key: dos_guard.sweep_interval
+- **Required**: True
+- **Type**: double
+- **Default value**: 1
+- **Constraints**: The value must be a positive double number
+ -   **Description**: Interval in seconds for DOS guard to sweep/clear its state.
+### Key: workers
+- **Required**: True
+- **Type**: int
+- **Default value**: The number of available CPU cores.
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Number of threads to process RPC requests.
+### Key: server.ip
+- **Required**: True
+- **Type**: string
+- **Default value**: None
+- **Constraints**: The value must be a valid IP address
+ -   **Description**: IP address of the Clio HTTP server.
+### Key: server.port
+- **Required**: True
+- **Type**: int
+- **Default value**: None
+- **Constraints**: The minimum value is `1`. The maximum value is `65535
+ -   **Description**: Port number of the Clio HTTP server.
+### Key: server.max_queue_size
+- **Required**: True
+- **Type**: int
+- **Default value**: 0
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Maximum size of the server's request queue. Value of 0 is no limit.
+### Key: server.local_admin
+- **Required**: False
+- **Type**: boolean
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: Indicates if the server should run with admin privileges. Only one of local_admin or admin_password can be set.
+### Key: server.admin_password
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: Password for Clio admin-only APIs. Only one of local_admin or admin_password can be set.
+### Key: server.processing_policy
+- **Required**: True
+- **Type**: string
+- **Default value**: parallel
+- **Constraints**: The value must be one of the following: `parallel, sequent`
+ -   **Description**: Could be "sequent" or "parallel". For the sequent policy, requests from a single client 
+        connection are processed one by one, with the next request read only after the previous one is processed. For the parallel policy, Clio will accept
+         all requests and process them in parallel, sending a reply for each request as soon as it is ready.
+### Key: server.parallel_requests_limit
+- **Required**: False
+- **Type**: int
+- **Default value**: None
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Optional parameter, used only if processing_strategy `parallel`. It limits the number of requests for a single client connection that are processed in parallel. If not specified, the limit is infinite.
+### Key: server.ws_max_sending_queue_size
+- **Required**: True
+- **Type**: int
+- **Default value**: 1500
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Maximum size of the websocket sending queue.
+### Key: prometheus.enabled
+- **Required**: True
+- **Type**: boolean
+- **Default value**: False
+- **Constraints**: None
+ -   **Description**: Enable or disable Prometheus metrics.
+### Key: prometheus.compress_reply
+- **Required**: True
+- **Type**: boolean
+- **Default value**: False
+- **Constraints**: None
+ -   **Description**: Enable or disable compression of Prometheus responses.
+### Key: io_threads
+- **Required**: True
+- **Type**: int
+- **Default value**: 2
+- **Constraints**: The minimum value is `1`. The maximum value is `65535`
+ -   **Description**: Number of I/O threads. Value cannot be less than 1
+### Key: subscription_workers
+- **Required**: True
+- **Type**: int
+- **Default value**: 1
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: The number of worker threads or processes that are responsible for managing and processing subscription-based tasks from rippled
+### Key: graceful_period
+- **Required**: True
+- **Type**: double
+- **Default value**: 10
+- **Constraints**: The value must be a positive double number
+ -   **Description**: Number of milliseconds server will wait to shutdown gracefully.
+### Key: cache.num_diffs
+- **Required**: True
+- **Type**: int
+- **Default value**: 32
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Number of diffs to cache. For more info, consult readme.md in etc
+### Key: cache.num_markers
+- **Required**: True
+- **Type**: int
+- **Default value**: 48
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Number of markers to cache.
+### Key: cache.num_cursors_from_diff
+- **Required**: True
+- **Type**: int
+- **Default value**: 0
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Num of cursors that are different.
+### Key: cache.num_cursors_from_account
+- **Required**: True
+- **Type**: int
+- **Default value**: 0
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Number of cursors from an account.
+### Key: cache.page_fetch_size
+- **Required**: True
+- **Type**: int
+- **Default value**: 512
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Page fetch size for cache operations.
+### Key: cache.load
+- **Required**: True
+- **Type**: string
+- **Default value**: async
+- **Constraints**: The value must be one of the following: `sync, async, none`
+ -   **Description**: Cache loading strategy ('sync' or 'async').
+### Key: log_channels.[].channel
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: The value must be one of the following: `General, WebServer, Backend, RPC, ETL, Subscriptions, Performance, Migration`
+ -   **Description**: Name of the log channel.'RPC', 'ETL', and 'Performance'
+### Key: log_channels.[].log_level
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: The value must be one of the following: `trace, debug, info, warning, error, fatal, count`
+ -   **Description**: Log level for the specific log channel.`warning`, `error`, `fatal`
+### Key: log_level
+- **Required**: True
+- **Type**: string
+- **Default value**: info
+- **Constraints**: The value must be one of the following: `trace, debug, info, warning, error, fatal, count`
+ -   **Description**: General logging level of Clio. This level will be applied to all log channels that do not have an explicitly defined logging level.
+### Key: log_format
+- **Required**: True
+- **Type**: string
+- **Default value**: %TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%
+- **Constraints**: None
+ -   **Description**: Format string for log messages.
+### Key: log_to_console
+- **Required**: True
+- **Type**: boolean
+- **Default value**: True
+- **Constraints**: None
+ -   **Description**: Enable or disable logging to console.
+### Key: log_directory
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: Directory path for log files.
+### Key: log_rotation_size
+- **Required**: True
+- **Type**: int
+- **Default value**: 2048
+- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`
+ -   **Description**: Log rotation size in megabytes. When the log file reaches this particular size, a new log file starts.
+### Key: log_directory_max_size
+- **Required**: True
+- **Type**: int
+- **Default value**: 51200
+- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`
+ -   **Description**: Maximum size of the log directory in megabytes.
+### Key: log_rotation_hour_interval
+- **Required**: True
+- **Type**: int
+- **Default value**: 12
+- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`
+ -   **Description**: Interval in hours for log rotation. If the current log file reaches this value in logging, a new log file starts.
+### Key: log_tag_style
+- **Required**: True
+- **Type**: string
+- **Default value**: none
+- **Constraints**: The value must be one of the following: `int, uint, null, none, uuid`
+ -   **Description**: Style for log tags.
+### Key: extractor_threads
+- **Required**: True
+- **Type**: int
+- **Default value**: 1
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Number of extractor threads.
+### Key: read_only
+- **Required**: True
+- **Type**: boolean
+- **Default value**: True
+- **Constraints**: None
+ -   **Description**: Indicates if the server should have read-only privileges.
+### Key: txn_threshold
+- **Required**: True
+- **Type**: int
+- **Default value**: 0
+- **Constraints**: The minimum value is `0`. The maximum value is `65535`
+ -   **Description**: Transaction threshold value.
+### Key: start_sequence
+- **Required**: False
+- **Type**: int
+- **Default value**: None
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Starting ledger index.
+### Key: finish_sequence
+- **Required**: False
+- **Type**: int
+- **Default value**: None
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: Ending ledger index.
+### Key: ssl_cert_file
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: Path to the SSL certificate file.
+### Key: ssl_key_file
+- **Required**: False
+- **Type**: string
+- **Default value**: None
+- **Constraints**: None
+ -   **Description**: Path to the SSL key file.
+### Key: api_version.default
+- **Required**: True
+- **Type**: int
+- **Default value**: 1
+- **Constraints**: The minimum value is `1`. The maximum value is `3`
+ -   **Description**: Default API version Clio will run on.
+### Key: api_version.min
+- **Required**: True
+- **Type**: int
+- **Default value**: 1
+- **Constraints**: The minimum value is `1`. The maximum value is `3`
+ -   **Description**: Minimum API version.
+### Key: api_version.max
+- **Required**: True
+- **Type**: int
+- **Default value**: 3
+- **Constraints**: The minimum value is `1`. The maximum value is `3`
+ -   **Description**: Maximum API version.
+### Key: migration.full_scan_threads
+- **Required**: True
+- **Type**: int
+- **Default value**: 2
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: The number of threads used to scan the table.
+### Key: migration.full_scan_jobs
+- **Required**: True
+- **Type**: int
+- **Default value**: 4
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: The number of coroutines used to scan the table.
+### Key: migration.cursors_per_job
+- **Required**: True
+- **Type**: int
+- **Default value**: 100
+- **Constraints**: The minimum value is `0`. The maximum value is `4294967295`
+ -   **Description**: The number of cursors each coroutine will scan.
+

docs/configure-clio.md

@@ -27,7 +27,7 @@ If you're running Clio and `rippled` on separate machines, in addition to uncomm
 
 2. Open a public, unencrypted WebSocket port on your `rippled` server.
 
-3. In the `rippled` config, change the IP specified for `secure_gateway`, under the `port_grpc` section, to the IP of your Clio server. This entry can take the form of a comma-separated list if you are running multiple Clio nodes.
+3. In the `rippled` config, change the IP specified for `secure_gateway`, under the `port_grpc` and websocket server sections, to the IP of your Clio server. This entry can take the form of a comma-separated list if you are running multiple Clio nodes.
 
 ## Ledger sequence
 
@@ -97,3 +97,11 @@ To enable the caching for a source, `forwarding_cache_timeout` value should be a
 
 `forwarding_cache_timeout` defines for how long (in seconds) a cache entry will be valid after being placed into the cache.
 Zero value turns off the cache feature.
+
+## Graceful shutdown (not fully implemented yet)
+
+Clio can be gracefully shut down by sending a `SIGINT` (Ctrl+C) or `SIGTERM` signal.
+The process will stop accepting new connections and will wait for the time specified in `graceful_period` config value (or 10 seconds by default).
+If Clio finishes all the scheduled operations before the end of the period, it will stop immediately.
+Otherwise, it will wait for the period to finish and then exit without finishing operations.
+If Clio receives a second signal during the period, it will stop immediately.

docs/coverage-report.md

@@ -27,15 +27,6 @@ In case of a spurious failure of unit tests, it is possible to re-run the `cover
 
 The default coverage report format is `html-details`, but developers can override it to any of the formats listed in `cmake/CodeCoverage.cmake` by setting `CODE_COVERAGE_REPORT_FORMAT` variable in `cmake`. For example, CI is setting this parameter to `xml` for the [codecov](https://codecov.io) integration.
 
-If some unit tests predictably fail (e.g., due to absence of a Cassandra database), it is possible to set unit tests options in the `CODE_COVERAGE_TESTS_ARGS` cmake variable, as demonstrated below:
-
-```sh
-cd .build
-conan install .. --output-folder . --build missing --settings build_type=Debug -o tests=True -o coverage=True
-cmake -DCODE_COVERAGE_REPORT_FORMAT=json-details -DCMAKE_BUILD_TYPE=Debug -DCODE_COVERAGE_TESTS_ARGS="--gtest_filter=-BackendCassandra*" -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake ..
-cmake --build . --target coverage_report
-```
-
 After the `coverage_report` target is completed, the generated coverage report will be stored inside the build directory as either:
 
 - A File named `coverage_report.*`, with a suitable extension for the report format.

docs/examples/config/cloud-example-config.json

@@ -1,43 +0,0 @@
-/*
- * This is an example configuration file. Please do not use without modifying to suit your needs.
- */
-{
-    "database": {
-        "type": "cassandra",
-        "cassandra": {
-            // This option can be used to setup a secure connect bundle connection
-            "secure_connect_bundle": "[path/to/zip. ignore if using contact_points]",
-            // The following options are used only if using contact_points
-            "contact_points": "[ip. ignore if using secure_connect_bundle]",
-            "port": "[port. ignore if using_secure_connect_bundle]",
-            // Authentication settings
-            "username": "[username, if any]",
-            "password": "[password, if any]",
-            // Other common settings
-            "keyspace": "clio",
-            "max_write_requests_outstanding": 25000,
-            "max_read_requests_outstanding": 30000,
-            "threads": 8
-        }
-    },
-    "etl_sources": [
-        {
-            "ip": "[rippled ip]",
-            "ws_port": "6006",
-            "grpc_port": "50051"
-        }
-    ],
-    "dos_guard": {
-        "whitelist": [
-            "127.0.0.1"
-        ]
-    },
-    "server": {
-        "ip": "0.0.0.0",
-        "port": 8080
-    },
-    "log_level": "debug",
-    "log_file": "./clio.log",
-    "extractor_threads": 8,
-    "read_only": false
-}

docs/examples/config/example-config.json

@@ -31,11 +31,17 @@
     "etl_sources": [
         {
             "ip": "127.0.0.1",
-            "ws_port": "6006",
+            "ws_port": "6005",
             "grpc_port": "50051"
         }
     ],
-    "forwarding_cache_timeout": 0.250, // in seconds, could be 0, which means no cache
+    "forwarding": {
+        "cache_timeout": 0.250, // in seconds, could be 0, which means no cache
+        "request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds)
+    },
+    "rpc": {
+        "cache_timeout": 0.5 // in seconds, could be 0, which means no cache for rpc
+    },
     "dos_guard": {
         // Comma-separated list of IPs to exclude from rate limiting
         "whitelist": [
@@ -64,8 +70,18 @@
         "admin_password": "xrp",
         // If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests
         // It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time
-        "local_amdin": false
+        "local_admin": false,
+        "processing_policy": "parallel", // Could be "sequent" or "parallel".
+        // For sequent policy request from one client connection will be processed one by one and the next one will not be read before
+        // the previous one is processed. For parallel policy Clio will take all requests and process them in parallel and
+        // send a reply for each request whenever it is ready.
+        "parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified.
+        // Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection.
+        "ws_max_sending_queue_size": 1500,
+        "__ng_web_server": false // Use ng web server. This is a temporary setting which will be deleted after switching to ng web server
     },
+    // Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet.
+    "graceful_period": 10.0,
     // Overrides log level on a per logging channel.
     // Defaults to global "log_level" for each unspecified channel.
     "log_channels": [
@@ -94,6 +110,15 @@
             "log_level": "trace"
         }
     ],
+    "cache": {
+        // Configure this to use either "num_diffs", "num_cursors_from_diff", or "num_cursors_from_account". By default, Clio uses "num_diffs".
+        "num_diffs": 32, // Generate the cursors from the latest ledger diff, then use the cursors to partition the ledger to load concurrently. The cursors number is affected by the busyness of the network.
+        // "num_cursors_from_diff": 3200, // Read the cursors from the diff table until we have enough cursors to partition the ledger to load concurrently. 
+        // "num_cursors_from_account": 3200, // Read the cursors from the account table until we have enough cursors to partition the ledger to load concurrently.
+        "num_markers": 48, // The number of markers is the number of coroutines to load the cache concurrently.
+        "page_fetch_size": 512, // The number of rows to load for each page.
+        "load": "async" // "sync" to load cache synchronously  or "async" to load cache asynchronously or "none"/"no" to turn off the cache.
+    },
     "prometheus": {
         "enabled": true,
         "compress_reply": true

docs/examples/infrastructure/README.md

@@ -1,5 +1,9 @@
 # Example of clio monitoring infrastructure
 
+> [!WARNING]
+> This is only an example of Grafana dashboard for Clio. It was created for demonstration purposes only and may contain errors.
+> Clio team would not recommend to relate on data from this dashboard or use it for monitoring your Clio instances.
+
 This directory contains an example of docker based infrastructure to collect and visualise metrics from clio.
 
 The structure of the directory:

docs/examples/infrastructure/grafana/clio_dashboard.json

@@ -18,8 +18,8 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
+  "id": 1,
   "links": [],
-  "liveNow": false,
   "panels": [
     {
       "datasource": {
@@ -78,6 +78,7 @@
         "graphMode": "area",
         "justifyMode": "auto",
         "orientation": "auto",
+        "percentChangeColorMode": "standard",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -85,9 +86,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "10.2.0",
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -113,41 +116,106 @@
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "palette-classic"
+            "mode": "thresholds"
           },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "blue",
+                  "index": 0,
+                  "text": "No"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 1,
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
             }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 3,
+        "x": 3,
+        "y": 0
+      },
+      "id": 14,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
           },
-          "mappings": [],
+          "editorMode": "code",
+          "expr": "read_only",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Read only",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "blue",
+                  "index": 0,
+                  "text": "No"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 1,
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
+            }
+          ],
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -166,23 +234,29 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 9,
-        "x": 3,
+        "w": 3,
+        "x": 6,
         "y": 0
       },
-      "id": 2,
+      "id": 15,
       "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": false
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
         },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -190,15 +264,100 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "work_queue_current_size{job=\"clio\"}",
+          "expr": "etl_writing",
           "instant": false,
-          "legendFormat": "work queue size",
+          "legendFormat": "__auto",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Work Queue Size",
-      "type": "timeseries"
+      "title": "Writing data to DB",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "No"
+                },
+                "1": {
+                  "color": "red",
+                  "index": 1,
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 3,
+        "x": 9,
+        "y": 0
+      },
+      "id": 16,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "percentChangeColorMode": "standard",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "etl_amendment_blocked",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Amendment blocked",
+      "type": "stat"
     },
     {
       "datasource": {
@@ -217,6 +376,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -255,7 +415,8 @@
                 "value": 80
               }
             ]
-          }
+          },
+          "unit": "s"
         },
         "overrides": []
       },
@@ -265,7 +426,7 @@
         "x": 12,
         "y": 0
       },
-      "id": 9,
+      "id": 13,
       "options": {
         "legend": {
           "calcs": [],
@@ -278,6 +439,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -285,14 +447,36 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "rate(rpc_method_total_number{job=\"clio\"}[$__rate_interval])",
+          "exemplar": false,
+          "expr": "timestamp(etl_last_publish_seconds) - etl_last_publish_seconds",
+          "format": "time_series",
           "instant": false,
-          "legendFormat": "{{method}} {{status}}",
+          "legendFormat": "ledger age",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "RPC Method Call Rate",
+      "title": "Ledger Age",
+      "transformations": [
+        {
+          "id": "filterByValue",
+          "options": {
+            "filters": [
+              {
+                "config": {
+                  "id": "lower",
+                  "options": {
+                    "value": 31500000
+                  }
+                },
+                "fieldName": "ledger age"
+              }
+            ],
+            "match": "all",
+            "type": "include"
+          }
+        }
+      ],
       "type": "timeseries"
     },
     {
@@ -312,6 +496,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -360,19 +545,20 @@
         "x": 0,
         "y": 8
       },
-      "id": 11,
+      "id": 2,
       "options": {
         "legend": {
           "calcs": [],
           "displayMode": "list",
           "placement": "bottom",
-          "showLegend": true
+          "showLegend": false
         },
         "tooltip": {
           "mode": "single",
           "sort": "none"
         }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -380,14 +566,14 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "subscriptions_current_number{job=\"clio\"}",
+          "expr": "work_queue_current_size{job=\"clio\"}",
           "instant": false,
-          "legendFormat": "{{collection}}{{stream}}",
+          "legendFormat": "work queue size",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Subscriptions Number",
+      "title": "Work Queue Size",
       "type": "timeseries"
     },
     {
@@ -407,6 +593,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -455,6 +642,200 @@
         "x": 12,
         "y": 8
       },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "rate(rpc_method_total_number{job=\"clio\"}[$__rate_interval])",
+          "instant": false,
+          "legendFormat": "{{method}} {{status}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "RPC Method Call Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "subscriptions_current_number{job=\"clio\"}",
+          "instant": false,
+          "legendFormat": "{{collection}}{{stream}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Subscriptions Number",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
       "id": 6,
       "options": {
         "legend": {
@@ -468,6 +849,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -503,6 +885,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -550,7 +933,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 16
+        "y": 24
       },
       "id": 10,
       "options": {
@@ -565,6 +948,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -572,7 +956,7 @@
             "uid": "PBFA97CFB590B2093"
           },
           "editorMode": "code",
-          "expr": "rpc_method_duration_us{job=\"clio\"}",
+          "expr": "sum by (method) (increase(rpc_method_duration_us[$__interval]))\n / \n sum by (method,) (increase(rpc_method_total_number{status=\"finished\"}[$__interval]))",
           "instant": false,
           "legendFormat": "{{method}}",
           "range": true,
@@ -599,6 +983,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -645,7 +1030,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 16
+        "y": 24
       },
       "id": 8,
       "options": {
@@ -660,6 +1045,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -694,6 +1080,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -740,7 +1127,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 24
+        "y": 32
       },
       "id": 4,
       "options": {
@@ -755,6 +1142,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -789,6 +1177,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 10,
             "gradientMode": "none",
@@ -839,7 +1228,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 24
+        "y": 32
       },
       "id": 12,
       "options": {
@@ -854,7 +1243,7 @@
           "sort": "none"
         }
       },
-      "pluginVersion": "10.2.0",
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -927,6 +1316,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -973,7 +1363,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 32
+        "y": 40
       },
       "id": 5,
       "options": {
@@ -988,6 +1378,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -1035,6 +1426,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -1081,7 +1473,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 32
+        "y": 40
       },
       "id": 3,
       "options": {
@@ -1096,6 +1488,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -1141,6 +1534,7 @@
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
+            "barWidthFactor": 0.6,
             "drawStyle": "line",
             "fillOpacity": 0,
             "gradientMode": "none",
@@ -1186,9 +1580,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 10,
+        "w": 12,
         "x": 0,
-        "y": 40
+        "y": 48
       },
       "id": 7,
       "options": {
@@ -1203,6 +1597,7 @@
           "sort": "none"
         }
       },
+      "pluginVersion": "11.4.0",
       "targets": [
         {
           "datasource": {
@@ -1221,8 +1616,9 @@
       "type": "timeseries"
     }
   ],
+  "preload": false,
   "refresh": "5s",
-  "schemaVersion": 38,
+  "schemaVersion": 40,
   "tags": [],
   "templating": {
     "list": []
@@ -1235,6 +1631,6 @@
   "timezone": "",
   "title": "Clio",
   "uid": "aeaae84e-c194-47b2-ad65-86e45eebb815",
-  "version": 3,
+  "version": 1,
   "weekStart": ""
 }

docs/img/doxygen-docs-output.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16e6f0a44b2dc7462f813f07ba754575e93033bc8bbfbe1d28f8bf1927915028
+size 394711

docs/metrics-and-static-analysis.md

@@ -1,30 +0,0 @@
-# Metrics and static analysis
-
-## Prometheus metrics collection
-
-Clio natively supports [Prometheus](https://prometheus.io/) metrics collection. It accepts Prometheus requests on the port configured in the `server` section of the config.
-
-Prometheus metrics are enabled by default, and replies to `/metrics` are compressed. To disable compression, and have human readable metrics, add `"prometheus": { "enabled": true, "compress_reply": false }` to Clio's config.
-
-To completely disable Prometheus metrics add `"prometheus": { "enabled": false }` to Clio's config.
-
-It is important to know that Clio responds to Prometheus request only if they are admin requests. If you are using the admin password feature, the same password should be provided in the Authorization header of Prometheus requests.
-
-You can find an example docker-compose file, with Prometheus and Grafana configs, in [examples/infrastructure](../docs/examples/infrastructure/).
-
-## Using `clang-tidy` for static analysis
-
-The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 17.0.
-
-Clang-tidy can be run by Cmake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command:
-
-```sh
-conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=True
-```
-
-By default Cmake will try to find `clang-tidy` automatically in your system.
-To force Cmake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example:
-
-```sh
-export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@17/bin/clang-tidy
-```

docs/run-clio.md

@@ -80,3 +80,15 @@ Clio will fallback to hardcoded defaults when these values are not specified in
 
 > [!TIP]
 > See the [example-config.json](../docs/examples/config/example-config.json) for more details.
+
+## Prometheus metrics collection
+
+Clio natively supports [Prometheus](https://prometheus.io/) metrics collection. It accepts Prometheus requests on the port configured in the `server` section of the config.
+
+Prometheus metrics are enabled by default, and replies to `/metrics` are compressed. To disable compression, and have human readable metrics, add `"prometheus": { "enabled": true, "compress_reply": false }` to Clio's config.
+
+To completely disable Prometheus metrics add `"prometheus": { "enabled": false }` to Clio's config.
+
+It is important to know that Clio responds to Prometheus request only if they are admin requests. If you are using the admin password feature, the same password should be provided in the Authorization header of Prometheus requests.
+
+You can find an example docker-compose file, with Prometheus and Grafana configs, in [examples/infrastructure](../docs/examples/infrastructure/).

docs/trouble_shooting.md

@@ -0,0 +1,47 @@
+# Troubleshooting Guide
+This guide will help you troubleshoot common issues of Clio.
+
+## Can't connect to DB
+If you see the error log message `Could not connect to Cassandra: No hosts available`, this means that Clio can't connect to the database. Check the following:
+- Make sure the database is running at the specified address and port.
+- Make sure the database is accessible from the machine where Clio is running.
+You can use [cqlsh](https://pypi.org/project/cqlsh/) to check the connection to the database.
+If you would like to run a local ScyllaDB, you can call:
+```sh
+docker run --rm -p 9042:9042 --name clio-scylla -d scylladb/scylla 
+```
+
+## Check the server status of Clio
+To check if Clio is syncing with rippled:
+```sh
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep seq
+```
+If Clio is syncing with rippled, the `seq` value will be increasing.
+
+## Clio fails to start
+If you see the error log message `Failed to fetch ETL state from...`, this means the configured rippled node is not reachable. Check the following:
+- Make sure the rippled node is running at the specified address and port.
+- Make sure the rippled node is accessible from the machine where Clio is running.
+
+If you would like to run Clio without an avaliable rippled node, you can add below setting to Clio's configuration file:
+```
+"allow_no_etl": true
+```
+
+## Clio is not added to secure_gateway in rippled's config
+If you see the warning message `AsyncCallData is_unlimited is false.`, this means that Clio is not added to the `secure_gateway` of `port_grpc` session in the rippled configuration file. It will slow down the sync process. Please add Clio's IP to the `secure_gateway` in the rippled configuration file for both grpc and ws port.
+
+## Clio is slow
+To speed up the response time, Clio has a cache inside. However, cache can take time to warm up. If you see slow response time, you can firstly check if cache is still loading.
+You can check the cache status by calling:
+```sh
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_full
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_enabled
+```
+If `is_full` is false, it means the cache is still loading. Normally, the Clio can respond quicker if cache finishs loading. If `is_enabled` is false, it means the cache is disabled in the configuration file or there is data corruption in the database. 
+
+## Receive error message `Too many requests`
+If client sees the error message `Too many requests`, this means that the client is blocked by Clio's DosGuard protection. You may want to add the client's IP to the whitelist in the configuration file, Or update other your DosGuard settings.
+
+
+

src/CMakeLists.txt

@@ -1,7 +1,10 @@
 add_subdirectory(util)
 add_subdirectory(data)
 add_subdirectory(etl)
+add_subdirectory(etlng)
 add_subdirectory(feed)
 add_subdirectory(rpc)
 add_subdirectory(web)
+add_subdirectory(migration)
+add_subdirectory(app)
 add_subdirectory(main)

src/app/CMakeLists.txt

@@ -0,0 +1,4 @@
+add_library(clio_app)
+target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp Stopper.cpp WebHandlers.cpp)
+
+target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc clio_migration)

src/app/CliArgs.cpp

@@ -0,0 +1,100 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "app/CliArgs.hpp"
+
+#include "migration/MigrationApplication.hpp"
+#include "util/build/Build.hpp"
+#include "util/newconfig/ConfigDescription.hpp"
+
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/program_options/positional_options.hpp>
+#include <boost/program_options/value_semantic.hpp>
+#include <boost/program_options/variables_map.hpp>
+
+#include <cstdlib>
+#include <filesystem>
+#include <iostream>
+#include <string>
+#include <utility>
+
+namespace app {
+
+CliArgs::Action
+CliArgs::parse(int argc, char const* argv[])
+{
+    namespace po = boost::program_options;
+    // clang-format off
+    po::options_description description("Options");
+    description.add_options()
+        ("help,h", "Print help message and exit")
+        ("version,v", "Print version and exit")
+        ("conf,c", po::value<std::string>()->default_value(kDEFAULT_CONFIG_PATH), "Configuration file")
+        ("ng-web-server,w", "Use ng-web-server")
+        ("migrate", po::value<std::string>(), "Start migration helper")
+        ("verify", "Checks the validity of config values")
+        ("config-description,d", po::value<std::string>(), "Generate config description markdown file")
+    ;
+    // clang-format on
+    po::positional_options_description positional;
+    positional.add("conf", 1);
+
+    po::variables_map parsed;
+    po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
+    po::notify(parsed);
+
+    if (parsed.count("help") != 0u) {
+        std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n" << description;
+        return Action{Action::Exit{EXIT_SUCCESS}};
+    }
+
+    if (parsed.count("version") != 0u) {
+        std::cout << util::build::getClioFullVersionString() << '\n';
+        return Action{Action::Exit{EXIT_SUCCESS}};
+    }
+
+    if (parsed.count("config-description") != 0u) {
+        std::filesystem::path const filePath = parsed["config-description"].as<std::string>();
+
+        auto const res = util::config::ClioConfigDescription::generateConfigDescriptionToFile(filePath);
+        if (res.has_value())
+            return Action{Action::Exit{EXIT_SUCCESS}};
+
+        std::cerr << res.error().error << std::endl;
+        return Action{Action::Exit{EXIT_FAILURE}};
+    }
+
+    auto configPath = parsed["conf"].as<std::string>();
+
+    if (parsed.count("migrate") != 0u) {
+        auto const opt = parsed["migrate"].as<std::string>();
+        if (opt == "status")
+            return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::status()}};
+        return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::migration(opt)}};
+    }
+
+    if (parsed.count("verify") != 0u)
+        return Action{Action::VerifyConfig{.configPath = std::move(configPath)}};
+
+    return Action{Action::Run{.configPath = std::move(configPath), .useNgWebServer = parsed.count("ng-web-server") != 0}
+    };
+}
+
+}  // namespace app

src/app/CliArgs.hpp

@@ -0,0 +1,108 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigrationApplication.hpp"
+#include "util/OverloadSet.hpp"
+
+#include <string>
+#include <variant>
+
+namespace app {
+
+/**
+ * @brief Parsed command line arguments representation.
+ */
+class CliArgs {
+public:
+    /**
+     * @brief Default configuration path.
+     */
+    static constexpr char kDEFAULT_CONFIG_PATH[] = "/etc/opt/clio/config.json";
+
+    /**
+     * @brief An action parsed from the command line.
+     */
+    class Action {
+    public:
+        /** @brief Run action. */
+        struct Run {
+            std::string configPath;  ///< Configuration file path.
+            bool useNgWebServer;     ///< Whether to use a ng web server
+        };
+
+        /** @brief Exit action. */
+        struct Exit {
+            int exitCode;  ///< Exit code.
+        };
+
+        /** @brief Migration action. */
+        struct Migrate {
+            std::string configPath;
+            MigrateSubCmd subCmd;
+        };
+
+        /** @brief Verify Config action. */
+        struct VerifyConfig {
+            std::string configPath;
+        };
+
+        /**
+         * @brief Construct an action from a Run.
+         *
+         * @param action Run action.
+         */
+        template <typename ActionType>
+            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit> or
+            std::is_same_v<ActionType, Migrate> or std::is_same_v<ActionType, VerifyConfig>
+        explicit Action(ActionType&& action) : action_(std::forward<ActionType>(action))
+        {
+        }
+
+        /**
+         * @brief Apply a function to the action.
+         *
+         * @tparam Processors Action processors types. Must be callable with the action type and return int.
+         * @param processors Action processors.
+         * @return Exit code.
+         */
+        template <typename... Processors>
+        int
+        apply(Processors&&... processors) const
+        {
+            return std::visit(util::OverloadSet{std::forward<Processors>(processors)...}, action_);
+        }
+
+    private:
+        std::variant<Run, Exit, Migrate, VerifyConfig> action_;
+    };
+
+    /**
+     * @brief Parse command line arguments.
+     *
+     * @param argc Number of arguments.
+     * @param argv Array of arguments.
+     * @return Parsed command line arguments.
+     */
+    static Action
+    parse(int argc, char const* argv[]);
+};
+
+}  // namespace app

src/app/ClioApplication.cpp

@@ -0,0 +1,204 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "app/ClioApplication.hpp"
+
+#include "app/Stopper.hpp"
+#include "app/WebHandlers.hpp"
+#include "data/AmendmentCenter.hpp"
+#include "data/BackendFactory.hpp"
+#include "data/LedgerCache.hpp"
+#include "etl/ETLService.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "etl/NetworkValidatedLedgers.hpp"
+#include "feed/SubscriptionManager.hpp"
+#include "migration/MigrationInspectorFactory.hpp"
+#include "rpc/Counters.hpp"
+#include "rpc/RPCEngine.hpp"
+#include "rpc/WorkQueue.hpp"
+#include "rpc/common/impl/HandlerProvider.hpp"
+#include "util/build/Build.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/prometheus/Prometheus.hpp"
+#include "web/AdminVerificationStrategy.hpp"
+#include "web/RPCServerHandler.hpp"
+#include "web/Server.hpp"
+#include "web/dosguard/DOSGuard.hpp"
+#include "web/dosguard/IntervalSweepHandler.hpp"
+#include "web/dosguard/WhitelistHandler.hpp"
+#include "web/ng/RPCServerHandler.hpp"
+#include "web/ng/Server.hpp"
+
+#include <boost/asio/io_context.hpp>
+
+#include <cstdint>
+#include <cstdlib>
+#include <memory>
+#include <optional>
+#include <thread>
+#include <utility>
+#include <vector>
+
+namespace app {
+
+namespace {
+
+/**
+ * @brief Start context threads
+ *
+ * @param ioc Context
+ * @param numThreads Number of worker threads to start
+ */
+void
+start(boost::asio::io_context& ioc, std::uint32_t numThreads)
+{
+    std::vector<std::thread> v;
+    v.reserve(numThreads - 1);
+    for (auto i = numThreads - 1; i > 0; --i)
+        v.emplace_back([&ioc] { ioc.run(); });
+
+    ioc.run();
+    for (auto& t : v)
+        t.join();
+}
+
+}  // namespace
+
+ClioApplication::ClioApplication(util::config::ClioConfigDefinition const& config)
+    : config_(config), signalsHandler_{config_}
+{
+    LOG(util::LogService::info()) << "Clio version: " << util::build::getClioFullVersionString();
+    PrometheusService::init(config);
+    signalsHandler_.subscribeToStop([this]() { appStopper_.stop(); });
+}
+
+int
+ClioApplication::run(bool const useNgWebServer)
+{
+    auto const threads = config_.get<uint16_t>("io_threads");
+    LOG(util::LogService::info()) << "Number of io threads = " << threads;
+
+    // IO context to handle all incoming requests, as well as other things.
+    // This is not the only io context in the application.
+    boost::asio::io_context ioc{threads};
+
+    // Rate limiter, to prevent abuse
+    auto whitelistHandler = web::dosguard::WhitelistHandler{config_};
+    auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler};
+    auto sweepHandler = web::dosguard::IntervalSweepHandler{config_, ioc, dosGuard};
+    auto cache = data::LedgerCache{};
+
+    // Interface to the database
+    auto backend = data::makeBackend(config_, cache);
+
+    auto const amendmentCenter = std::make_shared<data::AmendmentCenter const>(backend);
+
+    {
+        auto const migrationInspector = migration::makeMigrationInspector(config_, backend);
+        // Check if any migration is blocking Clio server starting.
+        if (migrationInspector->isBlockingClio() and backend->hardFetchLedgerRangeNoThrow()) {
+            LOG(util::LogService::error())
+                << "Existing Migration is blocking Clio, Please complete the database migration first.";
+            return EXIT_FAILURE;
+        }
+    }
+
+    // Manages clients subscribed to streams
+    auto subscriptions = feed::SubscriptionManager::makeSubscriptionManager(config_, backend, amendmentCenter);
+
+    // Tracks which ledgers have been validated by the network
+    auto ledgers = etl::NetworkValidatedLedgers::makeValidatedLedgers();
+
+    // Handles the connection to one or more rippled nodes.
+    // ETL uses the balancer to extract data.
+    // The server uses the balancer to forward RPCs to a rippled node.
+    // The balancer itself publishes to streams (transactions_proposed and accounts_proposed)
+    auto balancer = etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
+
+    // ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes
+    auto etl = etl::ETLService::makeETLService(config_, ioc, backend, subscriptions, balancer, ledgers);
+
+    auto workQueue = rpc::WorkQueue::makeWorkQueue(config_);
+    auto counters = rpc::Counters::makeCounters(workQueue);
+
+    auto const handlerProvider = std::make_shared<rpc::impl::ProductionHandlerProvider const>(
+        config_, backend, subscriptions, balancer, etl, amendmentCenter, counters
+    );
+
+    using RPCEngineType = rpc::RPCEngine<etl::LoadBalancer, rpc::Counters>;
+    auto const rpcEngine =
+        RPCEngineType::makeRPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
+
+    if (useNgWebServer or config_.get<bool>("server.__ng_web_server")) {
+        web::ng::RPCServerHandler<RPCEngineType, etl::ETLService> handler{config_, backend, rpcEngine, etl};
+
+        auto expectedAdminVerifier = web::makeAdminVerificationStrategy(config_);
+        if (not expectedAdminVerifier.has_value()) {
+            LOG(util::LogService::error()) << "Error creating admin verifier: " << expectedAdminVerifier.error();
+            return EXIT_FAILURE;
+        }
+        auto const adminVerifier = std::move(expectedAdminVerifier).value();
+
+        auto httpServer = web::ng::makeServer(config_, OnConnectCheck{dosGuard}, DisconnectHook{dosGuard}, ioc);
+
+        if (not httpServer.has_value()) {
+            LOG(util::LogService::error()) << "Error creating web server: " << httpServer.error();
+            return EXIT_FAILURE;
+        }
+
+        httpServer->onGet("/metrics", MetricsHandler{adminVerifier});
+        httpServer->onGet("/health", HealthCheckHandler{});
+        auto requestHandler = RequestHandler{adminVerifier, handler, dosGuard};
+        httpServer->onPost("/", requestHandler);
+        httpServer->onWs(std::move(requestHandler));
+
+        auto const maybeError = httpServer->run();
+        if (maybeError.has_value()) {
+            LOG(util::LogService::error()) << "Error starting web server: " << *maybeError;
+            return EXIT_FAILURE;
+        }
+
+        appStopper_.setOnStop(
+            Stopper::makeOnStopCallback(httpServer.value(), *balancer, *etl, *subscriptions, *backend, ioc)
+        );
+
+        // Blocks until stopped.
+        // When stopped, shared_ptrs fall out of scope
+        // Calls destructors on all resources, and destructs in order
+        start(ioc, threads);
+
+        return EXIT_SUCCESS;
+    }
+
+    // Init the web server
+    auto handler =
+        std::make_shared<web::RPCServerHandler<RPCEngineType, etl::ETLService>>(config_, backend, rpcEngine, etl);
+
+    auto const httpServer = web::makeHttpServer(config_, ioc, dosGuard, handler);
+
+    // Blocks until stopped.
+    // When stopped, shared_ptrs fall out of scope
+    // Calls destructors on all resources, and destructs in order
+    start(ioc, threads);
+
+    return EXIT_SUCCESS;
+}
+
+}  // namespace app

src/app/ClioApplication.hpp

@@ -0,0 +1,55 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "app/Stopper.hpp"
+#include "util/SignalsHandler.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+namespace app {
+
+/**
+ * @brief The main application class
+ */
+class ClioApplication {
+    util::config::ClioConfigDefinition const& config_;
+    util::SignalsHandler signalsHandler_;
+    Stopper appStopper_;
+
+public:
+    /**
+     * @brief Construct a new ClioApplication object
+     *
+     * @param config The configuration of the application
+     */
+    ClioApplication(util::config::ClioConfigDefinition const& config);
+
+    /**
+     * @brief Run the application
+     *
+     * @param useNgWebServer Whether to use the new web server
+     *
+     * @return exit code
+     */
+    int
+    run(bool useNgWebServer);
+};
+
+}  // namespace app

src/app/Stopper.cpp

@@ -0,0 +1,52 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "app/Stopper.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <functional>
+#include <thread>
+#include <utility>
+
+namespace app {
+
+Stopper::~Stopper()
+{
+    if (worker_.joinable())
+        worker_.join();
+}
+
+void
+Stopper::setOnStop(std::function<void(boost::asio::yield_context)> cb)
+{
+    boost::asio::spawn(ctx_, std::move(cb));
+}
+
+void
+Stopper::stop()
+{
+    // Do nothing if worker_ is already running
+    if (worker_.joinable())
+        return;
+
+    worker_ = std::thread{[this]() { ctx_.run(); }};
+}
+
+}  // namespace app

src/app/Stopper.hpp

@@ -0,0 +1,118 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "etl/ETLService.hpp"
+#include "etl/LoadBalancer.hpp"
+#include "feed/SubscriptionManagerInterface.hpp"
+#include "util/CoroutineGroup.hpp"
+#include "util/log/Logger.hpp"
+#include "web/ng/Server.hpp"
+
+#include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+
+#include <functional>
+#include <thread>
+
+namespace app {
+
+/**
+ * @brief Application stopper class. On stop it will create a new thread to run all the shutdown tasks.
+ */
+class Stopper {
+    boost::asio::io_context ctx_;
+    std::thread worker_;
+
+public:
+    /**
+     * @brief Destroy the Stopper object
+     */
+    ~Stopper();
+
+    /**
+     * @brief Set the callback to be called when the application is stopped.
+     *
+     * @param cb The callback to be called on application stop.
+     */
+    void
+    setOnStop(std::function<void(boost::asio::yield_context)> cb);
+
+    /**
+     * @brief Stop the application and run the shutdown tasks.
+     */
+    void
+    stop();
+
+    /**
+     * @brief Create a callback to be called on application stop.
+     *
+     * @param server The server to stop.
+     * @param balancer The load balancer to stop.
+     * @param etl The ETL service to stop.
+     * @param subscriptions The subscription manager to stop.
+     * @param backend The backend to stop.
+     * @param ioc The io_context to stop.
+     * @return The callback to be called on application stop.
+     */
+    template <
+        web::ng::SomeServer ServerType,
+        etl::SomeLoadBalancer LoadBalancerType,
+        etl::SomeETLService ETLServiceType>
+    static std::function<void(boost::asio::yield_context)>
+    makeOnStopCallback(
+        ServerType& server,
+        LoadBalancerType& balancer,
+        ETLServiceType& etl,
+        feed::SubscriptionManagerInterface& subscriptions,
+        data::BackendInterface& backend,
+        boost::asio::io_context& ioc
+    )
+    {
+        return [&](boost::asio::yield_context yield) {
+            util::CoroutineGroup coroutineGroup{yield};
+            coroutineGroup.spawn(yield, [&server](auto innerYield) {
+                server.stop(innerYield);
+                LOG(util::LogService::info()) << "Server stopped";
+            });
+            coroutineGroup.spawn(yield, [&balancer](auto innerYield) {
+                balancer.stop(innerYield);
+                LOG(util::LogService::info()) << "LoadBalancer stopped";
+            });
+            coroutineGroup.asyncWait(yield);
+
+            etl.stop();
+            LOG(util::LogService::info()) << "ETL stopped";
+
+            subscriptions.stop();
+            LOG(util::LogService::info()) << "SubscriptionManager stopped";
+
+            backend.waitForWritesToFinish();
+            LOG(util::LogService::info()) << "Backend writes finished";
+
+            ioc.stop();
+            LOG(util::LogService::info()) << "io_context stopped";
+        };
+    }
+};
+
+}  // namespace app

src/app/VerifyConfig.hpp

@@ -0,0 +1,58 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ConfigFileJson.hpp"
+
+#include <cstdlib>
+#include <iostream>
+#include <string_view>
+
+namespace app {
+
+/**
+ * @brief Verifies user's config values are correct
+ *
+ * @param configPath The path to config
+ * @return true if config values are all correct, false otherwise
+ */
+inline bool
+parseConfig(std::string_view configPath)
+{
+    using namespace util::config;
+
+    auto const json = ConfigFileJson::makeConfigFileJson(configPath);
+    if (!json.has_value()) {
+        std::cerr << "Error parsing json from config: " << configPath << "\n" << json.error().error << std::endl;
+        return false;
+    }
+    auto const errors = gClioConfig.parse(json.value());
+    if (errors.has_value()) {
+        for (auto const& err : errors.value()) {
+            std::cerr << "Issues found in provided config '" << configPath << "':\n";
+            std::cerr << err.error << std::endl;
+        }
+        return false;
+    }
+    return true;
+}
+
+}  // namespace app