feat: Implement MPT changes (#1147)

Implements https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens

.clang-tidy

@@ -4,6 +4,9 @@ Checks: '-*,
   bugprone-assert-side-effect,
   bugprone-bad-signal-to-kill-thread,
   bugprone-bool-pointer-implicit-conversion,
+  bugprone-casting-through-void,
+  bugprone-chained-comparison,
+  bugprone-compare-pointer-to-member-virtual-function,
   bugprone-copy-constructor-init,
   bugprone-dangling-handle,
   bugprone-dynamic-static-initializers,
@@ -11,6 +14,8 @@ Checks: '-*,
   bugprone-fold-init-type,
   bugprone-forward-declaration-namespace,
   bugprone-inaccurate-erase,
+  bugprone-inc-dec-in-conditions,
+  bugprone-incorrect-enable-if,
   bugprone-incorrect-roundings,
   bugprone-infinite-loop,
   bugprone-integer-division,
@@ -21,15 +26,16 @@ Checks: '-*,
   bugprone-misplaced-pointer-arithmetic-in-alloc,
   bugprone-misplaced-widening-cast,
   bugprone-move-forwarding-reference,
+  bugprone-multi-level-implicit-pointer-conversion,
   bugprone-multiple-new-in-one-expression,
   bugprone-multiple-statement-macro,
   bugprone-no-escape,
   bugprone-non-zero-enum-to-bool-conversion,
+  bugprone-optional-value-conversion,
   bugprone-parent-virtual-call,
   bugprone-posix-return,
   bugprone-redundant-branch-condition,
   bugprone-reserved-identifier,
-  bugprone-unused-return-value,
   bugprone-shared-ptr-array-mismatch,
   bugprone-signal-handler,
   bugprone-signed-char-misuse,
@@ -60,16 +66,20 @@ Checks: '-*,
   bugprone-unhandled-self-assignment,
   bugprone-unique-ptr-array-mismatch,
   bugprone-unsafe-functions,
+  bugprone-unused-local-non-trivial-variable,
   bugprone-unused-raii,
+  bugprone-unused-return-value,
   bugprone-use-after-move,
   bugprone-virtual-near-miss,
   cppcoreguidelines-init-variables,
   cppcoreguidelines-misleading-capture-default-by-value,
+  cppcoreguidelines-no-suspend-with-lock,
   cppcoreguidelines-pro-type-member-init,
   cppcoreguidelines-pro-type-static-cast-downcast,
   cppcoreguidelines-rvalue-reference-param-not-moved,
   cppcoreguidelines-use-default-member-init,
   cppcoreguidelines-virtual-class-destructor,
+  hicpp-ignored-remove-result,
   llvm-namespace-comment,
   misc-const-correctness,
   misc-definitions-in-headers,
@@ -91,6 +101,8 @@ Checks: '-*,
   modernize-use-equals-default,
   modernize-use-equals-delete,
   modernize-use-override,
+  modernize-use-starts-ends-with,
+  modernize-use-std-numbers,
   modernize-use-using,
   performance-faster-string-find,
   performance-for-range-copy,
@@ -100,6 +112,8 @@ Checks: '-*,
   performance-move-constructor-init,
   performance-no-automatic-move,
   performance-trivially-destructible,
+  readability-avoid-nested-conditional-operator,
+  readability-avoid-return-with-void-value,
   readability-braces-around-statements,
   readability-const-return-type,
   readability-container-contains,
@@ -112,9 +126,12 @@ Checks: '-*,
   readability-make-member-function-const,
   readability-misleading-indentation,
   readability-non-const-parameter,
+  readability-redundant-casting,
   readability-redundant-declaration,
+  readability-redundant-inline-specifier,
   readability-redundant-member-init,
   readability-redundant-string-init,
+  readability-reference-to-constructed-temporary,
   readability-simplify-boolean-expr,
   readability-static-accessed-through-instance,
   readability-static-definition-in-anonymous-namespace,
@@ -124,9 +141,8 @@ Checks: '-*,
 CheckOptions:
   readability-braces-around-statements.ShortStatementLines: 2
   bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true
-  bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc;::std::expected
-  misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*'
+  bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc
+  misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*;.*ranges_lower_bound\.h;time.h;stdlib.h'
 
-HeaderFilterRegex: '^.*/(src|unittests)/.*\.(h|hpp)$'
+HeaderFilterRegex: '^.*/(src|tests)/.*\.(h|hpp)$'
 WarningsAsErrors: '*'
-

.clangd

@@ -1,5 +1,13 @@
+CompileFlags:
+  Add: [-D__cpp_concepts=202002]
+
 Diagnostics:
   UnusedIncludes: Strict
   MissingIncludes: Strict
   Includes:
-    IgnoreHeader: ".*/(detail|impl)/.*"
+    IgnoreHeader:
+      - ".*/(detail|impl)/.*"
+      - ".*expected.*"
+      - ".*ranges_lower_bound.h"
+      - "time.h"
+      - "stdlib.h"

.gitattributes

@@ -0,0 +1,2 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.svg filter=lfs diff=lfs merge=lfs -text

.githooks/check-docs

@@ -13,14 +13,36 @@ TMPDIR=${ROOT}/.cache/doxygen
 TMPFILE=${TMPDIR}/docs.log
 DOCDIR=${TMPDIR}/out
 
+# Check doxygen is at all installed
 if [ -z "$DOXYGEN" ]; then
     # No hard error if doxygen is not installed yet
     cat <<EOF
 
                                    WARNING
 -----------------------------------------------------------------------------
-        'doxygen' is required to check documentation.
-        Please install it for next time. For the time being it's on CI.
+        'doxygen' is required to check documentation. 
+        Please install it for next time.
+
+        Your changes may fail to pass CI once pushed.
+-----------------------------------------------------------------------------
+
+EOF
+    exit 0
+fi
+
+# Check version of doxygen is at least 1.12
+version=$($DOXYGEN --version | grep -o '[0-9\.]*')
+
+if [[ "1.12.0" > "$version" ]]; then
+    # No hard error if doxygen version is not the one we want - let CI deal with it
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+        A minimum of version 1.12 of `which doxygen` is required.
+        Your version is $version. Please upgrade it for next time.
+
+        Your changes may fail to pass CI once pushed.
 -----------------------------------------------------------------------------
 
 EOF

.githooks/check-format

@@ -5,19 +5,33 @@
 # This script checks the format of the code and cmake files.
 # In many cases it will automatically fix the issues and abort the commit.
 
+no_formatted_directories_staged() {
+    staged_directories=$(git diff-index --cached --name-only HEAD | awk -F/ '{print $1}')
+    for sd in $staged_directories; do
+        if [[ "$sd" =~ ^(benchmark|cmake|src|tests)$ ]]; then
+            return 1
+        fi
+    done
+    return 0
+}
+
+if no_formatted_directories_staged ; then
+    exit 0
+fi
+
 echo "+ Checking code format..."
 
 # paths to check and re-format
-sources="src unittests"
+sources="src tests"
 formatter="clang-format -i"
 version=$($formatter --version | grep -o '[0-9\.]*')
 
-if [[ "17.0.0" > "$version" ]]; then
+if [[ "19.0.0" > "$version" ]]; then
     cat <<EOF
 
                                     ERROR
 -----------------------------------------------------------------------------
-            A minimum of version 17 of `which clang-format` is required.
+            A minimum of version 19 of `which clang-format` is required.
             Your version is $version.
             Please fix paths and run again.
 -----------------------------------------------------------------------------
@@ -59,7 +73,9 @@ function grep_code {
     grep -l "${1}" ${sources} -r --include \*.hpp --include \*.cpp
 }
 
-if [[ "$OSTYPE" == "darwin"* ]]; then
+GNU_SED=$(sed --version 2>&1 | grep -q 'GNU' && echo true || echo false)
+
+if [[ "$GNU_SED" == "false" ]]; then # macOS sed
     # make all includes to be <...> style
     grep_code '#include ".*"' | xargs sed -i '' -E 's|#include "(.*)"|#include <\1>|g'
 
@@ -71,11 +87,11 @@ else
     grep_code '#include ".*"' | xargs sed -i -E 's|#include "(.*)"|#include <\1>|g'
 
     # make local includes to be "..." style
-    main_src_dirs=$(find ./src -type d -maxdepth 1 -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g')
+    main_src_dirs=$(find ./src -maxdepth 1 -type d  -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g')
     grep_code "#include <\($main_src_dirs\)/.*>" | xargs sed -i -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g"
 fi
 
-cmake_dirs=$(echo CMake $sources)
+cmake_dirs=$(echo cmake $sources)
 cmake_files=$(find $cmake_dirs -type f \( -name "CMakeLists.txt" -o -name "*.cmake" \))
 cmake_files=$(echo $cmake_files ./CMakeLists.txt)
 
@@ -83,9 +99,10 @@ first=$(git diff $sources $cmake_files)
 find $sources -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.ipp' \) -print0 | xargs -0 $formatter
 cmake-format -i $cmake_files
 second=$(git diff $sources $cmake_files)
-changes=$(diff <(echo "$first") <(echo "$second") | wc -l | sed -e 's/^[[:space:]]*//')
+changes=$(diff <(echo "$first") <(echo "$second"))
+changes_number=$(echo -n "$changes" | wc -l | sed -e 's/^[[:space:]]*//')
 
-if [ "$changes" != "0" ]; then
+if [ "$changes_number" != "0" ]; then
     cat <<\EOF
 
                                    WARNING
@@ -95,5 +112,8 @@ if [ "$changes" != "0" ]; then
 -----------------------------------------------------------------------------
 
 EOF
+    if [[ "$1" == "--diff" ]]; then
+        echo "$changes"
+    fi
     exit 1
 fi

.githooks/post-checkout

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-checkout' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-checkout "$@"

.githooks/post-commit

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-commit' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-commit "$@"

.githooks/post-merge

@@ -0,0 +1,3 @@
+#!/bin/sh
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-merge' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+git lfs post-merge "$@"

.githooks/pre-commit

@@ -4,3 +4,4 @@
 
 source .githooks/check-format
 source .githooks/check-docs
+

.githooks/pre-push

@@ -0,0 +1,58 @@
+#!/bin/sh
+
+
+# git for-each-ref refs/tags  # see which tags are annotated and which are lightweight. Annotated tags are "tag" objects.
+# # Set these so your commits and tags are always signed
+# git config commit.gpgsign true
+# git config tag.gpgsign true
+
+verify_commit_signed() {
+    if git verify-commit HEAD &> /dev/null; then
+        :
+        # echo "HEAD commit seems signed..."
+    else
+        echo "HEAD commit isn't signed!"
+        exit 1
+    fi
+}
+
+verify_tag() {
+    if git describe --exact-match --tags HEAD &> /dev/null; then
+        : # You might be ok to push
+        # echo "Tag is annotated."
+        return 0
+    else
+        echo "Tag for [$version] not an annotated tag."
+        exit 1
+    fi
+}
+
+verify_tag_signed() {
+    if git verify-tag "$version" &> /dev/null ; then
+        : # ok, I guess we'll let you push
+        # echo "Tag appears signed"
+        return 0
+    else
+        echo "$version tag isn't signed"
+        echo "Sign it with [git tag -ams\"$version\" $version]"
+        exit 1
+    fi
+}
+
+while read local_ref local_oid remote_ref remote_oid; do
+    # Check some things if we're pushing a branch called "release/"
+    if echo "$remote_ref" | grep ^refs\/heads\/release\/ &> /dev/null ; then
+        version=$(git tag --points-at HEAD)
+        echo "Looks like you're trying to push a $version release..."
+        echo "Making sure you've signed and tagged it."
+        if verify_commit_signed && verify_tag && verify_tag_signed ; then
+            : # Ok, I guess you can push
+        else
+            exit 1
+        fi
+    fi
+done
+
+command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'pre-push' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; }
+
+git lfs pre-push "$@"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,6 +8,7 @@ assignees: ''
 ---
 
 <!-- Please search existing issues to avoid creating duplicates. -->
+<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
 
 ## Issue Description
 <!-- Provide a summary for your issue/bug. -->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -8,6 +8,7 @@ assignees: ''
 ---
 
 <!-- Please search existing issues to avoid creating duplicates. -->
+<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
 
 ## Summary
 <!-- Provide a summary to the feature request -->

.github/ISSUE_TEMPLATE/question.md

@@ -9,6 +9,7 @@ assignees: ''
 
 <!-- Please search existing issues to avoid creating duplicates. -->
 <!-- Consider starting a [discussion](https://github.com/XRPLF/clio/discussions) instead. -->
+<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
 
 ## Question
 <!-- Your question -->

.github/actions/build_docker_image/action.yml

@@ -0,0 +1,66 @@
+name: Build and push Docker image
+description: Build and push Docker image to DockerHub and GitHub Container Registry
+inputs:
+  image_name:
+    description: Name of the image to build
+    required: true
+  push_image:
+    description: Whether to push the image to the registry (true/false)
+    required: true
+  directory:
+    description: The directory containing the Dockerfile
+    required: true
+  tags:
+    description: Comma separated tags to apply to the image
+    required: true
+  platforms:
+    description: Platforms to build the image for (e.g. linux/amd64,linux/arm64)
+    required: true
+  description:
+    description: Short description of the image
+    required: true
+runs:
+  using: composite
+  steps:
+      - name: Login to DockerHub
+        if: ${{ inputs.push_image == 'true' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_PW }}
+
+      - name: Login to GitHub Container Registry
+        if: ${{ inputs.push_image == 'true' }}
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ env.GITHUB_TOKEN }}
+
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/metadata-action@v5
+        id: meta
+        with:
+          images: ${{ inputs.image_name }}
+          tags: ${{ inputs.tags }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: ${{ inputs.directory }}
+          platforms: ${{ inputs.platforms }}
+          push: ${{ inputs.push_image == 'true' }}
+          tags: ${{ steps.meta.outputs.tags }}
+
+      - name: Update DockerHub description
+        if: ${{ inputs.push_image == 'true' }}
+        uses: peter-evans/dockerhub-description@v4
+        with:
+          username: ${{ env.DOCKERHUB_USER }}
+          password: ${{ env.DOCKERHUB_PW }}
+          repository: ${{ inputs.image_name }}
+          short-description: ${{ inputs.description }}
+          readme-filepath: ${{ inputs.directory }}/README.md
+

.github/actions/code_coverage/action.yml

@@ -6,15 +6,19 @@ runs:
     - name: Run tests
       shell: bash
       run: |
-        build/clio_tests --backend_host=scylladb
+        build/clio_tests
 
     - name: Run gcovr
       shell: bash
       run: |
-        gcovr -e unittests --xml build/coverage_report.xml -j8 --exclude-throw-branches
+        gcovr -e tests \
+          -e src/data/cassandra \
+          -e src/data/CassandraBackend.hpp \
+          -e 'src/data/BackendFactory.*' \
+          --xml build/coverage_report.xml -j8 --exclude-throw-branches
 
     - name: Archive coverage report
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: coverage-report.xml
         path: build/coverage_report.xml

.github/actions/create_issue/action.yml

@@ -0,0 +1,35 @@
+name: Create an issue
+description: Create an issue
+inputs:
+  title:
+    description: Issue title
+    required: true
+  body:
+    description: Issue body
+    required: true
+  labels:
+    description: Comma-separated list of labels
+    required: true
+    default: 'bug'
+  assignees:
+    description: Comma-separated list of assignees
+    required: true
+    default: 'cindyyan317,godexsoft,kuznetsss'
+outputs:
+  created_issue_id:
+    description: Created issue id
+    value: ${{ steps.create_issue.outputs.created_issue }}
+runs:
+  using: composite
+  steps:
+      - name: Create an issue
+        id: create_issue
+        shell: bash
+        run: |
+          echo -e '${{ inputs.body }}' > issue.md
+          gh issue create --assignee '${{ inputs.assignees }}' --label '${{ inputs.labels }}' --title '${{ inputs.title }}' --body-file ./issue.md > create_issue.log
+          created_issue=$(cat create_issue.log | sed 's|.*/||')
+          echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
+          rm create_issue.log issue.md
+
+

.github/actions/generate/action.yml

@@ -16,6 +16,10 @@ inputs:
     description: Whether conan's coverage option should be on or not
     required: true
     default: 'false'
+  static:
+    description: Whether Clio is to be statically linked
+    required: true
+    default: 'false'
 runs:
   using: composite
   steps:
@@ -28,9 +32,10 @@ runs:
       env:
         BUILD_OPTION: "${{ inputs.conan_cache_hit == 'true' && 'missing' || '' }}"
         CODE_COVERAGE: "${{ inputs.code_coverage == 'true' && 'True' || 'False' }}"
+        STATIC_OPTION: "${{ inputs.static == 'true' && 'True' || 'False' }}"
       run: |
         cd build
-        conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:tests=True -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
+        conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:static="${STATIC_OPTION}" -o clio:tests=True -o clio:integration_tests=True -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }}
 
     - name: Run cmake
       shell: bash

.github/actions/prepare_runner/action.yml

@@ -12,9 +12,7 @@ runs:
       shell: bash
       run: |
           brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1
-          if ! command -v conan &> /dev/null; then
-            echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
-          fi
+          echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
 
     - name: Fix git permissions on Linux
       if: ${{ runner.os == 'Linux' }}

.github/actions/restore_cache/action.yml

@@ -4,6 +4,9 @@ inputs:
   conan_dir:
     description: Path to .conan directory
     required: true
+  conan_profile:
+    description: Conan profile name
+    required: true
   ccache_dir:
     description: Path to .ccache directory
     required: true
@@ -44,16 +47,16 @@ runs:
         echo "hash=$hash" >> $GITHUB_OUTPUT
 
     - name: Restore conan cache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       id: conan_cache
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ steps.conan_hash.outputs.hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ steps.conan_hash.outputs.hash }}
 
     - name: Restore ccache cache
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       id: ccache_cache
       if: ${{ env.CCACHE_DISABLE != '1' }}
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-${{ inputs.conan_profile }}-develop-${{ steps.git_common_ancestor.outputs.commit }}

.github/actions/save_cache/action.yml

@@ -4,6 +4,9 @@ inputs:
   conan_dir:
     description: Path to .conan directory
     required: true
+  conan_profile:
+    description: Conan profile name
+    required: true
   conan_hash:
     description: Hash to use as a part of conan cache key
     required: true
@@ -41,16 +44,16 @@ runs:
 
     - name: Save conan cache
       if: ${{ inputs.conan_cache_hit != 'true' }}
-      uses: actions/cache/save@v3
+      uses: actions/cache/save@v4
       with:
         path: ${{ inputs.conan_dir }}/data
-        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-develop-${{ inputs.conan_hash }}
+        key: clio-conan_data-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.conan_profile }}-develop-${{ inputs.conan_hash }}
 
     - name: Save ccache cache
       if: ${{ inputs.ccache_cache_hit != 'true' || inputs.ccache_cache_miss_rate == '100.0' }}
-      uses: actions/cache/save@v3
+      uses: actions/cache/save@v4
       with:
         path: ${{ inputs.ccache_dir }}
-        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
+        key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-${{ inputs.conan_profile }}-develop-${{ steps.git_common_ancestor.outputs.commit }}
 
 

.github/actions/setup_conan/action.yml

@@ -1,5 +1,9 @@
 name: Setup conan
 description: Setup conan profile and artifactory
+inputs:
+  conan_profile:
+    description: Conan profile name
+    required: true
 outputs:
   conan_profile:
     description: Created conan profile name
@@ -11,7 +15,7 @@ runs:
       if: ${{ runner.os == 'macOS' }}
       shell: bash
       env:
-        CONAN_PROFILE: clio_apple_clang_15
+        CONAN_PROFILE: apple_clang_15
       id: conan_setup_mac
       run: |
         echo "Creating $CONAN_PROFILE conan profile";
@@ -27,7 +31,7 @@ runs:
       shell: bash
       id: conan_setup_linux
       run: |
-        echo "created_conan_profile=default" >> $GITHUB_OUTPUT
+        echo "created_conan_profile=${{ inputs.conan_profile }}" >> $GITHUB_OUTPUT
 
     - name: Export output variable
       shell: bash

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "04:00"
+      timezone: "Etc/GMT"
+    reviewers:
+      - "cindyyan317"
+      - "godexsoft"
+      - "kuznetsss"
+    commit-message:
+      prefix: "[CI] "
+    target-branch: "develop"

.github/scripts/update-libxrpl-version

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Note: This script is intended to be run from the root of the repository.
+#
+# This script modifies conanfile.py such that the specified version of libXRPL is used.
+
+if [[ -z "$1" ]]; then
+    cat <<EOF
+
+                                    ERROR
+-----------------------------------------------------------------------------
+            Version should be passed as first argument to the script.
+-----------------------------------------------------------------------------
+
+EOF
+    exit 1
+fi
+
+VERSION=$1
+GNU_SED=$(sed --version 2>&1 | grep -q 'GNU' && echo true || echo false)
+
+echo "+ Updating required libXRPL version to $VERSION"
+
+if [[ "$GNU_SED" == "false" ]]; then
+    sed -i '' -E "s|'xrpl/[a-zA-Z0-9\\.\\-]+'|'xrpl/$VERSION'|g" conanfile.py
+else
+    sed -i -E "s|'xrpl/[a-zA-Z0-9\\.\\-]+'|'xrpl/$VERSION'|g" conanfile.py
+fi

.github/workflows/build.yml

@@ -13,13 +13,17 @@ jobs:
     container:
       image: rippleci/clio_ci:latest
     steps:
+      - name: Fix git permissions on Linux
+        shell: bash
+        run: git config --global --add safe.directory $PWD
+
       - uses: actions/checkout@v4
       - name: Run formatters
         id: run_formatters
         run: |
-          ./.githooks/check-format
+          ./.githooks/check-format --diff
         shell: bash
-  
+ 
   check_docs:
     name: Check documentation
     runs-on: ubuntu-20.04
@@ -35,8 +39,8 @@ jobs:
 
   build:
     name: Build
-    needs: 
-      - check_format 
+    needs:
+      - check_format
       - check_docs
     strategy:
       fail-fast: false
@@ -46,28 +50,42 @@ jobs:
             container:
               image: rippleci/clio_ci:latest
             build_type: Release
+            conan_profile: gcc
             code_coverage: false
+            static: true
           - os: heavy
             container:
               image: rippleci/clio_ci:latest
             build_type: Debug
+            conan_profile: gcc
             code_coverage: true
+            static: true
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            build_type: Release
+            conan_profile: clang
+            code_coverage: false
+            static: true
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            build_type: Debug
+            conan_profile: clang
+            code_coverage: false
+            static: true
           - os: macos14
             build_type: Release
             code_coverage: false
+            static: false
     runs-on: [self-hosted, "${{ matrix.os }}"]
     container: ${{ matrix.container }}
 
-    services:
-      scylladb:
-        image: ${{ (matrix.code_coverage) && 'scylladb/scylla' || '' }}
-        options: >-
-          --health-cmd "cqlsh -e 'describe cluster'" 
-          --health-interval 10s 
-          --health-timeout 5s 
-          --health-retries 5
-
     steps:
+      - name: Clean workdir
+        if: ${{ runner.os == 'macOS' }}
+        uses: kuznetsss/workspace-cleanup@1.0
+
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -80,12 +98,15 @@ jobs:
       - name: Setup conan
         uses: ./.github/actions/setup_conan
         id: conan
+        with:
+          conan_profile: ${{ matrix.conan_profile }}
 
       - name: Restore cache
         uses: ./.github/actions/restore_cache
         id: restore_cache
         with:
           conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
           ccache_dir: ${{ env.CCACHE_DIR }}
           build_type: ${{ matrix.build_type }}
           code_coverage: ${{ matrix.code_coverage }}
@@ -97,6 +118,7 @@ jobs:
           conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
           build_type: ${{ matrix.build_type }}
           code_coverage: ${{ matrix.code_coverage }}
+          static: ${{ matrix.static }}
 
       - name: Build Clio
         uses: ./.github/actions/build_clio
@@ -112,20 +134,20 @@ jobs:
 
       - name: Strip tests
         if: ${{ !matrix.code_coverage }}
-        run: strip build/clio_tests
+        run: strip build/clio_tests && strip build/clio_integration_tests
 
       - name: Upload clio_server
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: clio_server_${{ runner.os }}_${{ matrix.build_type }}
+          name: clio_server_${{ runner.os }}_${{ matrix.build_type }}_${{ steps.conan.outputs.conan_profile }}
           path: build/clio_server
 
       - name: Upload clio_tests
         if: ${{ !matrix.code_coverage }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: clio_tests_${{ runner.os }}
-          path: build/clio_tests
+          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ steps.conan.outputs.conan_profile }}
+          path: build/clio_*tests
 
       - name: Save cache
         uses: ./.github/actions/save_cache
@@ -138,6 +160,7 @@ jobs:
           ccache_cache_miss_rate: ${{ steps.ccache_stats.outputs.miss_rate }}
           build_type: ${{ matrix.build_type }}
           code_coverage: ${{ matrix.code_coverage }}
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
 
       # TODO: This is not a part of build process but it is the easiest way to do it here.
       # It will be refactored in https://github.com/XRPLF/clio/issues/1075
@@ -162,15 +185,34 @@ jobs:
           - os: heavy
             container:
               image: rippleci/clio_ci:latest
+            conan_profile: gcc
+            build_type: Release
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            conan_profile: clang
+            build_type: Release
+          - os: heavy
+            container:
+              image: rippleci/clio_ci:latest
+            conan_profile: clang
+            build_type: Debug
           - os: macos14
+            conan_profile: apple_clang_15
+            build_type: Release
     runs-on: [self-hosted, "${{ matrix.os }}"]
     container: ${{ matrix.container }}
 
     steps:
-      - uses: actions/download-artifact@v3
+      - name: Clean workdir
+        if: ${{ runner.os == 'macOS' }}
+        uses: kuznetsss/workspace-cleanup@1.0
+
+      - uses: actions/download-artifact@v4
         with:
-          name: clio_tests_${{ runner.os }}
+          name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
+
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests
-          ./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
+          ./clio_tests

.github/workflows/build_clio_docker_image.yml

@@ -0,0 +1,95 @@
+name: Build and publish Clio docker image
+on:
+  workflow_call:
+    inputs:
+      tags:
+        required: true
+        type: string
+        description: Comma separated tags for docker image
+      artifact_name:
+        type: string
+        description: Name of Github artifact to put into docker image
+      strip_binary:
+        type: boolean
+        description: Whether to strip clio binary
+        default: true
+      publish_image:
+        type: boolean
+        description: Whether to publish docker image
+        required: true
+
+  workflow_dispatch:
+    inputs:
+      tags:
+        required: true
+        type: string
+        description: Comma separated tags for docker image
+      clio_server_binary_url:
+        required: true
+        type: string
+        description: Url to download clio_server binary from
+      binary_sha256:
+        required: true
+        type: string
+        description: sha256 hash of the binary
+      strip_binary:
+        type: boolean
+        description: Whether to strip clio binary
+        default: true
+
+jobs:
+  build_and_publish_image:
+    name: Build and publish image
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download Clio binary from artifact
+        if: ${{ inputs.artifact_name != null }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.artifact_name }}
+          path: ./docker/clio/artifact/
+
+      - name: Download Clio binary from url
+        if: ${{ inputs.clio_server_binary_url != null }}
+        shell: bash
+        run: |
+          wget ${{inputs.clio_server_binary_url}} -P ./docker/clio/artifact/
+          if [ "$(sha256sum ./docker/clio/clio_server | awk '{print $1}')" != "${{inputs.binary_sha256}}" ]; then
+            echo "Binary sha256 sum doesn't match"
+            exit 1
+          fi
+      - name: Unpack binary
+        shell: bash
+        run: |
+          sudo apt update && sudo apt install -y tar unzip
+          cd docker/clio/artifact
+          artifact=$(find . -type f)
+          if [[ $artifact == *.zip ]]; then
+            unzip $artifact
+          elif [[ $artifact == *.tar.gz ]]; then
+            tar -xvf $artifact
+          fi
+          mv clio_server ../
+          cd ../
+          rm -rf ./artifact
+
+      - name: Strip binary
+        if: ${{ inputs.strip_binary }}
+        shell: bash
+        run: strip ./docker/clio/clio_server
+
+      - name: Build Docker image
+        uses: ./.github/actions/build_docker_image
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          image_name: rippleci/clio
+          push_image: ${{ inputs.publish_image }}
+          directory: docker/clio
+          tags: ${{ inputs.tags }}
+          platforms: linux/amd64
+          description: Clio is an XRP Ledger API server.

.github/workflows/check_libxrpl.yml

@@ -0,0 +1,91 @@
+name: Check new libXRPL
+on:
+  repository_dispatch:
+    types: [check_libxrpl]
+
+jobs:
+  build:
+    name: Build Clio / `libXRPL ${{ github.event.client_payload.version }}`
+    runs-on: [self-hosted, heavy]
+    container:
+      image: rippleci/clio_ci:latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            fetch-depth: 0
+
+      - name: Update libXRPL version requirement
+        shell: bash
+        run: |
+          ./.github/scripts/update-libxrpl-version ${{ github.event.client_payload.version }}
+
+      - name: Prepare runner
+        uses: ./.github/actions/prepare_runner
+        with:
+            disable_ccache: true
+
+      - name: Setup conan
+        uses: ./.github/actions/setup_conan
+        id: conan
+        with:
+          conan_profile: gcc
+
+      - name: Run conan and cmake
+        uses: ./.github/actions/generate
+        with:
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
+          conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
+          build_type: Release
+
+      - name: Build Clio
+        uses: ./.github/actions/build_clio
+
+      - name: Strip tests
+        run: strip build/clio_tests
+
+      - name: Upload clio_tests
+        uses: actions/upload-artifact@v4
+        with:
+          name: clio_tests_check_libxrpl
+          path: build/clio_tests
+
+  run_tests:
+    name: Run tests
+    needs: build
+    runs-on: [self-hosted, heavy]
+    container:
+      image: rippleci/clio_ci:latest
+
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: clio_tests_check_libxrpl
+
+      - name: Run clio_tests
+        run: |
+          chmod +x ./clio_tests
+          ./clio_tests
+
+  create_issue_on_failure:
+    name: Create an issue on failure
+    needs: [build, run_tests]
+    if: ${{ always() && contains(needs.*.result, 'failure') }}
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create an issue
+        uses: ./.github/actions/create_issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          labels: 'compatibility,bug'
+          title: 'Proposed libXRPL check failed'
+          body: >
+            Clio build or tests failed against `libXRPL ${{ github.event.client_payload.version }}`.
+
+            Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/

.github/workflows/check_pr_title.yml

@@ -0,0 +1,18 @@
+name: Check PR title
+on:
+  pull_request:
+    types: [opened, edited, reopened, synchronize]
+    branches: [develop]
+
+jobs:
+  check_title:
+    runs-on: ubuntu-20.04
+    # permissions:
+    #   pull-requests: write
+    steps:
+      - uses: ytanikin/PRConventionalCommits@1.3.0
+        with:
+          task_types: '["build","feat","fix","docs","test","ci","style","refactor","perf","chore"]'
+          add_label: false
+          # Turned off labelling because it leads to an error, see https://github.com/ytanikin/PRConventionalCommits/issues/19
+          # custom_labels: '{"build":"build", "feat":"enhancement", "fix":"bug", "docs":"documentation", "test":"testability", "ci":"ci", "style":"refactoring", "refactor":"refactoring", "perf":"performance", "chore":"tooling"}'

.github/workflows/clang-tidy.yml

@@ -33,6 +33,8 @@ jobs:
       - name: Setup conan
         uses: ./.github/actions/setup_conan
         id: conan
+        with:
+          conan_profile: clang
 
       - name: Restore cache
         uses: ./.github/actions/restore_cache
@@ -40,6 +42,7 @@ jobs:
         with:
           conan_dir: ${{ env.CONAN_USER_HOME }}/.conan
           ccache_dir: ${{ env.CCACHE_DIR }}
+          conan_profile: ${{ steps.conan.outputs.conan_profile }}
 
       - name: Run conan and cmake
         uses: ./.github/actions/generate
@@ -57,7 +60,7 @@ jobs:
         shell: bash
         id: run_clang_tidy
         run: |
-          run-clang-tidy-17 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
+          run-clang-tidy-19 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
 
       - name: Check format
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
@@ -76,18 +79,17 @@ jobs:
       - name: Create an issue
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
         id: create_issue
-        shell: bash
+        uses: ./.github/actions/create_issue
         env:
           GH_TOKEN: ${{ github.token }}
-        run: |
-          echo -e 'Clang-tidy found issues in the code:\n' > issue.md
-          echo -e "List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/" >> issue.md
-          gh issue create --assignee 'cindyyan317,godexsoft,kuznetsss' --label bug --title 'Clang-tidy found bugs in code🐛' --body-file ./issue.md > create_issue.log
-          created_issue=$(cat create_issue.log | sed 's|.*/||')
-          echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
-          rm create_issue.log issue.md
+        with:
+          title: 'Clang-tidy found bugs in code 🐛'
+          body: >
+            Clang-tidy found issues in the code:
 
-      - uses: crazy-max/ghaction-import-gpg@v5
+            List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/
+
+      - uses: crazy-max/ghaction-import-gpg@v6
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
         with:
           gpg_private_key: ${{ secrets.ACTIONS_GPG_PRIVATE_KEY }}
@@ -97,7 +99,7 @@ jobs:
 
       - name: Create PR with fixes
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v7
         env:
           GH_REPO: ${{ github.repository }}
           GH_TOKEN: ${{ github.token }}
@@ -107,8 +109,8 @@ jobs:
           branch: "clang_tidy/autofix"
           branch-suffix: timestamp
           delete-branch: true
-          title: "[CI] clang-tidy auto fixes"
-          body: "Fixes #${{ steps.create_issue.outputs.created_issue }}. Please review and commit clang-tidy fixes."
+          title: "style: clang-tidy auto fixes"
+          body: "Fixes #${{ steps.create_issue.outputs.created_issue_id }}. Please review and commit clang-tidy fixes."
           reviewers: "cindyyan317,godexsoft,kuznetsss"
 
       - name: Fail the job

.github/workflows/clang-tidy_on_fix_merged.yml

@@ -17,7 +17,7 @@ jobs:
         id: check
         shell: bash
         run: |
-          passed=$(if [[ $(git log -1 --pretty=format:%s | grep '\[CI\] clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi)
+          passed=$(if [[ $(git log -1 --pretty=format:%s | grep 'style: clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi)
           echo "passed=$passed" >> $GITHUB_OUTPUT
 
       - name: Run clang-tidy workflow

.github/workflows/docs.yml

@@ -1,7 +1,7 @@
 name: Documentation
 on:
   push:
-    branches: [release/*, develop]
+    branches: [develop]
   workflow_dispatch:
 
 permissions:
@@ -24,7 +24,9 @@ jobs:
       image: rippleci/clio_ci:latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with: 
+          lfs: true
 
       - name: Build docs
         run: |
@@ -32,13 +34,13 @@ jobs:
           cmake ../docs && cmake --build . --target docs
       
       - name: Setup Pages
-        uses: actions/configure-pages@v3
+        uses: actions/configure-pages@v5
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
           path: build_docs/html
-          name: docs-develop # TODO: use x.y.z for `release/x.y.z` branches and `develop` for latest dev docs
+          name: docs-develop
       
       - name: Deploy to GitHub Pages
         id: deployment

.github/workflows/nightly.yml

@@ -3,6 +3,10 @@ on:
   schedule:
     - cron:  '0 5 * * 1-5'
   workflow_dispatch:
+  pull_request:
+    paths:
+      - '.github/workflows/nightly.yml'
+      - '.github/workflows/build_clio_docker_image.yml'
 
 jobs:
   build:
@@ -13,18 +17,25 @@ jobs:
         include:
           - os: macos14
             build_type: Release
+            static: false
           - os: heavy
             build_type: Release
+            static: true
             container:
               image: rippleci/clio_ci:latest
           - os: heavy
             build_type: Debug
+            static: true
             container:
               image: rippleci/clio_ci:latest
     runs-on: [self-hosted, "${{ matrix.os }}"]
     container: ${{ matrix.container }}
 
     steps:
+      - name: Clean workdir
+        if: ${{ runner.os == 'macOS' }}
+        uses: kuznetsss/workspace-cleanup@1.0
+
       - uses: actions/checkout@v4
         with:
             fetch-depth: 0
@@ -37,6 +48,8 @@ jobs:
       - name: Setup conan
         uses: ./.github/actions/setup_conan
         id: conan
+        with:
+          conan_profile: gcc
 
       - name: Run conan and cmake
         uses: ./.github/actions/generate
@@ -44,18 +57,19 @@ jobs:
           conan_profile: ${{ steps.conan.outputs.conan_profile }}
           conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }}
           build_type: ${{ matrix.build_type }}
+          static: ${{ matrix.static }}
 
       - name: Build Clio
         uses: ./.github/actions/build_clio
 
       - name: Strip tests
-        run: strip build/clio_tests
+        run: strip build/clio_tests && strip build/clio_integration_tests
 
       - name: Upload clio_tests
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
-          path: build/clio_tests
+          path: build/clio_*tests
 
       - name: Compress clio_server
         shell: bash
@@ -64,7 +78,7 @@ jobs:
             tar czf ./clio_server_${{ runner.os }}_${{ matrix.build_type }}.tar.gz ./clio_server
 
       - name: Upload clio_server
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: clio_server_${{ runner.os }}_${{ matrix.build_type }}
           path: build/clio_server_${{ runner.os }}_${{ matrix.build_type }}.tar.gz
@@ -78,23 +92,53 @@ jobs:
         include:
           - os: macos14
             build_type: Release
+            integration_tests: false
           - os: heavy
             build_type: Release
+            container:
+              image: rippleci/clio_ci:latest
+            integration_tests: true  
           - os: heavy
             build_type: Debug
+            container:
+              image: rippleci/clio_ci:latest
+            integration_tests: true  
     runs-on: [self-hosted, "${{ matrix.os }}"]
+    container: ${{ matrix.container }}
+
+    services:
+      scylladb:
+        image: ${{ (matrix.integration_tests) && 'scylladb/scylla' || '' }}
+        options: >-
+          --health-cmd "cqlsh -e 'describe cluster'" 
+          --health-interval 10s 
+          --health-timeout 5s 
+          --health-retries 5
 
     steps:
-      - uses: actions/download-artifact@v3
+      - name: Clean workdir
+        if: ${{ runner.os == 'macOS' }}
+        uses: kuznetsss/workspace-cleanup@1.0
+
+      - uses: actions/download-artifact@v4
         with:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
 
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests
-          ./clio_tests --gtest_filter="-BackendCassandraBaseTest*:BackendCassandraTest*:BackendCassandraFactoryTestWithDB*"
+          ./clio_tests
+  
+  # To be enabled back once docker in mac runner arrives
+  # https://github.com/XRPLF/clio/issues/1400
+      - name: Run clio_integration_tests
+        if: matrix.integration_tests 
+        run: |
+          chmod +x ./clio_integration_tests
+          ./clio_integration_tests --backend_host=scylladb
 
   nightly_release:
+    if: ${{ github.event_name != 'pull_request' }}
     needs: run_tests
     runs-on: ubuntu-20.04
     env:
@@ -105,16 +149,16 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           path: nightly_release
+          pattern: clio_server_*
 
       - name: Prepare files
         shell: bash
         run: |
           cp ${{ github.workspace }}/.github/workflows/nightly_notes.md "${RUNNER_TEMP}/nightly_notes.md"
           cd nightly_release
-          rm -r clio_tests*
           for d in $(ls); do
             archive_name=$(ls $d)
             mv ${d}/${archive_name} ./
@@ -136,3 +180,36 @@ jobs:
           gh release create nightly --prerelease --title "Clio development (nightly) build" \
             --target $GITHUB_SHA --notes-file "${RUNNER_TEMP}/nightly_notes.md" \
             ./nightly_release/clio_server*
+
+  build_and_publish_docker_image:
+    uses: ./.github/workflows/build_clio_docker_image.yml
+    needs: run_tests
+    secrets: inherit
+    with:
+      tags: |
+          type=raw,value=nightly
+          type=raw,value=${{ github.sha }}
+      artifact_name: clio_server_Linux_Release
+      strip_binary: true
+      publish_image: ${{ github.event_name != 'pull_request' }}
+
+  create_issue_on_failure:
+    needs: [build, run_tests, nightly_release, build_and_publish_docker_image]
+    if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name != 'pull_request' }}
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Create an issue
+        uses: ./.github/actions/create_issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          title: 'Nightly release failed 🌙'
+          body: >
+            Nightly release failed:
+
+            Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/

.github/workflows/update_docker_ci.yml

@@ -1,40 +1,36 @@
 name: Update CI docker image
 on:
+  pull_request:
+    paths:
+      - 'docker/ci/**'
+      - 'docker/compilers/**'
+      - .github/workflows/update_docker_ci.yml
   push:
     branches: [develop]
     paths:
-      - 'docker/ci/**'
+      - 'docker/ci/**'        # CI image must update when either its dockerfile changes
+      - 'docker/compilers/**' # or any compilers changed and were pushed by hand
       - .github/workflows/update_docker_ci.yml
   workflow_dispatch:
 
 jobs:
   build_and_push:
     name: Build and push docker image
-    runs-on: ubuntu-20.04
+    runs-on: [self-hosted, heavy]
     steps:
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PW }}
-
       - uses: actions/checkout@v4
-      - uses: docker/setup-qemu-action@v3
-      - uses: docker/setup-buildx-action@v3
-      - uses: docker/metadata-action@v5
-        id: meta
+      - uses: ./.github/actions/build_docker_image
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          images: rippleci/clio_ci
+          image_name: rippleci/clio_ci
+          push_image: ${{ github.event_name != 'pull_request' }}
+          directory: docker/ci
           tags: |
             type=raw,value=latest
-            type=raw,value=gcc_11
-            type=raw,value=${{ env.GITHUB_SHA }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: ${{ github.workspace }}/docker/ci
+            type=raw,value=gcc_12_clang_16
+            type=raw,value=${{ github.sha }}
           platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-
+          description: CI image for XRPLF/clio.

.github/workflows/upload_coverage_report.yml

@@ -16,16 +16,16 @@ jobs:
           fetch-depth: 0
 
       - name: Download report artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: coverage-report.xml
           path: build
 
       - name: Upload coverage report
         if: ${{ hashFiles('build/coverage_report.xml') != '' }}
-        uses: wandalen/wretry.action@v1.3.0
+        uses: wandalen/wretry.action@v3.7.2
         with:
-          action: codecov/codecov-action@v3
+          action: codecov/codecov-action@v4
           with: |
             files: build/coverage_report.xml 
             fail_ci_if_error: false

.gitignore

@@ -8,4 +8,4 @@
 .DS_Store
 CMakeUserPresets.json
 config.json
-src/main/impl/Build.cpp
+src/util/build/Build.cpp

CMake/deps/Boost.cmake

@@ -1,4 +0,0 @@
-set(Boost_USE_STATIC_LIBS ON)
-set(Boost_USE_STATIC_RUNTIME ON)
-
-find_package(Boost 1.82 REQUIRED COMPONENTS program_options coroutine system log log_setup)

CMake/deps/cassandra.cmake

@@ -1 +0,0 @@
-find_package(cassandra-cpp-driver REQUIRED)

CMake/deps/gbench.cmake

@@ -1 +0,0 @@
-find_package(benchmark REQUIRED)

CMake/deps/libbacktrace.cmake

@@ -1,3 +0,0 @@
-target_compile_definitions(clio PUBLIC BOOST_STACKTRACE_LINK)
-target_compile_definitions(clio PUBLIC BOOST_STACKTRACE_USE_BACKTRACE)
-find_package(libbacktrace REQUIRED)

CMake/deps/libfmt.cmake

@@ -1 +0,0 @@
-find_package(fmt REQUIRED)

CMake/deps/libxrpl.cmake

@@ -1 +0,0 @@
-find_package(xrpl REQUIRED)

CMakeLists.txt

@@ -1,30 +1,37 @@
-cmake_minimum_required(VERSION 3.16.3)
-project(clio)
+cmake_minimum_required(VERSION 3.20)
+
+set(CMAKE_PROJECT_INCLUDE_BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ClioVersion.cmake)
+
+project(clio VERSION ${CLIO_VERSION} HOMEPAGE_URL "https://github.com/XRPLF/clio"
+        DESCRIPTION "An XRP Ledger API Server"
+)
 
 # =========================== Options ====================================== #
 option(verbose "Verbose build" FALSE)
-option(tests "Build tests" FALSE)
+option(tests "Build unit tests" FALSE)
+option(integration_tests "Build integration tests" FALSE)
 option(benchmark "Build benchmarks" FALSE)
 option(docs "Generate doxygen docs" FALSE)
 option(coverage "Build test coverage report" FALSE)
 option(packaging "Create distribution packages" FALSE)
 option(lint "Run clang-tidy checks during compilation" FALSE)
+option(static "Statically linked Clio" FALSE)
 # ========================================================================== #
 set(san "" CACHE STRING "Add sanitizer instrumentation")
 set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE)
 set_property(CACHE san PROPERTY STRINGS ";undefined;memory;address;thread")
 # ========================================================================== #
 
-# Include required modules
-include(CMake/Ccache.cmake)
-include(CheckCXXCompilerFlag)
-include(CMake/ClangTidy.cmake)
+set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
 
-# Set coverage build options
-if (tests AND coverage)
-  include(CMake/CodeCoverage.cmake)
-  append_coverage_compiler_flags()
-endif ()
+# Include required modules
+include(Ccache)
+include(CheckCXXCompilerFlag)
+include(ClangTidy)
+
+add_library(clio_options INTERFACE)
+target_compile_features(clio_options INTERFACE cxx_std_23) # Clio needs c++23 but deps can remain c++20 for now
+target_include_directories(clio_options INTERFACE ${CMAKE_SOURCE_DIR}/src)
 
 if (verbose)
   set(CMAKE_VERBOSE_MAKEFILE TRUE)
@@ -32,414 +39,28 @@ endif ()
 
 if (packaging)
   add_definitions(-DPKG=1)
+  target_compile_definitions(clio_options INTERFACE PKG=1)
 endif ()
 
-add_library(clio)
-
 # Clio tweaks and checks
-include(CMake/CheckCompiler.cmake)
-include(CMake/Settings.cmake)
-include(CMake/ClioVersion.cmake)
-include(CMake/SourceLocation.cmake)
+include(CheckCompiler)
+include(Settings)
+include(SourceLocation)
 
 # Clio deps
-include(CMake/deps/libxrpl.cmake)
-include(CMake/deps/Boost.cmake)
-include(CMake/deps/OpenSSL.cmake)
-include(CMake/deps/Threads.cmake)
-include(CMake/deps/libfmt.cmake)
-include(CMake/deps/cassandra.cmake)
-include(CMake/deps/libbacktrace.cmake)
+include(deps/libxrpl)
+include(deps/Boost)
+include(deps/OpenSSL)
+include(deps/Threads)
+include(deps/libfmt)
+include(deps/cassandra)
+include(deps/libbacktrace)
 
-# TODO: Include directory will be wrong when installed.
-target_include_directories(clio PUBLIC src)
-target_compile_features(clio PUBLIC cxx_std_20)
+add_subdirectory(src)
+add_subdirectory(tests)
 
-target_link_libraries(
-  clio
-  PUBLIC Boost::boost
-  PUBLIC Boost::coroutine
-  PUBLIC Boost::program_options
-  PUBLIC Boost::system
-  PUBLIC Boost::log
-  PUBLIC Boost::log_setup
-  PUBLIC Boost::stacktrace_backtrace
-  PUBLIC cassandra-cpp-driver::cassandra-cpp-driver
-  PUBLIC fmt::fmt
-  PUBLIC OpenSSL::Crypto
-  PUBLIC OpenSSL::SSL
-  PUBLIC xrpl::libxrpl
-  PUBLIC dl
-  PUBLIC libbacktrace::libbacktrace
-  INTERFACE Threads::Threads
-)
-
-if (is_gcc)
-  # FIXME: needed on gcc for now
-  target_compile_definitions(clio PUBLIC BOOST_ASIO_DISABLE_CONCEPTS)
-endif ()
-
-target_sources(
-  clio
-  PRIVATE # Main
-          src/main/impl/Build.cpp
-          # Backend
-          src/data/BackendCounters.cpp
-          src/data/BackendInterface.cpp
-          src/data/LedgerCache.cpp
-          src/data/cassandra/impl/Future.cpp
-          src/data/cassandra/impl/Cluster.cpp
-          src/data/cassandra/impl/Batch.cpp
-          src/data/cassandra/impl/Result.cpp
-          src/data/cassandra/impl/Tuple.cpp
-          src/data/cassandra/impl/SslContext.cpp
-          src/data/cassandra/Handle.cpp
-          src/data/cassandra/SettingsProvider.cpp
-          # ETL
-          src/etl/NFTHelpers.cpp
-          src/etl/ETLService.cpp
-          src/etl/ETLState.cpp
-          src/etl/LoadBalancer.cpp
-          src/etl/CacheLoaderSettings.cpp
-          src/etl/Source.cpp
-          src/etl/impl/ForwardingCache.cpp
-          src/etl/impl/ForwardingSource.cpp
-          src/etl/impl/GrpcSource.cpp
-          src/etl/impl/SubscriptionSource.cpp
-          # Feed
-          src/feed/SubscriptionManager.cpp
-          src/feed/impl/TransactionFeed.cpp
-          src/feed/impl/LedgerFeed.cpp
-          src/feed/impl/ProposedTransactionFeed.cpp
-          src/feed/impl/SingleFeedBase.cpp
-          # Web
-          src/web/impl/AdminVerificationStrategy.cpp
-          src/web/IntervalSweepHandler.cpp
-          src/web/Resolver.cpp
-          # RPC
-          src/rpc/Errors.cpp
-          src/rpc/Factories.cpp
-          src/rpc/AMMHelpers.cpp
-          src/rpc/RPCHelpers.cpp
-          src/rpc/Counters.cpp
-          src/rpc/WorkQueue.cpp
-          src/rpc/common/Specs.cpp
-          src/rpc/common/Validators.cpp
-          src/rpc/common/MetaProcessors.cpp
-          src/rpc/common/impl/APIVersionParser.cpp
-          src/rpc/common/impl/HandlerProvider.cpp
-          # RPC handlers
-          src/rpc/handlers/AccountChannels.cpp
-          src/rpc/handlers/AccountCurrencies.cpp
-          src/rpc/handlers/AccountInfo.cpp
-          src/rpc/handlers/AccountLines.cpp
-          src/rpc/handlers/AccountNFTs.cpp
-          src/rpc/handlers/AccountObjects.cpp
-          src/rpc/handlers/AccountOffers.cpp
-          src/rpc/handlers/AccountTx.cpp
-          src/rpc/handlers/AMMInfo.cpp
-          src/rpc/handlers/BookChanges.cpp
-          src/rpc/handlers/BookOffers.cpp
-          src/rpc/handlers/DepositAuthorized.cpp
-          src/rpc/handlers/GatewayBalances.cpp
-          src/rpc/handlers/Ledger.cpp
-          src/rpc/handlers/LedgerData.cpp
-          src/rpc/handlers/LedgerEntry.cpp
-          src/rpc/handlers/LedgerRange.cpp
-          src/rpc/handlers/NFTsByIssuer.cpp
-          src/rpc/handlers/NFTBuyOffers.cpp
-          src/rpc/handlers/NFTHistory.cpp
-          src/rpc/handlers/NFTInfo.cpp
-          src/rpc/handlers/NFTOffersCommon.cpp
-          src/rpc/handlers/NFTSellOffers.cpp
-          src/rpc/handlers/NoRippleCheck.cpp
-          src/rpc/handlers/Random.cpp
-          src/rpc/handlers/TransactionEntry.cpp
-          # Util
-          src/util/config/Config.cpp
-          src/util/log/Logger.cpp
-          src/util/prometheus/Http.cpp
-          src/util/prometheus/Label.cpp
-          src/util/prometheus/MetricBase.cpp
-          src/util/prometheus/MetricBuilder.cpp
-          src/util/prometheus/MetricsFamily.cpp
-          src/util/prometheus/OStream.cpp
-          src/util/prometheus/Prometheus.cpp
-          src/util/Random.cpp
-          src/util/Retry.cpp
-          src/util/requests/RequestBuilder.cpp
-          src/util/requests/Types.cpp
-          src/util/requests/WsConnection.cpp
-          src/util/requests/impl/SslContext.cpp
-          src/util/Taggable.cpp
-          src/util/TerminationHandler.cpp
-          src/util/TxUtils.cpp
-          src/util/LedgerUtils.cpp
-)
-
-# Clio server
-add_executable(clio_server src/main/Main.cpp)
-target_link_libraries(clio_server PRIVATE clio)
-target_link_options(
-  clio_server PRIVATE $<$<AND:$<NOT:$<BOOL:${APPLE}>>,$<NOT:$<BOOL:${san}>>>:-static-libstdc++ -static-libgcc>
-)
-
-# Unittesting
-if (tests)
-  set(TEST_TARGET clio_tests)
-  add_executable(
-    ${TEST_TARGET}
-    # Common
-    unittests/ConfigTests.cpp
-    unittests/data/BackendCountersTests.cpp
-    unittests/data/BackendCountersTests.cpp
-    unittests/data/BackendFactoryTests.cpp
-    unittests/data/BackendFactoryTests.cpp
-    unittests/data/cassandra/AsyncExecutorTests.cpp
-    # Webserver
-    unittests/data/cassandra/AsyncExecutorTests.cpp
-    # Webserver
-    unittests/data/cassandra/BackendTests.cpp
-    unittests/data/cassandra/BackendTests.cpp
-    unittests/data/cassandra/BaseTests.cpp
-    unittests/data/cassandra/BaseTests.cpp
-    unittests/data/cassandra/ExecutionStrategyTests.cpp
-    unittests/data/cassandra/ExecutionStrategyTests.cpp
-    unittests/data/cassandra/RetryPolicyTests.cpp
-    unittests/data/cassandra/RetryPolicyTests.cpp
-    unittests/data/cassandra/SettingsProviderTests.cpp
-    unittests/data/cassandra/SettingsProviderTests.cpp
-    unittests/DOSGuardTests.cpp
-    unittests/etl/AmendmentBlockHandlerTests.cpp
-    unittests/etl/AmendmentBlockHandlerTests.cpp
-    unittests/etl/CacheLoaderSettingsTests.cpp
-    unittests/etl/CacheLoaderTests.cpp
-    unittests/etl/CacheLoaderTests.cpp
-    unittests/etl/CursorProviderTests.cpp
-    unittests/etl/ETLStateTests.cpp
-    unittests/etl/ETLStateTests.cpp
-    unittests/etl/ExtractionDataPipeTests.cpp
-    unittests/etl/ExtractionDataPipeTests.cpp
-    unittests/etl/ExtractorTests.cpp
-    unittests/etl/ExtractorTests.cpp
-    unittests/etl/ForwardingCacheTests.cpp
-    unittests/etl/ForwardingSourceTests.cpp
-    unittests/etl/ForwardingSourceTests.cpp
-    unittests/etl/GrpcSourceTests.cpp
-    unittests/etl/GrpcSourceTests.cpp
-    unittests/etl/LedgerPublisherTests.cpp
-    unittests/etl/LedgerPublisherTests.cpp
-    unittests/etl/SourceTests.cpp
-    unittests/etl/SourceTests.cpp
-    unittests/etl/SubscriptionSourceDependenciesTests.cpp
-    unittests/etl/SubscriptionSourceDependenciesTests.cpp
-    unittests/etl/SubscriptionSourceTests.cpp
-    unittests/etl/SubscriptionSourceTests.cpp
-    unittests/etl/TransformerTests.cpp
-    # RPC
-    unittests/etl/TransformerTests.cpp
-    # RPC
-    unittests/feed/BookChangesFeedTests.cpp
-    unittests/feed/ForwardFeedTests.cpp
-    unittests/feed/LedgerFeedTests.cpp
-    unittests/feed/ProposedTransactionFeedTests.cpp
-    unittests/feed/SingleFeedBaseTests.cpp
-    unittests/feed/SubscriptionManagerTests.cpp
-    unittests/feed/TrackableSignalTests.cpp
-    unittests/feed/TransactionFeedTests.cpp
-    unittests/JsonUtilTests.cpp
-    unittests/LoggerTests.cpp
-    unittests/Main.cpp
-    unittests/Playground.cpp
-    unittests/ProfilerTests.cpp
-    unittests/rpc/AmendmentsTests.cpp
-    unittests/rpc/AmendmentsTests.cpp
-    unittests/rpc/APIVersionTests.cpp
-    unittests/rpc/APIVersionTests.cpp
-    unittests/rpc/BaseTests.cpp
-    unittests/rpc/BaseTests.cpp
-    unittests/rpc/CountersTests.cpp
-    unittests/rpc/CountersTests.cpp
-    unittests/rpc/ErrorTests.cpp
-    unittests/rpc/ErrorTests.cpp
-    unittests/rpc/ForwardingProxyTests.cpp
-    unittests/rpc/ForwardingProxyTests.cpp
-    unittests/rpc/handlers/AccountChannelsTests.cpp
-    unittests/rpc/handlers/AccountChannelsTests.cpp
-    unittests/rpc/handlers/AccountCurrenciesTests.cpp
-    unittests/rpc/handlers/AccountCurrenciesTests.cpp
-    unittests/rpc/handlers/AccountInfoTests.cpp
-    unittests/rpc/handlers/AccountInfoTests.cpp
-    unittests/rpc/handlers/AccountLinesTests.cpp
-    unittests/rpc/handlers/AccountLinesTests.cpp
-    unittests/rpc/handlers/AccountNFTsTests.cpp
-    unittests/rpc/handlers/AccountNFTsTests.cpp
-    unittests/rpc/handlers/AccountObjectsTests.cpp
-    unittests/rpc/handlers/AccountObjectsTests.cpp
-    unittests/rpc/handlers/AccountOffersTests.cpp
-    unittests/rpc/handlers/AccountOffersTests.cpp
-    unittests/rpc/handlers/AccountTxTests.cpp
-    unittests/rpc/handlers/AccountTxTests.cpp
-    unittests/rpc/handlers/AMMInfoTests.cpp
-    # Backend
-    unittests/rpc/handlers/AMMInfoTests.cpp
-    # Backend
-    unittests/rpc/handlers/BookChangesTests.cpp
-    unittests/rpc/handlers/BookChangesTests.cpp
-    unittests/rpc/handlers/BookOffersTests.cpp
-    unittests/rpc/handlers/BookOffersTests.cpp
-    unittests/rpc/handlers/DefaultProcessorTests.cpp
-    unittests/rpc/handlers/DefaultProcessorTests.cpp
-    unittests/rpc/handlers/DepositAuthorizedTests.cpp
-    unittests/rpc/handlers/DepositAuthorizedTests.cpp
-    unittests/rpc/handlers/GatewayBalancesTests.cpp
-    unittests/rpc/handlers/GatewayBalancesTests.cpp
-    unittests/rpc/handlers/LedgerDataTests.cpp
-    unittests/rpc/handlers/LedgerDataTests.cpp
-    unittests/rpc/handlers/LedgerEntryTests.cpp
-    unittests/rpc/handlers/LedgerEntryTests.cpp
-    unittests/rpc/handlers/LedgerRangeTests.cpp
-    unittests/rpc/handlers/LedgerRangeTests.cpp
-    unittests/rpc/handlers/LedgerTests.cpp
-    unittests/rpc/handlers/LedgerTests.cpp
-    unittests/rpc/handlers/NFTBuyOffersTests.cpp
-    unittests/rpc/handlers/NFTBuyOffersTests.cpp
-    unittests/rpc/handlers/NFTHistoryTests.cpp
-    unittests/rpc/handlers/NFTHistoryTests.cpp
-    unittests/rpc/handlers/NFTInfoTests.cpp
-    unittests/rpc/handlers/NFTInfoTests.cpp
-    unittests/rpc/handlers/NFTsByIssuerTest.cpp
-    unittests/rpc/handlers/NFTsByIssuerTest.cpp
-    unittests/rpc/handlers/NFTSellOffersTests.cpp
-    unittests/rpc/handlers/NFTSellOffersTests.cpp
-    unittests/rpc/handlers/NoRippleCheckTests.cpp
-    unittests/rpc/handlers/NoRippleCheckTests.cpp
-    unittests/rpc/handlers/PingTests.cpp
-    unittests/rpc/handlers/PingTests.cpp
-    unittests/rpc/handlers/RandomTests.cpp
-    unittests/rpc/handlers/RandomTests.cpp
-    unittests/rpc/handlers/ServerInfoTests.cpp
-    unittests/rpc/handlers/ServerInfoTests.cpp
-    unittests/rpc/handlers/SubscribeTests.cpp
-    unittests/rpc/handlers/SubscribeTests.cpp
-    unittests/rpc/handlers/TestHandlerTests.cpp
-    unittests/rpc/handlers/TestHandlerTests.cpp
-    unittests/rpc/handlers/TransactionEntryTests.cpp
-    unittests/rpc/handlers/TransactionEntryTests.cpp
-    unittests/rpc/handlers/TxTests.cpp
-    unittests/rpc/handlers/TxTests.cpp
-    unittests/rpc/handlers/UnsubscribeTests.cpp
-    unittests/rpc/handlers/UnsubscribeTests.cpp
-    unittests/rpc/handlers/VersionHandlerTests.cpp
-    unittests/rpc/handlers/VersionHandlerTests.cpp
-    unittests/rpc/JsonBoolTests.cpp
-    # RPC handlers
-    unittests/rpc/JsonBoolTests.cpp
-    # RPC handlers
-    unittests/rpc/RPCHelpersTests.cpp
-    unittests/rpc/RPCHelpersTests.cpp
-    unittests/rpc/WorkQueueTests.cpp
-    unittests/rpc/WorkQueueTests.cpp
-    unittests/util/AssertTests.cpp
-    unittests/util/async/AnyExecutionContextTests.cpp
-    unittests/util/async/AnyOperationTests.cpp
-    unittests/util/async/AnyStopTokenTests.cpp
-    unittests/util/async/AnyStrandTests.cpp
-    unittests/util/async/AsyncExecutionContextTests.cpp
-    # Requests framework
-    unittests/util/BatchingTests.cpp
-    unittests/util/LedgerUtilsTests.cpp
-    # Prometheus support
-    unittests/util/prometheus/CounterTests.cpp
-    unittests/util/prometheus/GaugeTests.cpp
-    unittests/util/prometheus/HistogramTests.cpp
-    unittests/util/prometheus/HttpTests.cpp
-    unittests/util/prometheus/LabelTests.cpp
-    unittests/util/prometheus/MetricBuilderTests.cpp
-    unittests/util/prometheus/MetricsFamilyTests.cpp
-    unittests/util/prometheus/OStreamTests.cpp
-    unittests/util/requests/RequestBuilderTests.cpp
-    unittests/util/requests/SslContextTests.cpp
-    unittests/util/requests/WsConnectionTests.cpp
-    # ETL
-    unittests/util/RetryTests.cpp
-    # Async framework
-    unittests/util/StringUtils.cpp
-    unittests/util/TestGlobals.cpp
-    unittests/util/TestHttpServer.cpp
-    unittests/util/TestObject.cpp
-    unittests/util/TestWsServer.cpp
-    unittests/util/TxUtilTests.cpp
-    unittests/web/AdminVerificationTests.cpp
-    unittests/web/RPCServerHandlerTests.cpp
-    unittests/web/ServerTests.cpp
-    unittests/web/SweepHandlerTests.cpp
-    # Feed
-    unittests/web/WhitelistHandlerTests.cpp
-  )
-
-  include(CMake/deps/gtest.cmake)
-
-  # See https://github.com/google/googletest/issues/3475
-  gtest_discover_tests(clio_tests DISCOVERY_TIMEOUT 90)
-
-  # Fix for dwarf5 bug on ci
-  target_compile_options(clio PUBLIC -gdwarf-4)
-
-  target_compile_definitions(${TEST_TARGET} PUBLIC UNITTEST_BUILD)
-  target_include_directories(${TEST_TARGET} PRIVATE unittests)
-  target_link_libraries(${TEST_TARGET} PUBLIC clio gtest::gtest)
-
-  # Generate `coverage_report` target if coverage is enabled
-  if (coverage)
-    if (DEFINED CODE_COVERAGE_REPORT_FORMAT)
-      set(CODE_COVERAGE_FORMAT ${CODE_COVERAGE_REPORT_FORMAT})
-    else ()
-      set(CODE_COVERAGE_FORMAT html-details)
-    endif ()
-
-    if (DEFINED CODE_COVERAGE_TESTS_ARGS)
-      set(TESTS_ADDITIONAL_ARGS ${CODE_COVERAGE_TESTS_ARGS})
-      separate_arguments(TESTS_ADDITIONAL_ARGS)
-    else ()
-      set(TESTS_ADDITIONAL_ARGS "")
-    endif ()
-
-    set(GCOVR_ADDITIONAL_ARGS --exclude-throw-branches -s)
-
-    setup_target_for_coverage_gcovr(
-      NAME
-      coverage_report
-      FORMAT
-      ${CODE_COVERAGE_FORMAT}
-      EXECUTABLE
-      clio_tests
-      EXECUTABLE_ARGS
-      --gtest_brief=1
-      ${TESTS_ADDITIONAL_ARGS}
-      EXCLUDE
-      "unittests"
-      DEPENDENCIES
-      clio_tests
-    )
-  endif ()
-endif ()
-
-# Benchmarks
 if (benchmark)
-  set(BENCH_TARGET clio_benchmarks)
-  add_executable(
-    ${BENCH_TARGET}
-    # Common
-    benchmarks/Main.cpp benchmarks/Playground.cpp
-    # ExecutionContext
-    benchmarks/util/async/ExecutionContextBenchmarks.cpp
-  )
-
-  include(CMake/deps/gbench.cmake)
-
-  target_include_directories(${BENCH_TARGET} PRIVATE benchmarks)
-  target_link_libraries(${BENCH_TARGET} PUBLIC clio benchmark::benchmark_main)
+  add_subdirectory(benchmarks)
 endif ()
 
 # Enable selected sanitizer if enabled via `san`
@@ -460,7 +81,7 @@ if (docs)
   add_subdirectory(docs)
 endif ()
 
-include(CMake/install/install.cmake)
+include(install/install)
 if (packaging)
-  include(CMake/packaging.cmake) # This file exists only in build runner
+  include(cmake/packaging.cmake) # This file exists only in build runner
 endif ()

CONTRIBUTING.md

@@ -6,7 +6,7 @@ To contribute, please:
 2. Create a new branch on which to commit/push your changes.
 3. Write and test your code.
 4. Ensure that your code compiles with the provided build engine and update the provided build engine as part of your PR where needed and where appropriate.
-5. Where applicable, write test cases for your code and include those in `unittests`.
+5. Where applicable, write test cases for your code and include those in the relevant subfolder under `tests`.
 6. Ensure your code passes automated checks (e.g. clang-format)
 7. Squash your commits (i.e. rebase) into as few commits as is reasonable to describe your changes at a high level (typically a single commit for a small change). See below for more details.
 8. Open a PR to the main repository onto the _develop_ branch, and follow the provided template.
@@ -21,7 +21,7 @@ git config --local core.hooksPath .githooks
 ```
 
 ## Git hooks dependencies
-The pre-commit hook requires `clang-format >= 17.0.0` and `cmake-format` to be installed on your machine.
+The pre-commit hook requires `clang-format >= 19.0.0` and `cmake-format` to be installed on your machine.
 `clang-format` can be installed using `brew` on macOS and default package manager on Linux.
 `cmake-format` can be installed using `pip`.
 The hook will also attempt to automatically use `doxygen` to verify that everything public in the codebase is covered by doc comments. If `doxygen` is not installed, the hook will raise a warning suggesting to install `doxygen` for future commits.
@@ -72,6 +72,9 @@ git push --force
 Clio uses `ccache` to speed up compilation. If you want to use it, please make sure it is installed on your machine.
 CMake will automatically detect it and use it if it is available.
 
+## Opening a pull request
+When a pull request is open CI will perform checks on the new code.
+Title of the pull request and squashed commit should follow [conventional commits specification](https://www.conventionalcommits.org/en/v1.0.0/).
 
 ## Fixing issues found during code review
 While your code is in review, it's possible that some changes will be requested by reviewer(s).
@@ -102,7 +105,7 @@ The button for that is near the bottom of the PR's page on GitHub.
 This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent.
 
 ## Formatting
-Code must conform to `clang-format` version 17, unless the result would be unreasonably difficult to read or maintain.
+Code must conform to `clang-format` version 19, unless the result would be unreasonably difficult to read or maintain.
 In most cases the pre-commit hook will take care of formatting and will fix any issues automatically.
 To manually format your code, use `clang-format -i <your changed files>` for C++ files and `cmake-format -i <your changed files>` for CMake files.
 

README.md

@@ -36,10 +36,12 @@ Below are some useful docs to learn more about Clio.
 - [How to configure Clio and rippled](./docs/configure-clio.md)
 - [How to run Clio](./docs/run-clio.md)
 - [Logging](./docs/logging.md)
+- [Troubleshooting guide](./docs/trouble_shooting.md)
 
 **General reference material:**
 
 - [API reference](https://xrpl.org/http-websocket-apis.html)
+- [Developer docs](https://xrplf.github.io/clio)
 - [Clio documentation](https://xrpl.org/the-clio-server.html#the-clio-server)
 
 ## 🆘 Help

benchmarks/CMakeLists.txt

@@ -0,0 +1,16 @@
+add_executable(clio_benchmark)
+
+target_sources(
+  clio_benchmark
+  PRIVATE # Common
+          Main.cpp
+          Playground.cpp
+          # ExecutionContext
+          util/async/ExecutionContextBenchmarks.cpp
+)
+
+include(deps/gbench)
+
+target_include_directories(clio_benchmark PRIVATE .)
+target_link_libraries(clio_benchmark PUBLIC clio_etl benchmark::benchmark_main)
+set_target_properties(clio_benchmark PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

benchmarks/util/async/ExecutionContextBenchmarks.cpp

@@ -31,7 +31,6 @@
 #include <cstdint>
 #include <latch>
 #include <optional>
-#include <stdexcept>
 #include <thread>
 #include <vector>
 

cmake/Build.cpp.in

@@ -17,12 +17,13 @@
 */
 //==============================================================================
 
-#include "main/Build.hpp"
+#include "util/build/Build.hpp"
 
 #include <string>
 
-namespace Build {
-static constexpr char versionString[] = "@VERSION@";
+namespace util::build {
+
+static constexpr char versionString[] = "@CLIO_VERSION@";
 
 std::string const&
 getClioVersionString()
@@ -38,4 +39,4 @@ getClioFullVersionString()
     return value;
 }
 
-}  // namespace Build
+}  // namespace util::build

cmake/CheckCompiler.cmake

@@ -1,20 +1,20 @@
 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
-    message(FATAL_ERROR "Clang 14+ required for building clio")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 16)
+    message(FATAL_ERROR "Clang 16+ required for building clio")
   endif ()
   set(is_clang TRUE)
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14)
-    message(FATAL_ERROR "AppleClang 14+ required for building clio")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15)
+    message(FATAL_ERROR "AppleClang 15+ required for building clio")
   endif ()
   set(is_appleclang TRUE)
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11)
-    message(FATAL_ERROR "GCC 11+ required for building clio")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 12)
+    message(FATAL_ERROR "GCC 12+ required for building clio")
   endif ()
   set(is_gcc TRUE)
 else ()
-  message(FATAL_ERROR "Supported compilers: AppleClang 14+, Clang 14+, GCC 11+")
+  message(FATAL_ERROR "Supported compilers: AppleClang 15+, Clang 16+, GCC 12+")
 endif ()
 
 if (san)

cmake/ClangTidy.cmake

@@ -8,7 +8,7 @@ if (lint)
     endif ()
     message(STATUS "Using clang-tidy from CLIO_CLANG_TIDY_BIN")
   else ()
-    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-17" "clang-tidy" REQUIRED)
+    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-19" "clang-tidy" REQUIRED)
   endif ()
 
   if (NOT _CLANG_TIDY_BIN)

cmake/ClioVersion.cmake

@@ -22,24 +22,27 @@ endif ()
 
 if (NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name
                                                              # YYYYMMDDHMS-<branch>-<git-rev>
-  execute_process(COMMAND date +%Y%m%d%H%M%S OUTPUT_VARIABLE DATE OUTPUT_STRIP_TRAILING_WHITESPACE)
-  set(VERSION "${DATE}-${BRANCH}-${REV}")
+  set(GIT_COMMAND show -s --date=format:%Y%m%d%H%M%S --format=%cd)
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE DATE
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  set(CLIO_VERSION "${DATE}-${BRANCH}-${REV}")
   set(DOC_CLIO_VERSION "develop")
 else ()
   set(GIT_COMMAND describe --tags)
   execute_process(
-    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE TAG_VERSION
+    COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE CLIO_TAG_VERSION
     OUTPUT_STRIP_TRAILING_WHITESPACE
   )
-  set(VERSION "${TAG_VERSION}-${REV}")
-  set(DOC_CLIO_VERSION "${TAG_VERSION}")
+  set(CLIO_VERSION "${CLIO_TAG_VERSION}")
+  set(DOC_CLIO_VERSION "${CLIO_TAG_VERSION}")
 endif ()
 
 if (CMAKE_BUILD_TYPE MATCHES Debug)
-  set(VERSION "${VERSION}+DEBUG")
+  set(CLIO_VERSION "${CLIO_VERSION}+DEBUG")
 endif ()
 
-message(STATUS "Build version: ${VERSION}")
-set(clio_version "${VERSION}")
+message(STATUS "Build version: ${CLIO_VERSION}")
 
-configure_file(${CMAKE_CURRENT_LIST_DIR}/Build.cpp.in ${CMAKE_CURRENT_LIST_DIR}/../src/main/impl/Build.cpp)
+configure_file(${CMAKE_CURRENT_LIST_DIR}/Build.cpp.in ${CMAKE_CURRENT_LIST_DIR}/../src/util/build/Build.cpp)

cmake/CodeCoverage.cmake

@@ -352,10 +352,10 @@ function (append_coverage_compiler_flags)
 endfunction () # append_coverage_compiler_flags
 
 # Setup coverage for specific library
-function (append_coverage_compiler_flags_to_target name)
+function (append_coverage_compiler_flags_to_target name mode)
   separate_arguments(_flag_list NATIVE_COMMAND "${COVERAGE_COMPILER_FLAGS}")
-  target_compile_options(${name} PRIVATE ${_flag_list})
+  target_compile_options(${name} ${mode} ${_flag_list})
   if (CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
-    target_link_libraries(${name} PRIVATE gcov)
+    target_link_libraries(${name} ${mode} gcov)
   endif ()
 endfunction ()

cmake/Settings.cmake

@@ -17,6 +17,13 @@ set(COMPILER_FLAGS
     -pedantic
     -Wpedantic
     -Wunused
+    # FIXME: The following bunch are needed for gcc12 atm.
+    -Wno-missing-requires
+    -Wno-restrict
+    -Wno-null-dereference
+    -Wno-maybe-uninitialized
+    -Wno-unknown-warning-option # and this to work with clang
+    # TODO: Address these and others in https://github.com/XRPLF/clio/issues/1273
 )
 
 # TODO: reenable when we change CI #884 if (is_gcc AND NOT lint) list(APPEND COMPILER_FLAGS -Wduplicated-branches
@@ -35,4 +42,4 @@ endif ()
 # See https://github.com/cpp-best-practices/cppbestpractices/blob/master/02-Use_the_Tools_Available.md#gcc--clang for
 # the flags description
 
-target_compile_options(clio PUBLIC ${COMPILER_FLAGS})
+target_compile_options(clio_options INTERFACE ${COMPILER_FLAGS})

cmake/SourceLocation.cmake

@@ -2,10 +2,10 @@ include(CheckIncludeFileCXX)
 
 check_include_file_cxx("source_location" SOURCE_LOCATION_AVAILABLE)
 if (SOURCE_LOCATION_AVAILABLE)
-  target_compile_definitions(clio PUBLIC "HAS_SOURCE_LOCATION")
+  target_compile_definitions(clio_options INTERFACE "HAS_SOURCE_LOCATION")
 endif ()
 
 check_include_file_cxx("experimental/source_location" EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)
 if (EXPERIMENTAL_SOURCE_LOCATION_AVAILABLE)
-  target_compile_definitions(clio PUBLIC "HAS_EXPERIMENTAL_SOURCE_LOCATION")
+  target_compile_definitions(clio_options INTERFACE "HAS_EXPERIMENTAL_SOURCE_LOCATION")
 endif ()

cmake/deps/Boost.cmake

@@ -0,0 +1,4 @@
+set(Boost_USE_STATIC_LIBS ON)
+set(Boost_USE_STATIC_RUNTIME ON)
+
+find_package(Boost 1.82 REQUIRED CONFIG COMPONENTS program_options coroutine system log log_setup)

cmake/deps/OpenSSL.cmake

@@ -1,3 +1,3 @@
-find_package(OpenSSL 1.1.1 REQUIRED)
+find_package(OpenSSL 1.1.1 REQUIRED CONFIG)
 
 set_target_properties(OpenSSL::SSL PROPERTIES INTERFACE_COMPILE_DEFINITIONS OPENSSL_NO_SSL2)

cmake/deps/Threads.cmake

@@ -1,2 +1,2 @@
 set(THREADS_PREFER_PTHREAD_FLAG ON)
-find_package(Threads)
+find_package(Threads REQUIRED)

cmake/deps/cassandra.cmake

@@ -0,0 +1 @@
+find_package(cassandra-cpp-driver REQUIRED CONFIG)

cmake/deps/gbench.cmake

@@ -0,0 +1 @@
+find_package(benchmark REQUIRED CONFIG)

cmake/deps/libbacktrace.cmake

@@ -0,0 +1,3 @@
+target_compile_definitions(clio_options INTERFACE BOOST_STACKTRACE_LINK)
+target_compile_definitions(clio_options INTERFACE BOOST_STACKTRACE_USE_BACKTRACE)
+find_package(libbacktrace REQUIRED CONFIG)

cmake/deps/libfmt.cmake

@@ -0,0 +1 @@
+find_package(fmt REQUIRED CONFIG)

cmake/deps/libxrpl.cmake

@@ -0,0 +1 @@
+find_package(xrpl REQUIRED CONFIG)

cmake/install/install.cmake

@@ -8,6 +8,6 @@ string(REGEX REPLACE "./clio_log" "/var/log/clio/" config "${config}")
 file(WRITE ${CMAKE_BINARY_DIR}/install-config.json "${config}")
 install(FILES ${CMAKE_BINARY_DIR}/install-config.json DESTINATION etc RENAME config.json)
 
-configure_file("${CMAKE_SOURCE_DIR}/CMake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service")
+configure_file("${CMAKE_SOURCE_DIR}/cmake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service")
 
 install(FILES "${CMAKE_BINARY_DIR}/clio.service" DESTINATION /lib/systemd/system)

conanfile.py

@@ -1,7 +1,6 @@
 from conan import ConanFile
 from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout
 
-
 class Clio(ConanFile):
     name = 'clio'
     license = 'ISC'
@@ -10,31 +9,35 @@ class Clio(ConanFile):
     description = 'Clio RPC server'
     settings = 'os', 'compiler', 'build_type', 'arch'
     options = {
-        'fPIC': [True, False],
+        'static': [True, False],              # static linkage
+        'fPIC': [True, False],                # unused?
         'verbose': [True, False],
-        'tests': [True, False],     # build unit tests; create `clio_tests` binary
-        'benchmark': [True, False], # build benchmarks; create `clio_benchmarks` binary
-        'docs': [True, False],      # doxygen API docs; create custom target 'docs'
-        'packaging': [True, False], # create distribution packages
-        'coverage': [True, False],  # build for test coverage report; create custom target `clio_tests-ccov`
-        'lint': [True, False],      # run clang-tidy checks during compilation
+        'tests': [True, False],               # build unit tests; create `clio_tests` binary
+        'integration_tests': [True, False],   # build integration tests; create `clio_integration_tests` binary
+        'benchmark': [True, False],           # build benchmarks; create `clio_benchmarks` binary
+        'docs': [True, False],                # doxygen API docs; create custom target 'docs'
+        'packaging': [True, False],           # create distribution packages
+        'coverage': [True, False],            # build for test coverage report; create custom target `clio_tests-ccov`
+        'lint': [True, False],                # run clang-tidy checks during compilation
     }
 
     requires = [
         'boost/1.82.0',
         'cassandra-cpp-driver/2.17.0',
         'fmt/10.1.1',
-        'protobuf/3.21.12',
+        'protobuf/3.21.9',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.2.0-b1',
+        'xrpl/2.3.0-rc1',
         'libbacktrace/cci.20210118'
     ]
 
     default_options = {
+        'static': False,
         'fPIC': True,
         'verbose': False,
         'tests': False,
+        'integration_tests': False,
         'benchmark': False,
         'packaging': False,
         'coverage': False,
@@ -42,6 +45,7 @@ class Clio(ConanFile):
         'docs': False,
         
         'xrpl/*:tests': False,
+        'xrpl/*:rocksdb': False,
         'cassandra-cpp-driver/*:shared': False,
         'date/*:header_only': True,
         'grpc/*:shared': False,
@@ -56,11 +60,11 @@ class Clio(ConanFile):
     }
 
     exports_sources = (
-        'CMakeLists.txt', 'CMake/*', 'src/*'
+        'CMakeLists.txt', 'cmake/*', 'src/*'
     )
 
     def requirements(self):
-        if self.options.tests:
+        if self.options.tests or self.options.integration_tests:
             self.requires('gtest/1.14.0')
         if self.options.benchmark:
             self.requires('benchmark/1.8.3')
@@ -79,11 +83,14 @@ class Clio(ConanFile):
     def generate(self):
         tc = CMakeToolchain(self)
         tc.variables['verbose'] = self.options.verbose
+        tc.variables['static'] = self.options.static
         tc.variables['tests'] = self.options.tests
+        tc.variables['integration_tests'] = self.options.integration_tests
         tc.variables['coverage'] = self.options.coverage
         tc.variables['lint'] = self.options.lint
         tc.variables['docs'] = self.options.docs
         tc.variables['packaging'] = self.options.packaging
+        tc.variables['benchmark'] = self.options.benchmark
         tc.generate()
 
     def build(self):

docker/centos/Dockerfile

@@ -1,49 +0,0 @@
-# FROM centos:7 as deps
-FROM centos:7 as build
-
-ENV CLIO_DIR=/opt/clio/
-# ENV OPENSSL_DIR=/opt/openssl
-
-RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl
-RUN yum install -y devtoolset-11
-ENV version=3.16
-ENV build=3
-# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
-COPY docker/shared/install_cmake.sh /install_cmake.sh
-RUN /install_cmake.sh 3.16.3 /usr/local
-RUN source /opt/rh/devtoolset-11/enable
-WORKDIR /tmp
-# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR
-COPY docker/centos/build_git_centos7.sh build_git_centos7.sh
-
-RUN ./build_git_centos7.sh
-RUN git clone https://github.com/openssl/openssl
-WORKDIR /tmp/openssl
-RUN git checkout OpenSSL_1_1_1q
-#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic
-RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \
-    make -j $(nproc) && \
-    make install_sw
-WORKDIR /tmp
-# FROM centos:7 as build
-
-RUN git clone https://github.com/xrplf/clio.git
-COPY docker/shared/build_boost.sh build_boost.sh
-ENV OPENSSL_ROOT=/opt/local/openssl
-ENV BOOST_ROOT=/boost
-RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0
-RUN yum install -y bison flex
-RUN yum install -y rpmdevtools rpmlint
-RUN source /opt/rh/devtoolset-11/enable && cd /tmp/clio && \
-    cmake -B build -DBUILD_TESTS=1 && \
-    cmake --build build --parallel $(nproc)
-RUN mkdir output
-RUN strip clio/build/clio_server && strip clio/build/clio_tests
-RUN cp clio/build/clio_tests output/ && cp clio/build/clio_server output/
-RUN cp clio/docs/examples/config/example-config.json output/example-config.json
-
-FROM centos:7
-COPY --from=build /tmp/output /clio
-RUN mkdir -p /opt/clio/etc && mv /clio/docs/examples/config/example-config.json /opt/clio/etc/config.json
-
-CMD ["/clio/clio_server", "/opt/clio/etc/config.json"]

docker/centos/build_git_centos7.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-GIT_VERSION="2.37.1"
-curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz
-tar zxvf git-${GIT_VERSION}.tar.gz
-cd git-${GIT_VERSION}
-
-yum install -y centos-release-scl epel-release
-yum update -y
-yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel  libcurl-devel
-
-source /opt/rh/devtoolset-11/enable
-make configure
-./configure
-make git -j$(nproc)
-make install git
-git  --version | cut -d ' ' -f3

docker/centos/install_cmake.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-CMAKE_VERSION=${1:-"3.16.3"}
-cd /tmp
-URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
-curl -OJLs $URL
-tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
-ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake

docker/ci/README.md

@@ -0,0 +1,15 @@
+# CI image for XRPLF/clio
+
+This image contains an environment to build [Clio](https://github.com/XRPLF/clio), check code and documentation.
+It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but can also be used to compile Clio locally.
+
+The image is based on Ubuntu 20.04 and contains:
+- clang 16
+- gcc 12.3
+- doxygen 1.10
+- gh 2.40
+- ccache 4.8.3
+- conan
+- and some other useful tools
+
+Conan is set up to build Clio without any additional steps. There are two preset conan profiles: `clang` and `gcc` to use corresponding compiler.

docker/ci/dockerfile

@@ -1,42 +1,46 @@
-FROM ubuntu:focal
+FROM rippleci/clio_clang:16
 ARG DEBIAN_FRONTEND=noninteractive
 ARG TARGETARCH
 
 SHELL ["/bin/bash", "-c"]
 USER root
-WORKDIR /root/
+WORKDIR /root
 
-ENV GCC_VERSION=11 \
-    CCACHE_VERSION=4.8.3 \
-    LLVM_TOOLS_VERSION=17 \
+ENV CCACHE_VERSION=4.10.2 \
+    LLVM_TOOLS_VERSION=19 \
     GH_VERSION=2.40.0 \
-    DOXYGEN_VERSION=1.10.0
+    DOXYGEN_VERSION=1.12.0
  
 # Add repositories
 RUN apt-get -qq update \
     && apt-get -qq install -y --no-install-recommends --no-install-suggests gnupg wget curl software-properties-common \
-    && add-apt-repository -y ppa:ubuntu-toolchain-r/test \
-    && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | apt-key add - \
-    && apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' \
     && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM_TOOLS_VERSION} main" >> /etc/apt/sources.list \
     && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 
 # Install packages
 RUN apt update -qq \
-    && apt install -y --no-install-recommends --no-install-suggests cmake python3 python3-pip sudo git \
-    ninja-build make pkg-config libzstd-dev libzstd1 g++-${GCC_VERSION} flex bison jq graphviz \
+    && apt install -y --no-install-recommends --no-install-suggests python3 python3-pip git git-lfs make ninja-build flex bison jq graphviz \
     clang-format-${LLVM_TOOLS_VERSION} clang-tidy-${LLVM_TOOLS_VERSION} clang-tools-${LLVM_TOOLS_VERSION} \
-    && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-${GCC_VERSION} 100 \
-    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-${GCC_VERSION} 100 \
     && update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-${LLVM_TOOLS_VERSION} 100 \
-    && apt-get clean && apt remove -y software-properties-common \
-    && pip3 install  -q --upgrade --no-cache-dir pip \
-    && pip3 install  -q --no-cache-dir conan==1.62 gcovr cmake-format
+    && pip3 install -q --upgrade --no-cache-dir pip && pip3 install -q --no-cache-dir conan==1.62 gcovr cmake cmake-format \
+    && apt-get clean && apt remove -y software-properties-common
+
+# Install gcc-12 and make ldconfig aware of the new libstdc++ location (for gcc)
+# Note: Clang is using libc++ instead
+COPY --from=rippleci/clio_gcc:12.3.0 /gcc12.deb /
+RUN apt update && apt-get install -y binutils libc6-dev \
+    && dpkg -i /gcc12.deb \
+    && rm -rf /gcc12.deb \
+    && ldconfig
+
+# Rewire to use gcc-12 as default compiler
+RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 100 \
+    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
+    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100
 
 WORKDIR /tmp
 
@@ -46,34 +50,51 @@ RUN wget "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/
     && cd "ccache-${CCACHE_VERSION}" \
     && mkdir build && cd build \
     && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install
+    && cmake --build . --target install \
+    && rm -rf /tmp/* /var/tmp/*
 
-# Install doxygen from sounce
+# Install doxygen from source
 RUN wget "https://github.com/doxygen/doxygen/releases/download/Release_${DOXYGEN_VERSION//./_}/doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
     && tar xf "doxygen-${DOXYGEN_VERSION}.src.tar.gz" \
     && cd "doxygen-${DOXYGEN_VERSION}" \
     && mkdir build && cd build \
     && cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
-    && cmake --build . --target install
+    && cmake --build . --target install \
+    && rm -rf /tmp/* /var/tmp/*
 
 # Install gh
 RUN wget https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
     && tar xf gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \
-    && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh
+    && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh \
+    && rm -rf /tmp/* /var/tmp/*
 
-# Clean up
-RUN rm -rf  /tmp/* /var/tmp/*
-
-WORKDIR /root/
+WORKDIR /root
 # Using root by default is not very secure but github checkout action doesn't work with any other user
 # https://github.com/actions/checkout/issues/956
 # And Github Actions doc recommends using root
 # https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
 
 # Setup conan
-RUN conan profile new default --detect \
-    && conan profile update settings.compiler.cppstd=20 default \
-    && conan profile update settings.compiler.libcxx=libstdc++11 default \
-    && conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
+RUN conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
 
+# Note: intentionally leaving cppstd=20
+RUN conan profile new gcc --detect \
+    && conan profile update settings.compiler=gcc gcc \
+    && conan profile update settings.compiler.version=12 gcc \
+    && conan profile update settings.compiler.cppstd=20 gcc \
+    && conan profile update settings.compiler.libcxx=libstdc++11 gcc \
+    && conan profile update env.CC=/usr/bin/gcc-12 gcc \
+    && conan profile update env.CXX=/usr/bin/g++-12 gcc \
+    && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/gcc-12\", \"cpp\": \"/usr/bin/g++-12\"}" gcc
 
+RUN conan profile new clang --detect \
+    && conan profile update settings.compiler=clang clang \
+    && conan profile update settings.compiler.version=16 clang \
+    && conan profile update settings.compiler.cppstd=20 clang \
+    && conan profile update settings.compiler.libcxx=libc++ clang \
+    && conan profile update env.CC=/usr/bin/clang-16 clang \
+    && conan profile update env.CXX=/usr/bin/clang++-16 clang \
+    && conan profile update env.CXXFLAGS="-DBOOST_ASIO_DISABLE_CONCEPTS" clang \
+    && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/clang-16\", \"cpp\": \"/usr/bin/clang++-16\"}" clang 
+
+RUN echo "include(gcc)" >> .conan/profiles/default

docker/clio/README.md

@@ -0,0 +1,23 @@
+# Clio official docker image
+
+[Clio](https://github.com/XRPLF/clio) is an XRP Ledger API server optimized for RPC calls over WebSocket or JSON-RPC.
+It stores validated historical ledger and transaction data in a space efficient format.
+
+This image contains `clio_server` binary allowing users to run Clio easily.
+
+## Clio configuration file
+
+Please note that while Clio requires a configuration file, this image doesn't include any default config.
+Your configuration file should be mounted under the path `/opt/clio/etc/config.json`.
+Clio repository provides an [example](https://github.com/XRPLF/clio/blob/develop/docs/examples/config/example-config.json) of the configuration file.
+
+Config file recommendations:
+- Set `log_to_console` to `false` if you want to avoid logs being written to `stdout`.
+- Set `log_directory` to `/opt/clio/log` to store logs in a volume.
+
+## Usage
+
+The following command can be used to run Clio in docker (assuming server's port is `51233` in your config):
+```bash
+docker run -d -v <path to your config.json>:/opt/clio/etc/config.json -v <path to store logs>:/opt/clio/log -p 51233:51233 rippleci/clio
+```

docker/clio/dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:22.04
+
+COPY ./clio_server /opt/clio/bin/clio_server
+
+RUN ln -s /opt/clio/bin/clio_server /usr/local/bin/clio_server && \
+    mkdir -p /opt/clio/etc/ && \
+    mkdir -p /opt/clio/log/ && \
+    groupadd -g 10001 clio && \
+    useradd -u 10000 -g 10001 -s /bin/bash clio && \
+    chown clio:clio /opt/clio/log && \
+    apt update && \
+    apt install -y libatomic1
+
+USER clio
+ENTRYPOINT ["/opt/clio/bin/clio_server"]
+CMD ["--conf", "/opt/clio/etc/config.json"]

docker/clio_docker/centos/build_boost.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -exu
-
-#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static
-# it's either those or link=static that halves the failures. probably link=static
-BOOST_VERSION=$1
-BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _)
-echo "BOOST_VERSION: ${BOOST_VERSION}"
-echo "BOOST_VERSION_: ${BOOST_VERSION_}"
-curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz"
-tar zxf "boost_${BOOST_VERSION_}.tar.gz"
-cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc)
-mkdir -p /boost && mv boost /boost && mv stage /boost

docker/clio_docker/centos/build_git_centos7.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-set -ex
-GIT_VERSION="2.37.1"
-curl -OJL https://github.com/git/git/archive/refs/tags/v${GIT_VERSION}.tar.gz
-tar zxvf git-${GIT_VERSION}.tar.gz
-cd git-${GIT_VERSION}
-
-yum install -y centos-release-scl epel-release
-yum update -y
-yum install -y devtoolset-11 autoconf gnu-getopt gettext zlib-devel  libcurl-devel
-
-source /opt/rh/devtoolset-11/enable
-make configure
-./configure
-make git -j$(nproc)
-make install git
-git  --version | cut -d ' ' -f3

docker/clio_docker/centos/dockerfile

@@ -1,34 +0,0 @@
-FROM centos:7
-
-ENV CLIO_DIR=/opt/clio/
-# ENV OPENSSL_DIR=/opt/openssl
-
-RUN yum -y install git epel-release centos-release-scl perl-IPC-Cmd openssl
-RUN yum install -y devtoolset-11
-ENV version=3.16
-ENV build=3
-# RUN curl -OJL https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
-COPY install_cmake.sh /install_cmake.sh
-RUN /install_cmake.sh 3.16.3 /usr/local
-RUN source /opt/rh/devtoolset-11/enable
-WORKDIR /tmp
-# RUN mkdir $OPENSSL_DIR && cd $OPENSSL_DIR
-COPY build_git_centos7.sh build_git_centos7.sh
-
-RUN ./build_git_centos7.sh
-RUN git clone https://github.com/openssl/openssl
-WORKDIR /tmp/openssl
-RUN git checkout OpenSSL_1_1_1q
-#--prefix=/usr --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic
-RUN SSLDIR=$(openssl version -d | cut -d: -f2 | tr -d [:space:]\") && ./config -fPIC --prefix=/usr --openssldir=${SSLDIR} zlib shared && \
-    make -j $(nproc) && \
-    make install_sw
-WORKDIR /tmp
-RUN git clone https://github.com/xrplf/clio.git
-COPY build_boost.sh build_boost.sh
-ENV OPENSSL_ROOT=/opt/local/openssl
-ENV BOOST_ROOT=/boost
-RUN source scl_source enable devtoolset-11 && /tmp/build_boost.sh 1.75.0
-RUN yum install -y bison flex
-RUN source /opt/rh/devtoolset-11/enable && \
-    cd /tmp/clio && cmake -B build -Dtests=0 -Dlocal_libarchive=1 -Dunity=0 -DBUILD_TESTS=0 && cmake --build build --parallel $(nproc)

docker/clio_docker/centos/install_cmake.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-CMAKE_VERSION=${1:-"3.16.3"}
-cd /tmp
-URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
-curl -OJLs $URL
-tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
-ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake

docker/compilers/clang-16/dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:focal 
+ARG DEBIAN_FRONTEND=noninteractive
+ARG TARGETARCH
+
+SHELL ["/bin/bash", "-c"]
+USER root
+WORKDIR /root
+
+ENV CLANG_VERSION=16
+
+RUN apt update -qq \
+    && apt install -qq -y --no-install-recommends --no-install-suggests \
+    wget software-properties-common gnupg
+
+RUN wget https://apt.llvm.org/llvm.sh \
+    && chmod +x llvm.sh \
+    && ./llvm.sh ${CLANG_VERSION} \
+    && rm -rf llvm.sh \
+    && apt-get install -y libc++-16-dev libc++abi-16-dev

docker/compilers/gcc-12/control.m4

@@ -0,0 +1,6 @@
+Package: gcc-12-ubuntu-UBUNTUVERSION
+Version: VERSION
+Architecture: TARGETARCH
+Maintainer: Alex Kremer <akremer@ripple.com>
+Description: Gcc VERSION build for ubuntu UBUNTUVERSION
+Depends: binutils, libc6-dev

docker/compilers/gcc-12/dockerfile

@@ -0,0 +1,74 @@
+FROM ubuntu:focal as build
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG TARGETARCH
+ARG UBUNTU_VERSION=20.04
+ARG GCC_VERSION=12.3.0
+ARG BUILD_VERSION=1
+
+RUN apt update && apt install -y wget build-essential file flex libz-dev libzstd-dev
+RUN wget https://gcc.gnu.org/pub/gcc/releases/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz \
+    && tar xf gcc-$GCC_VERSION.tar.gz \
+    && cd /gcc-$GCC_VERSION && ./contrib/download_prerequisites
+
+RUN mkdir /${TARGETARCH}-gcc-12
+WORKDIR /${TARGETARCH}-gcc-12
+RUN /gcc-$GCC_VERSION/configure \
+    --with-pkgversion="clio-build-$BUILD_VERSION https://github.com/XRPLF/clio" \
+    --enable-languages=c,c++ \
+    --prefix=/usr \
+    --with-gcc-major-version-only \
+    --program-suffix=-12 \
+    --enable-shared \
+    --enable-linker-build-id \
+    --libexecdir=/usr/lib \
+    --without-included-gettext \
+    --enable-threads=posix \
+    --libdir=/usr/lib \
+    --disable-nls \
+    --enable-clocale=gnu \
+    --enable-libstdcxx-backtrace=yes \
+    --enable-libstdcxx-debug \
+    --enable-libstdcxx-time=yes \
+    --with-default-libstdcxx-abi=new \
+    --enable-gnu-unique-object \
+    --disable-vtable-verify \
+    --enable-plugin \
+    --enable-default-pie \
+    --with-system-zlib \
+    --enable-libphobos-checking=release \
+    --with-target-system-zlib=auto \
+    --disable-werror \
+    --enable-cet \
+    --disable-multilib \
+    --without-cuda-driver \
+    --enable-checking=release \
+    && make -j`nproc` \
+    && make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION \
+    && mkdir -p /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64 \
+    && mv /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/lib64/libstdc++.so.6.0.30-gdb.py /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64/libstdc++.so.6.0.30-gdb.py
+
+# Generate deb
+WORKDIR /
+COPY control.m4 /
+COPY ld.so.conf /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/etc/ld.so.conf.d/1-gcc-12.conf
+
+RUN mkdir /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/DEBIAN \
+    && m4 -P -DUBUNTU_VERSION=$UBUNTU_VERSION -DVERSION=$GCC_VERSION-$BUILD_VERSION -DTARGETARCH=$TARGETARCH control.m4 > /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/DEBIAN/control \
+    && dpkg-deb --build --root-owner-group /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION /gcc12.deb
+
+# Create final image
+FROM ubuntu:focal as gcc
+COPY --from=build /gcc12.deb /
+
+# Make gcc-12 available but also leave gcc12.deb for others to copy if needed
+RUN apt update && apt-get install -y binutils libc6-dev \
+    && dpkg -i /gcc12.deb 
+
+RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 \
+    && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 \
+    && update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 100 \
+    && update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
+    && update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100

docker/compilers/gcc-12/ld.so.conf

@@ -0,0 +1,2 @@
+# Path to the directory containing libstdc++.so.6
+/usr/lib64

docker/develop/compose.yaml

@@ -0,0 +1,15 @@
+services:
+  clio_develop:
+    image: rippleci/clio_ci:latest
+    volumes:
+      - clio_develop_conan_data:/root/.conan/data
+      - clio_develop_ccache:/root/.ccache
+      - ../../:/root/clio
+      - clio_develop_build:/root/clio/build_docker
+    working_dir: /root/clio/build_docker
+    tty: true
+
+volumes:
+  clio_develop_conan_data:
+  clio_develop_ccache:
+  clio_develop_build:

docker/develop/run

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+script_dir=$(dirname $0)
+
+pushd $script_dir > /dev/null
+
+function start_container {
+    if [ -z "$(docker ps -q -f name=clio_develop)" ]; then
+        docker compose up -d
+    fi
+}
+
+function run {
+    start_container
+    docker compose exec clio_develop "$@"
+}
+
+function stop_container {
+    docker compose down
+}
+
+function open_terminal {
+    start_container
+    docker compose exec clio_develop /bin/bash
+}
+
+function print_help {
+    cat <<EOF
+run: Run a command inside the development container.
+
+Usage:
+    run [options or command]
+
+If no options are provided, the command will be executed inside the container.
+
+Options:
+-h, --help      Show this help message and exit.
+-t, --terminal  Open a terminal inside the container.
+-s, --stop      Stop the container.
+EOF
+}
+
+case $1 in
+    -h|--help)
+        print_help ;;
+
+    -t|--terminal)
+        open_terminal ;;
+
+    -s|--stop)
+        stop_container ;;
+
+    -*)
+        echo "Unknown option: $1"
+        print_help ;;
+
+    *)
+        run "$@" ;;
+esac
+
+popd > /dev/null
+

docker/shared/build_boost.sh

@@ -1,13 +0,0 @@
-#!/usr/bin/env bash
-set -exu
-
-#yum install wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-11-binutils zlib-static
-# it's either those or link=static that halves the failures. probably link=static
-BOOST_VERSION=$1
-BOOST_VERSION_=$(echo ${BOOST_VERSION} | tr . _)
-echo "BOOST_VERSION: ${BOOST_VERSION}"
-echo "BOOST_VERSION_: ${BOOST_VERSION_}"
-curl -OJLs "https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz"
-tar zxf "boost_${BOOST_VERSION_}.tar.gz"
-cd boost_${BOOST_VERSION_} && ./bootstrap.sh && ./b2 --without-python link=static -j$(nproc)
-mkdir -p /boost && mv boost /boost && mv stage /boost

docker/shared/install_cmake.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-CMAKE_VERSION=${1:-"3.16.3"}
-cd /tmp
-URL="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz"
-curl -OJLs $URL
-tar xzvf cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz
-mv cmake-${CMAKE_VERSION}-Linux-x86_64 /opt/
-ln -s /opt/cmake-${CMAKE_VERSION}-Linux-x86_64/bin/cmake /usr/local/bin/cmake

docker/shared/install_openssl.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-set -e

docker/ubuntu/Dockerfile

@@ -1,24 +0,0 @@
-FROM ubuntu:20.04 AS boost
-
-RUN apt-get update && apt-get install -y build-essential
-ARG BOOST_VERSION_=1_75_0
-ARG BOOST_VERSION=1.75.0
-COPY docker/shared/build_boost.sh .
-RUN apt install -y curl
-RUN ./build_boost.sh ${BOOST_VERSION}
-ENV BOOST_ROOT=/boost
-
-FROM ubuntu:20.04 AS build
-ENV BOOST_ROOT=/boost
-COPY --from=boost /boost /boost
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install --no-install-recommends -y build-essential software-properties-common pkg-config libssl-dev wget curl gpg git zlib1g-dev bison flex autoconf lsb-release
-RUN apt install -y gpg-agent
-RUN wget https://apt.llvm.org/llvm.sh
-RUN chmod +x llvm.sh && ./llvm.sh 14 && ./llvm.sh 15
-# COPY . /clio
-## Install cmake
-ARG CMAKE_VERSION=3.16.3
-COPY docker/shared/install_cmake.sh .
-RUN ./install_cmake.sh ${CMAKE_VERSION}
-ENV PATH="/opt/local/cmake/bin:$PATH"

docs/CMakeLists.txt

@@ -1,8 +1,8 @@
-cmake_minimum_required(VERSION 3.16.3)
+cmake_minimum_required(VERSION 3.20)
 project(docs)
 
-include(${CMAKE_CURRENT_SOURCE_DIR}/../CMake/ClioVersion.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/ClioVersion.cmake)
 
 # Generate `docs` target for doxygen documentation
 # Note: use `cmake --build . --target docs` from your `build` directory to generate the documentation
-include(${CMAKE_CURRENT_SOURCE_DIR}/../CMake/Docs.cmake)
+include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/Docs.cmake)

docs/build-clio.md

@@ -6,19 +6,22 @@ Clio is built with [CMake](https://cmake.org/) and uses [Conan](https://conan.io
 
 - [Python 3.7](https://www.python.org/downloads/)
 - [Conan 1.55](https://conan.io/downloads.html)
-- [CMake 3.16](https://cmake.org/download/)
+- [CMake 3.20](https://cmake.org/download/)
 - [**Optional**] [GCovr](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html): needed for code coverage generation
 - [**Optional**] [CCache](https://ccache.dev/): speeds up compilation if you are going to compile Clio often
 
 | Compiler    | Version |
 |-------------|---------|
-| GCC         | 11      |
-| Clang       | 14      |
-| Apple Clang | 14.0.3  |
+| GCC         | 12.3    |
+| Clang       | 16      |
+| Apple Clang | 15      |
 
 ### Conan Configuration
 
-Clio does not require anything but default settings in your (`~/.conan/profiles/default`) Conan profile. It's best to have no extra flags specified.
+Clio does not require anything other than `compiler.cppstd=20` in your (`~/.conan/profiles/default`) Conan profile.
+
+> [!NOTE]
+> Although Clio is built using C++23, it's required to set `compiler.cppstd=20` for the time being as some of Clio's dependencies are not yet capable of building under C++23.
 
 > Mac example:
 
@@ -29,10 +32,12 @@ os_build=Macos
 arch=armv8
 arch_build=armv8
 compiler=apple-clang
-compiler.version=14
+compiler.version=15
 compiler.libcxx=libc++
 build_type=Release
 compiler.cppstd=20
+[conf]
+tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]
 ```
 
 > Linux example:
@@ -44,7 +49,7 @@ os_build=Linux
 arch=x86_64
 arch_build=x86_64
 compiler=gcc
-compiler.version=11
+compiler.version=12
 compiler.libcxx=libstdc++11
 build_type=Release
 compiler.cppstd=20
@@ -88,6 +93,35 @@ If successful, `conan install` will find the required packages and `cmake` will
 > [!TIP]
 > To generate a Code Coverage report, include `-o coverage=True` in the `conan install` command above, along with `-o tests=True` to enable tests. After running the `cmake` commands, execute `make clio_tests-ccov`. The coverage report will be found at `clio_tests-llvm-cov/index.html`.
 
+> [!NOTE]
+> If you've built Clio before and the build is now failing, it's likely due to updated dependencies. Try deleting the build folder and then rerunning the Conan and CMake commands mentioned above.
+
+### Generating API docs for Clio
+
+The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen on your system.
+
+To generate the API docs:
+
+1. First, include `-o docs=True` in the conan install command. For example:
+
+    ```sh
+    mkdir build && cd build
+    conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False -o docs=True
+    ```
+
+2. Once that has completed successfully, run the `cmake` command and add the `--target docs` option:
+
+   ```sh
+   cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release ..
+   cmake --build . --parallel 8 --target docs
+   ```
+
+3. Go to `build/docs/html` to view the generated files.
+
+    Open the `index.html` file in your browser to see the documentation pages.
+
+    ![API index page](./img/doxygen-docs-output.png "API index page")
+
 ## Building Clio with Docker
 
 It is also possible to build Clio using [Docker](https://www.docker.com/) if you don't want to install all the dependencies on your machine.
@@ -107,3 +141,43 @@ If you wish to develop against a `rippled` instance running in standalone mode t
 
 1. Advance the `rippled` ledger to at least ledger 256.
 2. Wait 10 minutes before first starting Clio against this standalone node.
+
+## Building with a Custom `libxrpl`
+
+Sometimes, during development, you need to build against a custom version of `libxrpl`. (For example, you may be developing compatibility for a proposed amendment that is not yet merged to the main `rippled` codebase.) To build Clio with compatibility for a custom fork or branch of `rippled`, follow these steps:
+
+1. First, pull/clone the appropriate `rippled` fork and switch to the branch you want to build. For example, the following example uses an in-development build with [XLS-33d Multi-Purpose Tokens](https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens):
+
+    ```sh
+    git clone https://github.com/shawnxie999/rippled/
+    cd rippled
+    git switch mpt-1.1
+    ```
+
+2. Export a custom package to your local Conan store using a user/channel:
+
+    ```sh
+    conan export . my/feature
+    ```
+
+3. Patch your local Clio build to use the right package.
+
+    Edit `conanfile.py` (from the Clio repository root). Replace the `xrpl` requirement with the custom package version from the previous step. This must also include the current version number from your `rippled` branch. For example:
+
+    ```py
+    # ... (excerpt from conanfile.py)
+        requires = [
+        'boost/1.82.0',
+        'cassandra-cpp-driver/2.17.0',
+        'fmt/10.1.1',
+        'protobuf/3.21.9',
+        'grpc/1.50.1',
+        'openssl/1.1.1u',
+        'xrpl/2.3.0-b1@my/feature', # Update this line
+        'libbacktrace/cci.20210118'
+    ]
+    ```
+
+4. Build Clio as you would have before.
+
+    See [Building Clio](#building-clio) for details.

docs/configure-clio.md

@@ -27,7 +27,7 @@ If you're running Clio and `rippled` on separate machines, in addition to uncomm
 
 2. Open a public, unencrypted WebSocket port on your `rippled` server.
 
-3. In the `rippled` config, change the IP specified for `secure_gateway`, under the `port_grpc` section, to the IP of your Clio server. This entry can take the form of a comma-separated list if you are running multiple Clio nodes.
+3. In the `rippled` config, change the IP specified for `secure_gateway`, under the `port_grpc` and websocket server sections, to the IP of your Clio server. This entry can take the form of a comma-separated list if you are running multiple Clio nodes.
 
 ## Ledger sequence
 
@@ -97,3 +97,11 @@ To enable the caching for a source, `forwarding_cache_timeout` value should be a
 
 `forwarding_cache_timeout` defines for how long (in seconds) a cache entry will be valid after being placed into the cache.
 Zero value turns off the cache feature.
+
+## Graceful shutdown (not fully implemented yet)
+
+Clio can be gracefully shut down by sending a `SIGINT` (Ctrl+C) or `SIGTERM` signal.
+The process will stop accepting new connections and will wait for the time specified in `graceful_period` config value (or 10 seconds by default).
+If Clio finishes all the scheduled operations before the end of the period, it will stop immediately.
+Otherwise, it will wait for the period to finish and then exit without finishing operations.
+If Clio receives a second signal during the period, it will stop immediately.

docs/coverage-report.md

@@ -25,16 +25,7 @@ The above steps are automated into a single target `coverage_report`. The instru
 
 In case of a spurious failure of unit tests, it is possible to re-run the `coverage_report` target without rebuilding the `clio_tests` binary (since it is simply a dependency of the coverage report target).
 
-The default coverage report format is `html-details`, but developers can override it to any of the formats listed in `CMake/CodeCoverage.cmake` by setting `CODE_COVERAGE_REPORT_FORMAT` variable in `cmake`. For example, CI is setting this parameter to `xml` for the [codecov](https://codecov.io) integration.
-
-If some unit tests predictably fail (e.g., due to absence of a Cassandra database), it is possible to set unit tests options in the `CODE_COVERAGE_TESTS_ARGS` cmake variable, as demonstrated below:
-
-```sh
-cd .build
-conan install .. --output-folder . --build missing --settings build_type=Debug -o tests=True -o coverage=True
-cmake -DCODE_COVERAGE_REPORT_FORMAT=json-details -DCMAKE_BUILD_TYPE=Debug -DCODE_COVERAGE_TESTS_ARGS="--gtest_filter=-BackendCassandra*" -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake ..
-cmake --build . --target coverage_report
-```
+The default coverage report format is `html-details`, but developers can override it to any of the formats listed in `cmake/CodeCoverage.cmake` by setting `CODE_COVERAGE_REPORT_FORMAT` variable in `cmake`. For example, CI is setting this parameter to `xml` for the [codecov](https://codecov.io) integration.
 
 After the `coverage_report` target is completed, the generated coverage report will be stored inside the build directory as either:
 

docs/examples/config/example-config.json

@@ -31,11 +31,17 @@
     "etl_sources": [
         {
             "ip": "127.0.0.1",
-            "ws_port": "6006",
+            "ws_port": "6005",
             "grpc_port": "50051"
         }
     ],
-    "forwarding_cache_timeout": 0.250, // in seconds, could be 0, which means no cache
+    "forwarding": {
+        "cache_timeout": 0.250, // in seconds, could be 0, which means no cache
+        "request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds)
+    },
+    "rpc": {
+        "cache_timeout": 0.5 // in seconds, could be 0, which means no cache for rpc
+    },
     "dos_guard": {
         // Comma-separated list of IPs to exclude from rate limiting
         "whitelist": [
@@ -64,8 +70,17 @@
         "admin_password": "xrp",
         // If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests
         // It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time
-        "local_amdin": false
+        "local_admin": false,
+        "processing_policy": "parallel", // Could be "sequent" or "parallel".
+        // For sequent policy request from one client connection will be processed one by one and the next one will not be read before
+        // the previous one is processed. For parallel policy Clio will take all requests and process them in parallel and
+        // send a reply for each request whenever it is ready.
+        "parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified.
+        // Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection.
+        "ws_max_sending_queue_size": 1500
     },
+    // Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet.
+    "graceful_period": 10.0,
     // Overrides log level on a per logging channel.
     // Defaults to global "log_level" for each unspecified channel.
     "log_channels": [
@@ -94,6 +109,15 @@
             "log_level": "trace"
         }
     ],
+    "cache": {
+        // Configure this to use either "num_diffs", "num_cursors_from_diff", or "num_cursors_from_account". By default, Clio uses "num_diffs".
+        "num_diffs": 32, // Generate the cursors from the latest ledger diff, then use the cursors to partition the ledger to load concurrently. The cursors number is affected by the busyness of the network.
+        // "num_cursors_from_diff": 3200, // Read the cursors from the diff table until we have enough cursors to partition the ledger to load concurrently. 
+        // "num_cursors_from_account": 3200, // Read the cursors from the account table until we have enough cursors to partition the ledger to load concurrently.
+        "num_markers": 48, // The number of markers is the number of coroutines to load the cache concurrently.
+        "page_fetch_size": 512, // The number of rows to load for each page.
+        "load": "async" // "sync" to load cache synchronously  or "async" to load cache asynchronously or "none"/"no" to turn off the cache.
+    },
     "prometheus": {
         "enabled": true,
         "compress_reply": true

docs/examples/infrastructure/grafana/clio_dashboard.json

@@ -18,6 +18,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
+  "id": 1,
   "links": [],
   "liveNow": false,
   "panels": [
@@ -85,9 +86,11 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "10.2.0",
+      "pluginVersion": "10.4.0",
       "targets": [
         {
           "datasource": {
@@ -105,6 +108,372 @@
       "title": "Service state",
       "type": "stat"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "blue",
+                  "index": 0,
+                  "text": "No"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 1,
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 3,
+        "x": 3,
+        "y": 0
+      },
+      "id": 14,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "read_only",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Read only",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "blue",
+                  "index": 0,
+                  "text": "No"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 1,
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 3,
+        "x": 6,
+        "y": 0
+      },
+      "id": 15,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "etl_writing",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Writing data to DB",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "No"
+                },
+                "1": {
+                  "color": "red",
+                  "index": 1,
+                  "text": "Yes"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 3,
+        "x": 9,
+        "y": 0
+      },
+      "id": 16,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "10.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "etl_amendment_blocked",
+          "instant": false,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Amendment blocked",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "timestamp(etl_last_publish_seconds) - etl_last_publish_seconds",
+          "format": "time_series",
+          "instant": false,
+          "legendFormat": "ledger age",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Ledger Age",
+      "transformations": [
+        {
+          "id": "filterByValue",
+          "options": {
+            "filters": [
+              {
+                "config": {
+                  "id": "lower",
+                  "options": {
+                    "value": 31500000
+                  }
+                },
+                "fieldName": "ledger age"
+              }
+            ],
+            "match": "all",
+            "type": "include"
+          }
+        }
+      ],
+      "type": "timeseries"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -166,9 +535,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 9,
-        "x": 3,
-        "y": 0
+        "w": 12,
+        "x": 0,
+        "y": 8
       },
       "id": 2,
       "options": {
@@ -263,7 +632,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 0
+        "y": 8
       },
       "id": 9,
       "options": {
@@ -358,7 +727,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 8
+        "y": 16
       },
       "id": 11,
       "options": {
@@ -453,7 +822,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 8
+        "y": 16
       },
       "id": 6,
       "options": {
@@ -550,7 +919,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 16
+        "y": 24
       },
       "id": 10,
       "options": {
@@ -645,7 +1014,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 16
+        "y": 24
       },
       "id": 8,
       "options": {
@@ -740,7 +1109,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 24
+        "y": 32
       },
       "id": 4,
       "options": {
@@ -839,7 +1208,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 24
+        "y": 32
       },
       "id": 12,
       "options": {
@@ -973,7 +1342,7 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 32
+        "y": 40
       },
       "id": 5,
       "options": {
@@ -1081,7 +1450,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 32
+        "y": 40
       },
       "id": 3,
       "options": {
@@ -1186,9 +1555,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 10,
+        "w": 12,
         "x": 0,
-        "y": 40
+        "y": 48
       },
       "id": 7,
       "options": {
@@ -1222,7 +1591,7 @@
     }
   ],
   "refresh": "5s",
-  "schemaVersion": 38,
+  "schemaVersion": 39,
   "tags": [],
   "templating": {
     "list": []
@@ -1235,6 +1604,6 @@
   "timezone": "",
   "title": "Clio",
   "uid": "aeaae84e-c194-47b2-ad65-86e45eebb815",
-  "version": 3,
+  "version": 1,
   "weekStart": ""
 }

docs/img/doxygen-docs-output.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16e6f0a44b2dc7462f813f07ba754575e93033bc8bbfbe1d28f8bf1927915028
+size 394711

docs/metrics-and-static-analysis.md

@@ -14,7 +14,7 @@ You can find an example docker-compose file, with Prometheus and Grafana configs
 
 ## Using `clang-tidy` for static analysis
 
-The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 17.0.
+The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 19.0.
 
 Clang-tidy can be run by Cmake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command:
 
@@ -26,5 +26,5 @@ By default Cmake will try to find `clang-tidy` automatically in your system.
 To force Cmake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example:
 
 ```sh
-export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@17/bin/clang-tidy
+export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@19/bin/clang-tidy
 ```

docs/trouble_shooting.md

@@ -0,0 +1,47 @@
+# Troubleshooting Guide
+This guide will help you troubleshoot common issues of Clio.
+
+## Can't connect to DB
+If you see the error log message `Could not connect to Cassandra: No hosts available`, this means that Clio can't connect to the database. Check the following:
+- Make sure the database is running at the specified address and port.
+- Make sure the database is accessible from the machine where Clio is running.
+You can use [cqlsh](https://pypi.org/project/cqlsh/) to check the connection to the database.
+If you would like to run a local ScyllaDB, you can call:
+```sh
+docker run --rm -p 9042:9042 --name clio-scylla -d scylladb/scylla 
+```
+
+## Check the server status of Clio
+To check if Clio is syncing with rippled:
+```sh
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep seq
+```
+If Clio is syncing with rippled, the `seq` value will be increasing.
+
+## Clio fails to start
+If you see the error log message `Failed to fetch ETL state from...`, this means the configured rippled node is not reachable. Check the following:
+- Make sure the rippled node is running at the specified address and port.
+- Make sure the rippled node is accessible from the machine where Clio is running.
+
+If you would like to run Clio without an avaliable rippled node, you can add below setting to Clio's configuration file:
+```
+"allow_no_etl": true
+```
+
+## Clio is not added to secure_gateway in rippled's config
+If you see the warning message `AsyncCallData is_unlimited is false.`, this means that Clio is not added to the `secure_gateway` of `port_grpc` session in the rippled configuration file. It will slow down the sync process. Please add Clio's IP to the `secure_gateway` in the rippled configuration file for both grpc and ws port.
+
+## Clio is slow
+To speed up the response time, Clio has a cache inside. However, cache can take time to warm up. If you see slow response time, you can firstly check if cache is still loading.
+You can check the cache status by calling:
+```sh
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_full
+curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_enabled
+```
+If `is_full` is false, it means the cache is still loading. Normally, the Clio can respond quicker if cache finishs loading. If `is_enabled` is false, it means the cache is disabled in the configuration file or there is data corruption in the database. 
+
+## Receive error message `Too many requests`
+If client sees the error message `Too many requests`, this means that the client is blocked by Clio's DosGuard protection. You may want to add the client's IP to the whitelist in the configuration file, Or update other your DosGuard settings.
+
+
+

src/CMakeLists.txt

@@ -0,0 +1,9 @@
+add_subdirectory(util)
+add_subdirectory(data)
+add_subdirectory(etl)
+add_subdirectory(etlng)
+add_subdirectory(feed)
+add_subdirectory(rpc)
+add_subdirectory(web)
+add_subdirectory(app)
+add_subdirectory(main)

src/app/CMakeLists.txt

@@ -0,0 +1,4 @@
+add_library(clio_app)
+target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp)
+
+target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc)

src/app/CliArgs.cpp

@@ -0,0 +1,70 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "app/CliArgs.hpp"
+
+#include "util/build/Build.hpp"
+
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/program_options/positional_options.hpp>
+#include <boost/program_options/value_semantic.hpp>
+#include <boost/program_options/variables_map.hpp>
+
+#include <cstdlib>
+#include <iostream>
+#include <string>
+#include <utility>
+
+namespace app {
+
+CliArgs::Action
+CliArgs::parse(int argc, char const* argv[])
+{
+    namespace po = boost::program_options;
+    // clang-format off
+    po::options_description description("Options");
+    description.add_options()
+        ("help,h", "print help message and exit")
+        ("version,v", "print version and exit")
+        ("conf,c", po::value<std::string>()->default_value(defaultConfigPath), "configuration file")
+    ;
+    // clang-format on
+    po::positional_options_description positional;
+    positional.add("conf", 1);
+
+    po::variables_map parsed;
+    po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
+    po::notify(parsed);
+
+    if (parsed.count("help") != 0u) {
+        std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n" << description;
+        return Action{Action::Exit{EXIT_SUCCESS}};
+    }
+
+    if (parsed.count("version") != 0u) {
+        std::cout << util::build::getClioFullVersionString() << '\n';
+        return Action{Action::Exit{EXIT_SUCCESS}};
+    }
+
+    auto configPath = parsed["conf"].as<std::string>();
+    return Action{Action::Run{std::move(configPath)}};
+}
+
+}  // namespace app