Compare commits

..

1 Commits

Author SHA1 Message Date
mathbunnyru
99a33777c9 style: Update pre-commit hooks 2025-07-02 15:24:51 +00:00
211 changed files with 1778 additions and 2569 deletions

View File

@@ -16,7 +16,6 @@ coverage:
#
# More info: https://github.com/XRPLF/clio/pull/2066
ignore:
- "benchmarks"
- "tests"
- "src/data/cassandra/"
- "src/data/CassandraBackend.hpp"

View File

@@ -17,9 +17,6 @@ inputs:
platforms:
description: Platforms to build the image for (e.g. linux/amd64,linux/arm64)
required: true
build_args:
description: List of build-time variables
required: false
dockerhub_repo:
description: DockerHub repository name
@@ -33,14 +30,14 @@ runs:
steps:
- name: Login to DockerHub
if: ${{ inputs.push_image == 'true' && inputs.dockerhub_repo != '' }}
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_PW }}
- name: Login to GitHub Container Registry
if: ${{ inputs.push_image == 'true' }}
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@@ -51,7 +48,7 @@ runs:
cache-image: false
- uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
- uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5.8.0
- uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
id: meta
with:
images: ${{ inputs.images }}
@@ -64,4 +61,13 @@ runs:
platforms: ${{ inputs.platforms }}
push: ${{ inputs.push_image == 'true' }}
tags: ${{ steps.meta.outputs.tags }}
build-args: ${{ inputs.build_args }}
- name: Update DockerHub description
if: ${{ inputs.push_image == 'true' && inputs.dockerhub_repo != '' }}
uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_PW }}
repository: ${{ inputs.dockerhub_repo }}
short-description: ${{ inputs.dockerhub_description }}
readme-filepath: ${{ inputs.directory }}/README.md

View File

@@ -15,7 +15,6 @@ runs:
shell: bash
run: |
gcovr \
-e benchmarks \
-e tests \
-e src/data/cassandra \
-e src/data/CassandraBackend.hpp \

View File

@@ -17,10 +17,6 @@ inputs:
description: Whether to build integration tests
required: true
default: "true"
build_benchmark:
description: Whether to build benchmark tests
required: true
default: "true"
code_coverage:
description: Whether conan's coverage option should be on or not
required: true
@@ -33,10 +29,6 @@ inputs:
description: Whether to enable compiler trace reports
required: true
default: "false"
use_mold:
description: Whether to use mold linker
required: true
default: "false"
runs:
using: composite
@@ -52,9 +44,7 @@ runs:
CODE_COVERAGE: "${{ inputs.code_coverage == 'true' && 'True' || 'False' }}"
STATIC_OPTION: "${{ inputs.static == 'true' && 'True' || 'False' }}"
INTEGRATION_TESTS_OPTION: "${{ inputs.build_integration_tests == 'true' && 'True' || 'False' }}"
BENCHMARK_OPTION: "${{ inputs.build_benchmark == 'true' && 'True' || 'False' }}"
TIME_TRACE: "${{ inputs.time_trace == 'true' && 'True' || 'False' }}"
USE_MOLD: "${{ inputs.use_mold == 'true' && 'True' || 'False' }}"
run: |
cd build
conan \
@@ -65,11 +55,9 @@ runs:
-o "&:static=${STATIC_OPTION}" \
-o "&:tests=True" \
-o "&:integration_tests=${INTEGRATION_TESTS_OPTION}" \
-o "&:benchmark=${BENCHMARK_OPTION}" \
-o "&:lint=False" \
-o "&:coverage=${CODE_COVERAGE}" \
-o "&:time_trace=${TIME_TRACE}" \
-o "&:use_mold=${USE_MOLD}" \
--profile:all "${{ inputs.conan_profile }}"
- name: Run cmake
@@ -81,13 +69,11 @@ runs:
endsWith(inputs.conan_profile, '.tsan') && '-Dsan=thread' ||
endsWith(inputs.conan_profile, '.ubsan') && '-Dsan=undefined' ||
'' }}
USE_MOLD_OPTION: "${{ inputs.use_mold == 'true' && '-DUSE_MOLD=ON' || '' }}"
run: |
cd build
cmake \
-DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
-DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \
"${USE_MOLD_OPTION}" \
"${SANITIZER_OPTION}" \
.. \
-G Ninja

View File

@@ -18,7 +18,6 @@ runs:
ca-certificates \
ccache \
clang-build-analyzer \
cmake \
conan \
gh \
jq \
@@ -27,6 +26,25 @@ runs:
pkg-config
echo "/opt/homebrew/opt/conan@2/bin" >> $GITHUB_PATH
- name: Install CMake 3.31.6 on mac
if: ${{ runner.os == 'macOS' }}
shell: bash
run: |
# Uninstall any existing cmake
brew uninstall --formula cmake --ignore-dependencies || true
# Download specific cmake formula
FORMULA_URL="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb"
FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4"
mkdir -p /tmp/homebrew-formula
curl -s -L "$FORMULA_URL" -o /tmp/homebrew-formula/cmake.rb
echo "$FORMULA_EXPECTED_SHA256 /tmp/homebrew-formula/cmake.rb" | shasum -a 256 -c
# Install cmake from the specific formula with force flag
brew install --formula --quiet --force /tmp/homebrew-formula/cmake.rb
- name: Fix git permissions on Linux
if: ${{ runner.os == 'Linux' }}
shell: bash

View File

@@ -1,11 +0,0 @@
[settings]
arch={{detect_api.detect_arch()}}
build_type=Release
compiler=apple-clang
compiler.cppstd=20
compiler.libcxx=libc++
compiler.version=17
os=Macos
[conf]
grpc/1.50.1:tools.build:cxxflags+=["-Wno-missing-template-arg-list-after-template-kw"]

View File

@@ -3,7 +3,7 @@ import itertools
import json
LINUX_OS = ["heavy", "heavy-arm64"]
LINUX_CONTAINERS = ['{ "image": "ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5" }']
LINUX_CONTAINERS = ['{ "image": "ghcr.io/xrplf/clio-ci:latest" }']
LINUX_COMPILERS = ["gcc", "clang"]
MACOS_OS = ["macos15"]
@@ -22,6 +22,9 @@ def generate_matrix():
itertools.product(MACOS_OS, MACOS_CONTAINERS, MACOS_COMPILERS),
):
for sanitizer_ext, build_type in itertools.product(SANITIZER_EXT, BUILD_TYPES):
# libbacktrace doesn't build on arm64 with gcc.tsan
if os == "heavy-arm64" and compiler == "gcc" and sanitizer_ext == ".tsan":
continue
configurations.append(
{
"os": os,

View File

@@ -8,11 +8,7 @@ REPO_DIR="$(cd "$CURRENT_DIR/../../../" && pwd)"
CONAN_DIR="${CONAN_HOME:-$HOME/.conan2}"
PROFILES_DIR="$CONAN_DIR/profiles"
if [[ -z "$CI" ]]; then
APPLE_CLANG_PROFILE="$CURRENT_DIR/apple-clang-local.profile"
else
APPLE_CLANG_PROFILE="$CURRENT_DIR/apple-clang-ci.profile"
fi
APPLE_CLANG_PROFILE="$CURRENT_DIR/apple-clang.profile"
GCC_PROFILE="$REPO_DIR/docker/ci/conan/gcc.profile"
CLANG_PROFILE="$REPO_DIR/docker/ci/conan/clang.profile"
@@ -21,7 +17,7 @@ SANITIZER_TEMPLATE_FILE="$REPO_DIR/docker/ci/conan/sanitizer_template.profile"
rm -rf "$CONAN_DIR"
conan remote add --index 0 ripple https://conan.ripplex.io
conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev
cp "$REPO_DIR/docker/ci/conan/global.conf" "$CONAN_DIR/global.conf"

View File

@@ -2,9 +2,9 @@ name: Build
on:
push:
branches: [release/*, develop]
branches: [master, release/*, develop]
pull_request:
branches: [release/*, develop]
branches: [master, release/*, develop]
paths:
- .github/workflows/build.yml
@@ -28,9 +28,8 @@ on:
workflow_dispatch:
concurrency:
# Develop branch: Each run gets unique group (using run_number) for parallel execution
# Other branches: Shared group with cancel-in-progress to stop old runs when new commits are pushed
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/develop' && github.run_number || 'branch' }}
# Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
@@ -43,10 +42,7 @@ jobs:
os: [heavy]
conan_profile: [gcc, clang]
build_type: [Release, Debug]
container:
[
'{ "image": "ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5" }',
]
container: ['{ "image": "ghcr.io/xrplf/clio-ci:latest" }']
static: [true]
include:
@@ -73,7 +69,7 @@ jobs:
uses: ./.github/workflows/build_impl.yml
with:
runs_on: heavy
container: '{ "image": "ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5" }'
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
conan_profile: gcc
build_type: Debug
disable_cache: false
@@ -90,12 +86,12 @@ jobs:
needs: build-and-test
runs-on: heavy
container:
image: ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5
image: ghcr.io/xrplf/clio-ci:latest
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v5
- uses: actions/download-artifact@v4
with:
name: clio_server_Linux_Release_gcc

View File

@@ -57,12 +57,6 @@ on:
type: string
default: all
expected_version:
description: Expected version of the clio_server binary
required: false
type: string
default: ""
jobs:
build:
uses: ./.github/workflows/build_impl.yml
@@ -77,7 +71,6 @@ jobs:
upload_clio_server: ${{ inputs.upload_clio_server }}
targets: ${{ inputs.targets }}
analyze_build_time: false
expected_version: ${{ inputs.expected_version }}
test:
needs: build

View File

@@ -48,7 +48,7 @@ jobs:
- name: Download Clio binary from artifact
if: ${{ inputs.artifact_name != null }}
uses: actions/download-artifact@v5
uses: actions/download-artifact@v4
with:
name: ${{ inputs.artifact_name }}
path: ./docker/clio/artifact/
@@ -73,8 +73,7 @@ jobs:
elif [[ $artifact == *.tar.gz ]]; then
tar -xvf $artifact
fi
chmod +x ./clio_server
mv ./clio_server ../
mv clio_server ../
cd ../
rm -rf ./artifact

View File

@@ -53,12 +53,6 @@ on:
required: true
type: boolean
expected_version:
description: Expected version of the clio_server binary
required: false
type: string
default: ""
secrets:
CODECOV_TOKEN:
required: false
@@ -77,10 +71,6 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# We need to fetch tags to have correct version in the release
# The workaround is based on https://github.com/actions/checkout/issues/1467
fetch-tags: true
ref: ${{ github.ref }}
- name: Prepare runner
uses: ./.github/actions/prepare_runner
@@ -110,7 +100,6 @@ jobs:
code_coverage: ${{ inputs.code_coverage }}
static: ${{ inputs.static }}
time_trace: ${{ inputs.analyze_build_time }}
use_mold: ${{ runner.os != 'macOS' }}
- name: Build Clio
uses: ./.github/actions/build_clio
@@ -185,6 +174,7 @@ jobs:
# This is run as part of the build job, because it requires the following:
# - source code
# - generated source code (Build.cpp)
# - conan packages
# - .gcno files in build directory
#
@@ -193,18 +183,6 @@ jobs:
if: ${{ inputs.code_coverage }}
uses: ./.github/actions/code_coverage
- name: Verify expected version
if: ${{ inputs.expected_version != '' }}
shell: bash
run: |
set -e
EXPECTED_VERSION="clio-${{ inputs.expected_version }}"
actual_version=$(./build/clio_server --version)
if [[ "$actual_version" != "$EXPECTED_VERSION" ]]; then
echo "Expected version '$EXPECTED_VERSION', but got '$actual_version'"
exit 1
fi
# `codecov/codecov-action` will rerun `gcov` if it's available and build directory is present
# To prevent this from happening, we run this action in a separate workflow
#

View File

@@ -17,18 +17,13 @@ jobs:
name: Build Clio / `libXRPL ${{ github.event.client_payload.version }}`
runs-on: heavy
container:
image: ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5
image: ghcr.io/xrplf/clio-ci:latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Prepare runner
uses: ./.github/actions/prepare_runner
with:
disable_ccache: true
- name: Update libXRPL version requirement
shell: bash
run: |
@@ -38,7 +33,12 @@ jobs:
- name: Update conan lockfile
shell: bash
run: |
conan lock create . -o '&:tests=True' -o '&:benchmark=True' --profile:all ${{ env.CONAN_PROFILE }}
conan lock create . -o '&:tests=True' -o '&:benchmark=True'
- name: Prepare runner
uses: ./.github/actions/prepare_runner
with:
disable_ccache: true
- name: Run conan and cmake
uses: ./.github/actions/generate
@@ -62,10 +62,10 @@ jobs:
needs: build
runs-on: heavy
container:
image: ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5
image: ghcr.io/xrplf/clio-ci:latest
steps:
- uses: actions/download-artifact@v5
- uses: actions/download-artifact@v4
with:
name: clio_tests_check_libxrpl

View File

@@ -10,7 +10,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: ytanikin/pr-conventional-commits@b72758283dcbee706975950e96bc4bf323a8d8c0 # v1.4.2
- uses: ytanikin/pr-conventional-commits@8267db1bacc237419f9ed0228bb9d94e94271a1d # v1.4.1
with:
task_types: '["build","feat","fix","docs","test","ci","style","refactor","perf","chore"]'
add_label: false

View File

@@ -24,7 +24,7 @@ jobs:
clang_tidy:
runs-on: heavy
container:
image: ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5
image: ghcr.io/xrplf/clio-ci:latest
permissions:
contents: write

View File

@@ -14,7 +14,7 @@ jobs:
build:
runs-on: ubuntu-latest
container:
image: ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5
image: ghcr.io/xrplf/clio-ci:latest
steps:
- name: Checkout
@@ -22,11 +22,6 @@ jobs:
with:
lfs: true
- name: Prepare runner
uses: ./.github/actions/prepare_runner
with:
disable_ccache: true
- name: Create build directory
run: mkdir build_docs

View File

@@ -39,17 +39,17 @@ jobs:
conan_profile: gcc
build_type: Release
static: true
container: '{ "image": "ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5" }'
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
- os: heavy
conan_profile: gcc
build_type: Debug
static: true
container: '{ "image": "ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5" }'
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
- os: heavy
conan_profile: gcc.ubsan
build_type: Release
static: false
container: '{ "image": "ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5" }'
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
uses: ./.github/workflows/build_and_test.yml
with:
@@ -72,7 +72,7 @@ jobs:
include:
- os: heavy
conan_profile: clang
container: '{ "image": "ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5" }'
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
static: true
- os: macos15
conan_profile: apple-clang
@@ -96,10 +96,11 @@ jobs:
uses: ./.github/workflows/release_impl.yml
with:
overwrite_release: true
prerelease: true
title: "Clio development (nightly) build"
version: nightly
header: >
# Release notes
> **Note:** Please remember that this is a development release and it is not recommended for production use.
Changelog (including previous releases): <https://github.com/XRPLF/clio/commits/nightly>

View File

@@ -40,11 +40,8 @@ jobs:
GH_REPO: ${{ github.repository }}
GH_TOKEN: ${{ github.token }}
with:
commit-message: "style: Update pre-commit hooks"
committer: Clio CI <skuznetsov@ripple.com>
branch: update/pre-commit-hooks
branch-suffix: timestamp
delete-branch: true
title: "style: Update pre-commit hooks"
commit-message: "style: Update pre-commit hooks"
body: Update versions of pre-commit hooks to latest version.
reviewers: "godexsoft,kuznetsss,PeterChen13579,mathbunnyru"

View File

@@ -3,14 +3,15 @@ name: Run pre-commit hooks
on:
pull_request:
push:
branches: [develop]
branches:
- develop
workflow_dispatch:
jobs:
run-hooks:
runs-on: heavy
container:
image: ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5
image: ghcr.io/xrplf/clio-ci:latest
steps:
- name: Checkout Repo ⚡️

View File

@@ -29,7 +29,7 @@ jobs:
conan_profile: gcc
build_type: Release
static: true
container: '{ "image": "ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5" }'
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
uses: ./.github/workflows/build_and_test.yml
with:
@@ -42,17 +42,15 @@ jobs:
run_integration_tests: true
upload_clio_server: true
disable_cache: true
expected_version: ${{ github.event_name == 'push' && github.ref_name || '' }}
release:
needs: build-and-test
uses: ./.github/workflows/release_impl.yml
with:
overwrite_release: false
prerelease: ${{ contains(github.ref_name, '-') }}
title: "${{ github.ref_name}}"
version: "${{ github.ref_name }}"
header: >
${{ contains(github.ref_name, '-') && '> **Note:** Please remember that this is a release candidate and it is not recommended for production use.' || '' }}
generate_changelog: ${{ !contains(github.ref_name, '-') }}
# Introducing Clio version ${{ github.ref_name }}
generate_changelog: true
draft: true

View File

@@ -8,11 +8,6 @@ on:
required: true
type: boolean
prerelease:
description: "Create a prerelease"
required: true
type: boolean
title:
description: "Release title"
required: true
@@ -30,19 +25,19 @@ on:
generate_changelog:
description: "Generate changelog"
required: true
required: false
type: boolean
draft:
description: "Create a draft release"
required: true
required: false
type: boolean
jobs:
release:
runs-on: heavy
container:
image: ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5
image: ghcr.io/xrplf/clio-ci:latest
env:
GH_REPO: ${{ github.repository }}
GH_TOKEN: ${{ github.token }}
@@ -60,7 +55,7 @@ jobs:
with:
disable_ccache: true
- uses: actions/download-artifact@v5
- uses: actions/download-artifact@v4
with:
path: release_artifacts
pattern: clio_server_*
@@ -68,9 +63,7 @@ jobs:
- name: Create release notes
shell: bash
run: |
echo "# Release notes" > "${RUNNER_TEMP}/release_notes.md"
echo "" >> "${RUNNER_TEMP}/release_notes.md"
printf '%s\n' "${{ inputs.header }}" >> "${RUNNER_TEMP}/release_notes.md"
printf '%s\n' "${{ inputs.header }}" > "${RUNNER_TEMP}/release_notes.md"
- name: Generate changelog
shell: bash
@@ -79,13 +72,25 @@ jobs:
LAST_TAG="$(gh release view --json tagName -q .tagName)"
LAST_TAG_COMMIT="$(git rev-parse $LAST_TAG)"
BASE_COMMIT="$(git merge-base HEAD $LAST_TAG_COMMIT)"
git-cliff "${BASE_COMMIT}..HEAD" --ignore-tags "nightly|-b|-rc"
git-cliff "${BASE_COMMIT}..HEAD" --ignore-tags "nightly|-b"
cat CHANGELOG.md >> "${RUNNER_TEMP}/release_notes.md"
- name: Prepare release artifacts
shell: bash
run: .github/scripts/prepare-release-artifacts.sh release_artifacts
- name: Append sha256 checksums
shell: bash
working-directory: release_artifacts
run: |
{
echo '## SHA256 checksums'
echo
echo '```'
cat *.sha256sum
echo '```'
} >> "${RUNNER_TEMP}/release_notes.md"
- name: Upload release notes
uses: actions/upload-artifact@v4
with:
@@ -104,7 +109,7 @@ jobs:
shell: bash
run: |
gh release create "${{ inputs.version }}" \
${{ inputs.prerelease && '--prerelease' || '' }} \
${{ inputs.overwrite_release && '--prerelease' || '' }} \
--title "${{ inputs.title }}" \
--target "${GITHUB_SHA}" \
${{ inputs.draft && '--draft' || '' }} \

View File

@@ -37,20 +37,22 @@ jobs:
strategy:
fail-fast: false
matrix:
compiler: [gcc, clang]
sanitizer_ext: [.asan, .tsan, .ubsan]
build_type: [Release, Debug]
compiler: ["gcc", "clang"]
sanitizer_ext: [".asan", ".tsan", ".ubsan"]
exclude:
# Currently, clang.tsan unit tests hang
- compiler: clang
sanitizer_ext: .tsan
uses: ./.github/workflows/build_and_test.yml
with:
runs_on: heavy
container: '{ "image": "ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5" }'
container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }'
disable_cache: true
conan_profile: ${{ matrix.compiler }}${{ matrix.sanitizer_ext }}
build_type: ${{ matrix.build_type }}
build_type: Release
static: false
# Currently, both gcc.tsan and clang.tsan unit tests hang
run_unit_tests: ${{ matrix.sanitizer_ext != '.tsan' }}
run_unit_tests: true
run_integration_tests: false
upload_clio_server: false
targets: clio_tests clio_integration_tests

View File

@@ -54,7 +54,7 @@ jobs:
with:
fetch-depth: 0
- uses: actions/download-artifact@v5
- uses: actions/download-artifact@v4
with:
name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
@@ -85,7 +85,7 @@ jobs:
if: env.SANITIZER_IGNORE_ERRORS == 'true' && steps.check_report.outputs.found_report == 'true'
uses: actions/upload-artifact@v4
with:
name: sanitizer_report_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
name: ${{ inputs.conan_profile }}_report
path: .sanitizer-report/*
include-hidden-files: true
@@ -144,7 +144,7 @@ jobs:
sleep 5
done
- uses: actions/download-artifact@v5
- uses: actions/download-artifact@v4
with:
name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}

View File

@@ -29,27 +29,12 @@ concurrency:
cancel-in-progress: false
env:
CLANG_MAJOR_VERSION: 19
GCC_MAJOR_VERSION: 14
GCC_VERSION: 14.3.0
GHCR_REPO: ghcr.io/${{ github.repository_owner }}
jobs:
repo:
name: Calculate repo name
runs-on: ubuntu-latest
outputs:
GHCR_REPO: ${{ steps.set-ghcr-repo.outputs.GHCR_REPO }}
steps:
- name: Set GHCR_REPO
id: set-ghcr-repo
run: |
echo "GHCR_REPO=$(echo ghcr.io/${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> ${GITHUB_OUTPUT}
gcc-amd64:
name: Build and push GCC docker image (amd64)
runs-on: heavy
needs: repo
steps:
- uses: actions/checkout@v4
@@ -68,26 +53,22 @@ jobs:
DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
with:
images: |
${{ needs.repo.outputs.GHCR_REPO }}/clio-gcc
${{ env.GHCR_REPO }}/clio-gcc
rippleci/clio_gcc
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/compilers/gcc
tags: |
type=raw,value=amd64-latest
type=raw,value=amd64-${{ env.GCC_MAJOR_VERSION }}
type=raw,value=amd64-${{ env.GCC_VERSION }}
type=raw,value=amd64-12
type=raw,value=amd64-12.3.0
type=raw,value=amd64-${{ github.sha }}
platforms: linux/amd64
build_args: |
GCC_MAJOR_VERSION=${{ env.GCC_MAJOR_VERSION }}
GCC_VERSION=${{ env.GCC_VERSION }}
dockerhub_repo: rippleci/clio_gcc
dockerhub_description: GCC compiler for XRPLF/clio.
gcc-arm64:
name: Build and push GCC docker image (arm64)
runs-on: heavy-arm64
needs: repo
steps:
- uses: actions/checkout@v4
@@ -106,26 +87,23 @@ jobs:
DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
with:
images: |
${{ needs.repo.outputs.GHCR_REPO }}/clio-gcc
${{ env.GHCR_REPO }}/clio-gcc
rippleci/clio_gcc
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/compilers/gcc
tags: |
type=raw,value=arm64-latest
type=raw,value=arm64-${{ env.GCC_MAJOR_VERSION }}
type=raw,value=arm64-${{ env.GCC_VERSION }}
type=raw,value=arm64-12
type=raw,value=arm64-12.3.0
type=raw,value=arm64-${{ github.sha }}
platforms: linux/arm64
build_args: |
GCC_MAJOR_VERSION=${{ env.GCC_MAJOR_VERSION }}
GCC_VERSION=${{ env.GCC_VERSION }}
dockerhub_repo: rippleci/clio_gcc
dockerhub_description: GCC compiler for XRPLF/clio.
gcc-merge:
name: Merge and push multi-arch GCC docker image
runs-on: heavy
needs: [repo, gcc-amd64, gcc-arm64]
needs: [gcc-amd64, gcc-arm64]
steps:
- uses: actions/checkout@v4
@@ -141,7 +119,7 @@ jobs:
- name: Login to GitHub Container Registry
if: github.event_name != 'pull_request'
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
@@ -149,19 +127,23 @@ jobs:
- name: Login to DockerHub
if: github.event_name != 'pull_request'
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_PW }}
- name: Make GHCR_REPO lowercase
run: |
echo "GHCR_REPO_LC=$(echo ${{env.GHCR_REPO}} | tr '[:upper:]' '[:lower:]')" >> ${GITHUB_ENV}
- name: Create and push multi-arch manifest
if: github.event_name != 'pull_request' && steps.changed-files.outputs.any_changed == 'true'
run: |
for image in ${{ needs.repo.outputs.GHCR_REPO }}/clio-gcc rippleci/clio_gcc; do
for image in ${{ env.GHCR_REPO_LC }}/clio-gcc rippleci/clio_gcc; do
docker buildx imagetools create \
-t $image:latest \
-t $image:${{ env.GCC_MAJOR_VERSION }} \
-t $image:${{ env.GCC_VERSION }} \
-t $image:12 \
-t $image:12.3.0 \
-t $image:${{ github.sha }} \
$image:arm64-latest \
$image:amd64-latest
@@ -170,7 +152,6 @@ jobs:
clang:
name: Build and push Clang docker image
runs-on: heavy
needs: repo
steps:
- uses: actions/checkout@v4
@@ -189,24 +170,21 @@ jobs:
DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
with:
images: |
${{ needs.repo.outputs.GHCR_REPO }}/clio-clang
${{ env.GHCR_REPO }}/clio-clang
rippleci/clio_clang
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/compilers/clang
tags: |
type=raw,value=latest
type=raw,value=${{ env.CLANG_MAJOR_VERSION }}
type=raw,value=16
type=raw,value=${{ github.sha }}
platforms: linux/amd64,linux/arm64
build_args: |
CLANG_MAJOR_VERSION=${{ env.CLANG_MAJOR_VERSION }}
dockerhub_repo: rippleci/clio_clang
dockerhub_description: Clang compiler for XRPLF/clio.
tools-amd64:
name: Build and push tools docker image (amd64)
tools:
name: Build and push tools docker image
runs-on: heavy
needs: [repo, gcc-merge]
steps:
- uses: actions/checkout@v4
@@ -223,87 +201,18 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
images: |
${{ needs.repo.outputs.GHCR_REPO }}/clio-tools
${{ env.GHCR_REPO }}/clio-tools
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/tools
tags: |
type=raw,value=amd64-latest
type=raw,value=amd64-${{ github.sha }}
platforms: linux/amd64
build_args: |
GHCR_REPO=${{ needs.repo.outputs.GHCR_REPO }}
GCC_VERSION=${{ env.GCC_VERSION }}
tools-arm64:
name: Build and push tools docker image (arm64)
runs-on: heavy-arm64
needs: [repo, gcc-merge]
steps:
- uses: actions/checkout@v4
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
with:
files: "docker/tools/**"
- uses: ./.github/actions/build_docker_image
if: steps.changed-files.outputs.any_changed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
images: |
${{ needs.repo.outputs.GHCR_REPO }}/clio-tools
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/tools
tags: |
type=raw,value=arm64-latest
type=raw,value=arm64-${{ github.sha }}
platforms: linux/arm64
build_args: |
GHCR_REPO=${{ needs.repo.outputs.GHCR_REPO }}
GCC_VERSION=${{ env.GCC_VERSION }}
tools-merge:
name: Merge and push multi-arch tools docker image
runs-on: heavy
needs: [repo, tools-amd64, tools-arm64]
steps:
- uses: actions/checkout@v4
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46.0.5
with:
files: "docker/tools/**"
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Container Registry
if: github.event_name != 'pull_request'
uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Create and push multi-arch manifest
if: github.event_name != 'pull_request' && steps.changed-files.outputs.any_changed == 'true'
run: |
image=${{ needs.repo.outputs.GHCR_REPO }}/clio-tools
docker buildx imagetools create \
-t $image:latest \
-t $image:${{ github.sha }} \
$image:arm64-latest \
$image:amd64-latest
type=raw,value=latest
type=raw,value=${{ github.sha }}
platforms: linux/amd64,linux/arm64
ci:
name: Build and push CI docker image
runs-on: heavy
needs: [repo, gcc-merge, clang, tools-merge]
needs: [gcc-merge, clang, tools]
steps:
- uses: actions/checkout@v4
@@ -314,19 +223,14 @@ jobs:
DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }}
with:
images: |
${{ needs.repo.outputs.GHCR_REPO }}/clio-ci
${{ env.GHCR_REPO }}/clio-ci
rippleci/clio_ci
push_image: ${{ github.event_name != 'pull_request' }}
directory: docker/ci
tags: |
type=raw,value=latest
type=raw,value=gcc_${{ env.GCC_MAJOR_VERSION }}_clang_${{ env.CLANG_MAJOR_VERSION }}
type=raw,value=gcc_12_clang_16
type=raw,value=${{ github.sha }}
platforms: linux/amd64,linux/arm64
build_args: |
GHCR_REPO=${{ needs.repo.outputs.GHCR_REPO }}
CLANG_MAJOR_VERSION=${{ env.CLANG_MAJOR_VERSION }}
GCC_MAJOR_VERSION=${{ env.GCC_MAJOR_VERSION }}
GCC_VERSION=${{ env.GCC_VERSION }}
dockerhub_repo: rippleci/clio_ci
dockerhub_description: CI image for XRPLF/clio.

View File

@@ -10,32 +10,29 @@ on:
required: false
default: false
type: boolean
force_upload:
description: "Force upload of all dependencies"
required: false
default: false
type: boolean
pull_request:
branches: [develop]
branches:
- develop
paths:
- .github/workflows/upload_conan_deps.yml
- .github/actions/generate/action.yml
- .github/actions/prepare_runner/action.yml
- ".github/scripts/conan/**"
- "!.github/scripts/conan/apple-clang-local.profile"
- .github/scripts/conan/generate_matrix.py
- .github/scripts/conan/init.sh
- conanfile.py
- conan.lock
push:
branches: [develop]
branches:
- develop
paths:
- .github/workflows/upload_conan_deps.yml
- .github/actions/generate/action.yml
- .github/actions/prepare_runner/action.yml
- ".github/scripts/conan/**"
- "!.github/scripts/conan/apple-clang-local.profile"
- .github/scripts/conan/generate_matrix.py
- .github/scripts/conan/init.sh
- conanfile.py
- conan.lock
@@ -102,4 +99,4 @@ jobs:
- name: Upload Conan packages
if: github.event_name != 'pull_request' && github.event_name != 'schedule'
run: conan upload "*" -r=ripple --confirm ${{ github.event.inputs.force_upload == 'true' && '--force' || '' }}
run: conan upload "*" -r=ripple --confirm

View File

@@ -18,7 +18,7 @@ jobs:
fetch-depth: 0
- name: Download report artifact
uses: actions/download-artifact@v5
uses: actions/download-artifact@v4
with:
name: coverage-report.xml
path: build

1
.gitignore vendored
View File

@@ -9,3 +9,4 @@
.sanitizer-report
CMakeUserPresets.json
config.json
src/util/build/Build.cpp

View File

@@ -55,12 +55,6 @@ repos:
--ignore-words=pre-commit-hooks/codespell_ignore.txt,
]
- repo: https://github.com/trufflesecurity/trufflehog
rev: a05cf0859455b5b16317ee22d809887a4043cdf0 # frozen: v3.90.2
hooks:
- id: trufflehog
entry: trufflehog git file://. --since-commit HEAD --max-depth=1 --no-verification --fail
# Running some C++ hooks before clang-format
# to ensure that the style is consistent.
- repo: local
@@ -86,7 +80,7 @@ repos:
language: script
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: 182152eb8c5ce1cf5299b956b04392c86bd8a126 # frozen: v20.1.8
rev: 6b9072cd80691b1b48d80046d884409fb1d962d1 # frozen: v20.1.7
hooks:
- id: clang-format
args: [--style=file]

View File

@@ -1,5 +1,7 @@
cmake_minimum_required(VERSION 3.20)
set(CMAKE_PROJECT_INCLUDE_BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ClioVersion.cmake)
project(clio VERSION ${CLIO_VERSION} HOMEPAGE_URL "https://github.com/XRPLF/clio"
DESCRIPTION "An XRP Ledger API Server"
)
@@ -16,7 +18,6 @@ option(lint "Run clang-tidy checks during compilation" FALSE)
option(static "Statically linked Clio" FALSE)
option(snapshot "Build snapshot tool" FALSE)
option(time_trace "Build using -ftime-trace to create compiler trace reports" FALSE)
option(use_mold "Use mold linker" FALSE)
# ========================================================================== #
set(san "" CACHE STRING "Add sanitizer instrumentation")
@@ -30,7 +31,6 @@ set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
include(Ccache)
include(CheckCXXCompilerFlag)
include(ClangTidy)
include(Linker)
add_library(clio_options INTERFACE)
target_compile_features(clio_options INTERFACE cxx_std_23) # Clio needs c++23 but deps can remain c++20 for now
@@ -69,17 +69,15 @@ endif ()
# Enable selected sanitizer if enabled via `san`
if (san)
set(SUPPORTED_SANITIZERS "address" "thread" "memory" "undefined")
if (NOT san IN_LIST SUPPORTED_SANITIZERS)
list(FIND SUPPORTED_SANITIZERS "${san}" INDEX)
if (INDEX EQUAL -1)
message(FATAL_ERROR "Error: Unsupported sanitizer '${san}'. Supported values are: ${SUPPORTED_SANITIZERS}.")
endif ()
# Sanitizers recommend minimum of -O1 for reasonable performance so we enable it for debug builds
set(SAN_OPTIMIZATION_FLAG "")
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
set(SAN_OPTIMIZATION_FLAG -O1)
endif ()
target_compile_options(clio_options INTERFACE ${SAN_OPTIMIZATION_FLAG} ${SAN_FLAG} -fno-omit-frame-pointer)
target_compile_options(
clio_options INTERFACE # Sanitizers recommend minimum of -O1 for reasonable performance
$<$<CONFIG:Debug>:-O1> ${SAN_FLAG} -fno-omit-frame-pointer
)
target_compile_definitions(
clio_options INTERFACE $<$<STREQUAL:${san},address>:SANITIZER=ASAN> $<$<STREQUAL:${san},thread>:SANITIZER=TSAN>
$<$<STREQUAL:${san},memory>:SANITIZER=MSAN> $<$<STREQUAL:${san},undefined>:SANITIZER=UBSAN>

View File

@@ -7,12 +7,10 @@ target_sources(
Playground.cpp
# ExecutionContext
util/async/ExecutionContextBenchmarks.cpp
# Logger
util/log/LoggerBenchmark.cpp
)
include(deps/gbench)
target_include_directories(clio_benchmark PRIVATE .)
target_link_libraries(clio_benchmark PUBLIC clio_util benchmark::benchmark_main)
target_link_libraries(clio_benchmark PUBLIC clio_etl benchmark::benchmark_main)
set_target_properties(clio_benchmark PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})

View File

@@ -1,135 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
#include "util/prometheus/Prometheus.hpp"
#include <benchmark/benchmark.h>
#include <boost/log/core/core.hpp>
#include <boost/log/utility/setup/common_attributes.hpp>
#include <barrier>
#include <chrono>
#include <cstddef>
#include <filesystem>
#include <string>
#include <thread>
#include <vector>
using namespace util;
struct BenchmarkLoggingInitializer {
static constexpr auto kLOG_FORMAT = "%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%";
static void
initFileLogging(LogService::FileLoggingParams const& params)
{
LogService::initFileLogging(params, kLOG_FORMAT);
}
};
namespace {
std::string
uniqueLogDir()
{
auto const epochTime = std::chrono::high_resolution_clock::now().time_since_epoch();
auto const tmpDir = std::filesystem::temp_directory_path();
std::string const dirName =
"logs_" + std::to_string(std::chrono::duration_cast<std::chrono::microseconds>(epochTime).count());
return tmpDir / "clio_benchmark" / dirName;
}
} // anonymous namespace
static void
benchmarkConcurrentFileLogging(benchmark::State& state)
{
boost::log::add_common_attributes();
auto const numThreads = static_cast<size_t>(state.range(0));
auto const messagesPerThread = static_cast<size_t>(state.range(1));
PrometheusService::init(config::getClioConfig());
auto const logDir = uniqueLogDir();
for (auto _ : state) {
state.PauseTiming();
std::filesystem::create_directories(logDir);
BenchmarkLoggingInitializer::initFileLogging({
.logDir = logDir,
.rotationSizeMB = 5,
.dirMaxSizeMB = 125,
.rotationHours = 24,
});
std::vector<std::thread> threads;
threads.reserve(numThreads);
std::chrono::high_resolution_clock::time_point start;
std::barrier barrier(numThreads, [&state, &start]() {
state.ResumeTiming();
start = std::chrono::high_resolution_clock::now();
});
for (size_t threadNum = 0; threadNum < numThreads; ++threadNum) {
threads.emplace_back([threadNum, messagesPerThread, &barrier]() {
barrier.arrive_and_wait();
Logger const threadLogger("Thread_" + std::to_string(threadNum));
for (size_t messageNum = 0; messageNum < messagesPerThread; ++messageNum) {
LOG(threadLogger.info()) << "Test log message #" << messageNum;
}
});
}
for (auto& thread : threads) {
thread.join();
}
boost::log::core::get()->flush();
auto const end = std::chrono::high_resolution_clock::now();
state.SetIterationTime(std::chrono::duration_cast<std::chrono::duration<double>>(end - start).count());
std::filesystem::remove_all(logDir);
}
auto const totalMessages = numThreads * messagesPerThread;
state.counters["TotalMessagesRate"] = benchmark::Counter(totalMessages, benchmark::Counter::kIsRate);
state.counters["Threads"] = numThreads;
state.counters["MessagesPerThread"] = messagesPerThread;
}
// One line of log message is around 110 bytes
// So, 100K messages is around 10.5MB
BENCHMARK(benchmarkConcurrentFileLogging)
->ArgsProduct({
// Number of threads
{1, 2, 4, 8},
// Messages per thread
{10'000, 100'000, 500'000},
})
->UseManualTime()
->Unit(benchmark::kMillisecond);

View File

@@ -23,22 +23,19 @@
namespace util::build {
#ifndef CLIO_VERSION
#error "CLIO_VERSION must be defined"
#endif
static constexpr char kVERSION_STRING[] = CLIO_VERSION;
static constexpr char versionString[] = "@CLIO_VERSION@"; // NOLINT(readability-identifier-naming)
std::string const&
getClioVersionString()
{
static std::string const value = kVERSION_STRING; // NOLINT(readability-identifier-naming)
static std::string const value = versionString; // NOLINT(readability-identifier-naming)
return value;
}
std::string const&
getClioFullVersionString()
{
static std::string const value = "clio-" + getClioVersionString(); // NOLINT(readability-identifier-naming)
static std::string const value = "clio-" + getClioVersionString(); // NOLINT(readability-identifier-naming)
return value;
}

View File

@@ -1,41 +1,42 @@
#[===================================================================[
write version to source
#]===================================================================]
find_package(Git REQUIRED)
set(GIT_COMMAND describe --tags --exact-match)
set(GIT_COMMAND rev-parse --short HEAD)
execute_process(
COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND}
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
OUTPUT_VARIABLE TAG
RESULT_VARIABLE RC
COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE REV
OUTPUT_STRIP_TRAILING_WHITESPACE
)
if (RC EQUAL 0)
# if we are on a tag, use the tag name
set(CLIO_VERSION "${TAG}")
set(DOC_CLIO_VERSION "${TAG}")
else ()
# if not, use YYYYMMDDHMS-<branch>-<git-rev>
set(GIT_COMMAND branch --show-current)
execute_process(
COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE BRANCH
OUTPUT_STRIP_TRAILING_WHITESPACE
)
if (BRANCH STREQUAL "")
set(BRANCH "dev")
endif ()
if (NOT (BRANCH MATCHES master OR BRANCH MATCHES release/*)) # for develop and any other branch name
# YYYYMMDDHMS-<branch>-<git-rev>
set(GIT_COMMAND show -s --date=format:%Y%m%d%H%M%S --format=%cd)
execute_process(
COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE DATE
OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY
COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE DATE
OUTPUT_STRIP_TRAILING_WHITESPACE
)
set(GIT_COMMAND branch --show-current)
execute_process(
COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE BRANCH
OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY
)
set(GIT_COMMAND rev-parse --short HEAD)
execute_process(
COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE REV
OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY
)
set(CLIO_VERSION "${DATE}-${BRANCH}-${REV}")
set(DOC_CLIO_VERSION "develop")
else ()
set(GIT_COMMAND describe --tags)
execute_process(
COMMAND ${GIT_EXECUTABLE} ${GIT_COMMAND} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE CLIO_TAG_VERSION
OUTPUT_STRIP_TRAILING_WHITESPACE
)
set(CLIO_VERSION "${CLIO_TAG_VERSION}")
set(DOC_CLIO_VERSION "${CLIO_TAG_VERSION}")
endif ()
if (CMAKE_BUILD_TYPE MATCHES Debug)
@@ -43,3 +44,5 @@ if (CMAKE_BUILD_TYPE MATCHES Debug)
endif ()
message(STATUS "Build version: ${CLIO_VERSION}")
configure_file(${CMAKE_CURRENT_LIST_DIR}/Build.cpp.in ${CMAKE_CURRENT_LIST_DIR}/../src/util/build/Build.cpp)

View File

@@ -1,8 +0,0 @@
if (use_mold)
if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
message(STATUS "Using Mold linker")
set(CMAKE_LINKER_TYPE MOLD)
else ()
message(FATAL_ERROR "Mold linker is only supported on Linux.")
endif ()
endif ()

View File

@@ -1,46 +1,47 @@
{
"version": "0.5",
"requires": [
"zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1754412218.488",
"xxhash/0.8.2#7856c968c985b2981b707ee8f2413b2b%1754325010.01",
"xrpl/2.5.0@clio/boost-odr#f68e48da1490c0a583052e4f068ada55%1754325014.392",
"sqlite3/3.47.0#7a0904fd061f5f8a2366c294f9387830%1754325009.708",
"soci/4.0.3#a9f8d773cd33e356b5879a4b0564f287%1754412158.144",
"re2/20230301#dfd6e2bf050eb90ddd8729cfb4c844a4%1754412148.014",
"rapidjson/cci.20220822#1b9d8c2256876a154172dc5cfbe447c6%1754325007.656",
"protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1754412120.055",
"openssl/1.1.1v#216374e4fb5b2e0f5ab1fb6f27b5b434%1754325006.553",
"nudb/2.0.8#63990d3e517038e04bf529eb8167f69f%1754325004.398",
"minizip/1.2.13#9e87d57804bd372d6d1e32b1871517a3%1754325004.374",
"lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1754412069.24",
"libuv/1.46.0#dc28c1f653fa197f00db5b577a6f6011%1754325003.592",
"libiconv/1.17#1e65319e945f2d31941a9d28cc13c058%1754325002.385",
"libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1754410401.723",
"libarchive/3.7.6#7e902bb4ac1d20504fb330c0dd040192%1754325001.673",
"http_parser/2.9.4#98d91690d6fd021e9e624218a85d9d97%1754325001.385",
"gtest/1.14.0#f8f0757a574a8dd747d16af62d6eb1b7%1754325000.842",
"grpc/1.50.1#02291451d1e17200293a409410d1c4e1%1754412166.358",
"fmt/11.2.0#579bb2cdf4a7607621beea4eb4651e0f%1754324999.086",
"date/3.0.3#cf28fe9c0aab99fe12da08aa42df65e1%1754324996.727",
"cassandra-cpp-driver/2.17.0#e50919efac8418c26be6671fd702540a%1754324997.363",
"c-ares/1.34.5#b78b91e7cfb1f11ce777a285bbf169c6%1754412042.679",
"bzip2/1.0.8#00b4a4658791c1f06914e087f0e792f5%1754412214.559",
"boost/1.83.0#5d975011d65b51abb2d2f6eb8386b368%1754325043.336",
"benchmark/1.9.4#ce4403f7a24d3e1f907cd9da4b678be4%1749892625.885",
"abseil/20230802.1#f0f91485b111dc9837a68972cb19ca7b%1754412054.336"
"zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1750263732.782",
"xxhash/0.8.2#7856c968c985b2981b707ee8f2413b2b%1750263730.908",
"xrpl/2.5.0#7880d1696f11fceb1d498570f1a184c8%1751035267.743",
"sqlite3/3.47.0#7a0904fd061f5f8a2366c294f9387830%1750263721.79",
"soci/4.0.3#a9f8d773cd33e356b5879a4b0564f287%1750263717.455",
"re2/20230301#dfd6e2bf050eb90ddd8729cfb4c844a4%1750263715.145",
"rapidjson/cci.20220822#1b9d8c2256876a154172dc5cfbe447c6%1750263713.526",
"protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1750263698.841",
"openssl/1.1.1v#216374e4fb5b2e0f5ab1fb6f27b5b434%1750263685.885",
"nudb/2.0.8#63990d3e517038e04bf529eb8167f69f%1750263683.814",
"minizip/1.2.13#9e87d57804bd372d6d1e32b1871517a3%1750263681.745",
"lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1750263679.891",
"libuv/1.46.0#78565d142ac7102776256328a26cdf60%1750263677.819",
"libiconv/1.17#1ae2f60ab5d08de1643a22a81b360c59%1750257497.552",
"libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1750263675.748",
"libarchive/3.7.6#e0453864b2a4d225f06b3304903cb2b7%1750263671.05",
"http_parser/2.9.4#98d91690d6fd021e9e624218a85d9d97%1750263668.751",
"gtest/1.14.0#f8f0757a574a8dd747d16af62d6eb1b7%1750263666.833",
"grpc/1.50.1#02291451d1e17200293a409410d1c4e1%1750263646.614",
"fmt/11.2.0#579bb2cdf4a7607621beea4eb4651e0f%1746298708.362",
"fmt/10.1.1#021e170cf81db57da82b5f737b6906c1%1750263644.741",
"date/3.0.3#cf28fe9c0aab99fe12da08aa42df65e1%1750263643.099",
"cassandra-cpp-driver/2.17.0#e50919efac8418c26be6671fd702540a%1750263632.157",
"c-ares/1.34.5#b78b91e7cfb1f11ce777a285bbf169c6%1750263630.06",
"bzip2/1.0.8#00b4a4658791c1f06914e087f0e792f5%1750263627.95",
"boost/1.83.0#8eb22f36ddfb61f54bbc412c4555bd66%1750263616.444",
"benchmark/1.8.3#1a2ce62c99e2b3feaa57b1f0c15a8c46%1724323740.181",
"abseil/20230802.1#f0f91485b111dc9837a68972cb19ca7b%1750263609.776"
],
"build_requires": [
"zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1754412218.488",
"protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1754412120.055",
"protobuf/3.21.9#64ce20e1d9ea24f3d6c504015d5f6fa8%1754325048.831",
"cmake/3.31.8#dde3bde00bb843687e55aea5afa0e220%1754412060.968",
"b2/5.3.3#107c15377719889654eb9a162a673975%1754412065.321"
"zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1750263732.782",
"protobuf/3.21.12#d927114e28de9f4691a6bbcdd9a529d1%1750263698.841",
"protobuf/3.21.9#64ce20e1d9ea24f3d6c504015d5f6fa8%1750263690.822",
"cmake/3.31.7#57c3e118bcf267552c0ea3f8bee1e7d5%1749863707.208",
"b2/5.3.2#7b5fabfe7088ae933fb3e78302343ea0%1750263614.565"
],
"python_requires": [],
"overrides": {
"boost/1.83.0": [
null,
"boost/1.83.0#5d975011d65b51abb2d2f6eb8386b368"
"boost/1.83.0#8eb22f36ddfb61f54bbc412c4555bd66"
],
"protobuf/3.21.9": [
null,

View File

@@ -20,8 +20,7 @@ class ClioConan(ConanFile):
'coverage': [True, False], # build for test coverage report; create custom target `clio_tests-ccov`
'lint': [True, False], # run clang-tidy checks during compilation
'snapshot': [True, False], # build export/import snapshot tool
'time_trace': [True, False], # build using -ftime-trace to create compiler trace reports
'use_mold': [True, False], # use mold linker for faster linking
'time_trace': [True, False] # build using -ftime-trace to create compiler trace reports
}
requires = [
@@ -31,7 +30,7 @@ class ClioConan(ConanFile):
'protobuf/3.21.12',
'grpc/1.50.1',
'openssl/1.1.1v',
'xrpl/2.5.0@clio/boost-odr',
'xrpl/2.5.0',
'zlib/1.3.1',
'libbacktrace/cci.20210118'
]
@@ -48,7 +47,6 @@ class ClioConan(ConanFile):
'docs': False,
'snapshot': False,
'time_trace': False,
'use_mold': False,
'xrpl/*:tests': False,
'xrpl/*:rocksdb': False,
@@ -73,7 +71,7 @@ class ClioConan(ConanFile):
if self.options.tests or self.options.integration_tests:
self.requires('gtest/1.14.0')
if self.options.benchmark:
self.requires('benchmark/1.9.4')
self.requires('benchmark/1.8.3')
def configure(self):
if self.settings.compiler == 'apple-clang':

View File

@@ -1,11 +1,7 @@
ARG GHCR_REPO=invalid
ARG CLANG_MAJOR_VERSION=invalid
ARG GCC_VERSION=invalid
FROM ghcr.io/xrplf/clio-gcc:12.3.0 AS clio-gcc
FROM ghcr.io/xrplf/clio-tools:latest AS clio-tools
FROM ${GHCR_REPO}/clio-gcc:${GCC_VERSION} AS clio-gcc
FROM ${GHCR_REPO}/clio-tools:latest AS clio-tools
FROM ${GHCR_REPO}/clio-clang:${CLANG_MAJOR_VERSION}
FROM ghcr.io/xrplf/clio-clang:16
ARG DEBIAN_FRONTEND=noninteractive
@@ -53,40 +49,36 @@ RUN apt-get update \
# lxml 6.0.0 is not compatible with our image
'lxml<6.0.0' \
\
cmake \
cmake==3.31.6 \
conan==2.17.0 \
gcovr \
pre-commit \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ARG GCC_MAJOR_VERSION=invalid
# Install custom-built gcc and make ldconfig aware of the new libstdc++ location (for gcc)
# Install gcc-12 and make ldconfig aware of the new libstdc++ location (for gcc)
# Note: Clang is using libc++ instead
COPY --from=clio-gcc /gcc${GCC_MAJOR_VERSION}.deb /
COPY --from=clio-gcc /gcc12.deb /
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
binutils \
libc6-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& dpkg -i /gcc${GCC_MAJOR_VERSION}.deb \
&& rm -rf /gcc${GCC_MAJOR_VERSION}.deb \
&& dpkg -i /gcc12.deb \
&& rm -rf /gcc12.deb \
&& ldconfig
# Rewire to use our custom-built gcc as default compiler
RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${GCC_MAJOR_VERSION} 100 \
&& update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-${GCC_MAJOR_VERSION} 100 \
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${GCC_MAJOR_VERSION} 100 \
&& update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-${GCC_MAJOR_VERSION} 100 \
&& update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-${GCC_MAJOR_VERSION} 100 \
&& update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-${GCC_MAJOR_VERSION} 100 \
&& update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-${GCC_MAJOR_VERSION} 100
# Rewire to use gcc-12 as default compiler
RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \
&& update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 \
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 \
&& update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 \
&& update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 100 \
&& update-alternatives --install /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-12 100 \
&& update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-12 100
COPY --from=clio-tools \
/usr/local/bin/mold \
/usr/local/bin/ld.mold \
/usr/local/bin/ccache \
/usr/local/bin/doxygen \
/usr/local/bin/ClangBuildAnalyzer \
@@ -97,7 +89,7 @@ COPY --from=clio-tools \
WORKDIR /root
# Setup conan
RUN conan remote add --index 0 ripple https://conan.ripplex.io
RUN conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev
WORKDIR /root/.conan2
COPY conan/global.conf ./global.conf

View File

@@ -6,14 +6,13 @@ It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but c
The image is based on Ubuntu 20.04 and contains:
- ccache 4.11.3
- Clang 19
- clang 16.0.6
- ClangBuildAnalyzer 1.6.0
- Conan 2.17.0
- Doxygen 1.12
- GCC 14.3.0
- conan 2.17.0
- doxygen 1.12
- gcc 12.3.0
- gh 2.74
- git-cliff 2.9.1
- mold 2.40.1
- and some other useful tools
Conan is set up to build Clio without any additional steps.

View File

@@ -4,9 +4,8 @@ build_type=Release
compiler=clang
compiler.cppstd=20
compiler.libcxx=libc++
compiler.version=19
compiler.version=16
os=Linux
[conf]
tools.build:compiler_executables={"c": "/usr/bin/clang-19", "cpp": "/usr/bin/clang++-19"}
grpc/1.50.1:tools.build:cxxflags+=["-Wno-missing-template-arg-list-after-template-kw"]
tools.build:compiler_executables={"c": "/usr/bin/clang-16", "cpp": "/usr/bin/clang++-16"}

View File

@@ -4,8 +4,8 @@ build_type=Release
compiler=gcc
compiler.cppstd=20
compiler.libcxx=libstdc++11
compiler.version=14
compiler.version=12
os=Linux
[conf]
tools.build:compiler_executables={"c": "/usr/bin/gcc-14", "cpp": "/usr/bin/g++-14"}
tools.build:compiler_executables={"c": "/usr/bin/gcc-12", "cpp": "/usr/bin/g++-12"}

View File

@@ -1,20 +1,18 @@
FROM ubuntu:22.04
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
libatomic1 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd -g 10001 clio \
&& useradd -u 10000 -g 10001 -s /bin/bash clio
COPY ./clio_server /opt/clio/bin/clio_server
RUN ln -s /opt/clio/bin/clio_server /usr/local/bin/clio_server \
&& mkdir -p /opt/clio/etc/ \
&& mkdir -p /opt/clio/log/ \
&& chown clio:clio /opt/clio/log
&& groupadd -g 10001 clio \
&& useradd -u 10000 -g 10001 -s /bin/bash clio \
&& chown clio:clio /opt/clio/log \
&& apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
libatomic1 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
USER clio
ENTRYPOINT ["/opt/clio/bin/clio_server"]

View File

@@ -8,6 +8,8 @@ SHELL ["/bin/bash", "-c"]
USER root
WORKDIR /root
ARG CLANG_VERSION=16
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
wget \
@@ -16,17 +18,13 @@ RUN apt-get update \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ARG CLANG_MAJOR_VERSION=invalid
# Bump this version to force rebuild of the image
ARG BUILD_VERSION=0
RUN wget --progress=dot:giga https://apt.llvm.org/llvm.sh \
&& chmod +x llvm.sh \
&& ./llvm.sh ${CLANG_MAJOR_VERSION} \
&& ./llvm.sh ${CLANG_VERSION} \
&& rm -rf llvm.sh \
&& apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
libc++-${CLANG_MAJOR_VERSION}-dev \
libc++abi-${CLANG_MAJOR_VERSION}-dev \
libc++-${CLANG_VERSION}-dev \
libc++abi-${CLANG_VERSION}-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

View File

@@ -1,14 +1,16 @@
ARG UBUNTU_VERSION=20.04
ARG GCC_MAJOR_VERSION=invalid
ARG GCC_MAJOR_VERSION=12
FROM ubuntu:$UBUNTU_VERSION AS build
ARG UBUNTU_VERSION
ARG GCC_MAJOR_VERSION
ARG BUILD_VERSION=0
ARG GCC_MINOR_VERSION=3
ARG GCC_PATCH_VERSION=0
ARG GCC_VERSION=${GCC_MAJOR_VERSION}.${GCC_MINOR_VERSION}.${GCC_PATCH_VERSION}
ARG BUILD_VERSION=6
ARG DEBIAN_FRONTEND=noninteractive
ARG TARGETARCH
@@ -25,8 +27,6 @@ RUN apt-get update \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ARG GCC_VERSION
WORKDIR /
RUN wget --progress=dot:giga https://gcc.gnu.org/pub/gcc/releases/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz \
&& tar xf gcc-$GCC_VERSION.tar.gz
@@ -68,13 +68,12 @@ RUN /gcc-$GCC_VERSION/configure \
--enable-checking=release
RUN make -j "$(nproc)"
RUN make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION
RUN export GDB_AUTOLOAD_DIR="/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64" \
&& mkdir -p "$GDB_AUTOLOAD_DIR" \
RUN make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION \
&& mkdir -p /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64 \
&& mv \
/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/lib64/libstdc++.so.*-gdb.py \
$GDB_AUTOLOAD_DIR/
/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/lib64/libstdc++.so.6.0.30-gdb.py \
/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64/libstdc++.so.6.0.30-gdb.py
# Generate deb
WORKDIR /

View File

@@ -1,4 +1,4 @@
Package: gcc-14-ubuntu-UBUNTUVERSION
Package: gcc-12-ubuntu-UBUNTUVERSION
Version: VERSION
Architecture: TARGETARCH
Maintainer: Alex Kremer <akremer@ripple.com>

View File

@@ -1,6 +1,6 @@
services:
clio_develop:
image: ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5
image: ghcr.io/xrplf/clio-ci:latest
volumes:
- clio_develop_conan_data:/root/.conan2/p
- clio_develop_ccache:/root/.ccache

View File

@@ -1,41 +1,24 @@
ARG GHCR_REPO=invalid
ARG GCC_VERSION=invalid
FROM ${GHCR_REPO}/clio-gcc:${GCC_VERSION}
FROM ubuntu:20.04
ARG DEBIAN_FRONTEND=noninteractive
ARG TARGETARCH
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
ARG BUILD_VERSION=1
RUN apt-get update \
&& apt-get install -y --no-install-recommends --no-install-suggests \
bison \
build-essential \
cmake \
flex \
ninja-build \
python3 \
python3-pip \
software-properties-common \
wget \
&& pip3 install -q --no-cache-dir \
cmake \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /tmp
ARG MOLD_VERSION=2.40.1
RUN wget --progress=dot:giga "https://github.com/rui314/mold/archive/refs/tags/v${MOLD_VERSION}.tar.gz" \
&& tar xf "v${MOLD_VERSION}.tar.gz" \
&& cd "mold-${MOLD_VERSION}" \
&& mkdir build \
&& cd build \
&& cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
&& ninja install \
&& rm -rf /tmp/* /var/tmp/*
ARG CCACHE_VERSION=4.11.3
RUN wget --progress=dot:giga "https://github.com/ccache/ccache/releases/download/v${CCACHE_VERSION}/ccache-${CCACHE_VERSION}.tar.gz" \
&& tar xf "ccache-${CCACHE_VERSION}.tar.gz" \
@@ -43,7 +26,7 @@ RUN wget --progress=dot:giga "https://github.com/ccache/ccache/releases/download
&& mkdir build \
&& cd build \
&& cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DENABLE_TESTING=False .. \
&& ninja install \
&& cmake --build . --target install \
&& rm -rf /tmp/* /var/tmp/*
ARG DOXYGEN_VERSION=1.12.0
@@ -53,7 +36,7 @@ RUN wget --progress=dot:giga "https://github.com/doxygen/doxygen/releases/downlo
&& mkdir build \
&& cd build \
&& cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
&& ninja install \
&& cmake --build . --target install \
&& rm -rf /tmp/* /var/tmp/*
ARG CLANG_BUILD_ANALYZER_VERSION=1.6.0
@@ -63,7 +46,7 @@ RUN wget --progress=dot:giga "https://github.com/aras-p/ClangBuildAnalyzer/archi
&& mkdir build \
&& cd build \
&& cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
&& ninja install \
&& cmake --build . --target install \
&& rm -rf /tmp/* /var/tmp/*
ARG GIT_CLIFF_VERSION=2.9.1

View File

@@ -7,7 +7,7 @@
- [Python 3.7](https://www.python.org/downloads/)
- [Conan 2.17.0](https://conan.io/downloads.html)
- [CMake 3.20](https://cmake.org/download/)
- [CMake 3.20, <4.0](https://cmake.org/download/)
- [**Optional**] [GCovr](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html): needed for code coverage generation
- [**Optional**] [CCache](https://ccache.dev/): speeds up compilation if you are going to compile Clio often
@@ -35,7 +35,7 @@ The default profile is the file in `~/.conan2/profiles/default`.
Here are some examples of possible profiles:
**Mac apple-clang 17 example**:
**Mac apple-clang 16 example**:
```text
[settings]
@@ -44,7 +44,7 @@ build_type=Release
compiler=apple-clang
compiler.cppstd=20
compiler.libcxx=libc++
compiler.version=17
compiler.version=16
os=Macos
[conf]
@@ -84,7 +84,7 @@ core.upload:parallel={{os.cpu_count()}}
Make sure artifactory is setup with Conan.
```sh
conan remote add --index 0 ripple https://conan.ripplex.io
conan remote add --index 0 ripple http://18.143.149.228:8081/artifactory/api/conan/dev
```
Now you should be able to download the prebuilt dependencies (including `xrpl` package) on supported platforms.
@@ -163,7 +163,7 @@ To generate the API docs:
It is also possible to build Clio using [Docker](https://www.docker.com/) if you don't want to install all the dependencies on your machine.
```sh
docker run -it ghcr.io/xrplf/clio-ci:a446d85297b3006e6d2c4dc7640368f096afecf5
docker run -it ghcr.io/xrplf/clio-ci:latest
git clone https://github.com/XRPLF/clio
mkdir build && cd build
conan install .. --output-folder . --build missing --settings build_type=Release -o '&:tests=True'

View File

@@ -155,7 +155,7 @@ This document provides a list of all available Clio configuration properties in
- **Required**: True
- **Type**: boolean
- **Default value**: `False`
- **Default value**: `True`
- **Constraints**: None
- **Description**: If set to `True`, allows Clio to start without any ETL source.
@@ -331,7 +331,7 @@ This document provides a list of all available Clio configuration properties in
- **Required**: True
- **Type**: boolean
- **Default value**: `True`
- **Default value**: `False`
- **Constraints**: None
- **Description**: Enables or disables Prometheus metrics.
@@ -339,7 +339,7 @@ This document provides a list of all available Clio configuration properties in
- **Required**: True
- **Type**: boolean
- **Default value**: `True`
- **Default value**: `False`
- **Constraints**: None
- **Description**: Enables or disables compression of Prometheus responses.
@@ -428,7 +428,7 @@ This document provides a list of all available Clio configuration properties in
- **Required**: False
- **Type**: string
- **Default value**: None
- **Constraints**: The value must be one of the following: `trace`, `debug`, `info`, `warning`, `error`, `fatal`.
- **Constraints**: The value must be one of the following: `trace`, `debug`, `info`, `warning`, `error`, `fatal`, `count`.
- **Description**: The log level for the specific log channel.
### log_level
@@ -436,7 +436,7 @@ This document provides a list of all available Clio configuration properties in
- **Required**: True
- **Type**: string
- **Default value**: `info`
- **Constraints**: The value must be one of the following: `trace`, `debug`, `info`, `warning`, `error`, `fatal`.
- **Constraints**: The value must be one of the following: `trace`, `debug`, `info`, `warning`, `error`, `fatal`, `count`.
- **Description**: The general logging level of Clio. This level is applied to all log channels that do not have an explicitly defined logging level.
### log_format
@@ -451,7 +451,7 @@ This document provides a list of all available Clio configuration properties in
- **Required**: True
- **Type**: boolean
- **Default value**: `False`
- **Default value**: `True`
- **Constraints**: None
- **Description**: Enables or disables logging to the console.
@@ -507,7 +507,7 @@ This document provides a list of all available Clio configuration properties in
- **Required**: True
- **Type**: boolean
- **Default value**: `False`
- **Default value**: `True`
- **Constraints**: None
- **Description**: Indicates if the server is allowed to write data to the database.

View File

@@ -31,7 +31,7 @@
"etl_sources": [
{
"ip": "127.0.0.1",
"ws_port": "6006",
"ws_port": "6005",
"grpc_port": "50051"
}
],

View File

@@ -10,5 +10,4 @@ target_link_libraries(
clio_web
clio_rpc
clio_migration
PRIVATE Boost::program_options
)

View File

@@ -41,6 +41,7 @@
#include "util/build/Build.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
#include "util/prometheus/Prometheus.hpp"
#include "web/AdminVerificationStrategy.hpp"
#include "web/RPCServerHandler.hpp"
#include "web/Server.hpp"
@@ -90,6 +91,7 @@ ClioApplication::ClioApplication(util::config::ClioConfigDefinition const& confi
: config_(config), signalsHandler_{config_}
{
LOG(util::LogService::info()) << "Clio version: " << util::build::getClioFullVersionString();
PrometheusService::init(config);
signalsHandler_.subscribeToStop([this]() { appStopper_.stop(); });
}

View File

@@ -19,8 +19,6 @@
#include "app/Stopper.hpp"
#include "util/Spawn.hpp"
#include <boost/asio/spawn.hpp>
#include <functional>
@@ -38,7 +36,7 @@ Stopper::~Stopper()
void
Stopper::setOnStop(std::function<void(boost::asio::yield_context)> cb)
{
util::spawn(ctx_, std::move(cb));
boost::asio::spawn(ctx_, std::move(cb));
}
void

View File

@@ -44,7 +44,7 @@ parseConfig(std::string_view configPath)
std::cerr << "Error parsing json from config: " << configPath << "\n" << json.error().error << std::endl;
return false;
}
auto const errors = getClioConfig().parse(json.value());
auto const errors = gClioConfig.parse(json.value());
if (errors.has_value()) {
for (auto const& err : errors.value()) {
std::cerr << "Issues found in provided config '" << configPath << "':\n";

View File

@@ -97,7 +97,7 @@ HealthCheckHandler::operator()(
boost::asio::yield_context
)
{
static constexpr auto kHEALTH_CHECK_HTML = R"html(
static auto constexpr kHEALTH_CHECK_HTML = R"html(
<!DOCTYPE html>
<html>
<head><title>Test page for Clio</title></head>

View File

@@ -21,7 +21,6 @@
#include "cluster/ClioNode.hpp"
#include "data/BackendInterface.hpp"
#include "util/Spawn.hpp"
#include "util/log/Logger.hpp"
#include <boost/asio/spawn.hpp>
@@ -63,7 +62,7 @@ ClusterCommunicationService::ClusterCommunicationService(
void
ClusterCommunicationService::run()
{
util::spawn(strand_, [this](boost::asio::yield_context yield) {
boost::asio::spawn(strand_, [this](boost::asio::yield_context yield) {
boost::asio::steady_timer timer(yield.get_executor());
while (true) {
timer.expires_after(readInterval_);
@@ -72,7 +71,7 @@ ClusterCommunicationService::run()
}
});
util::spawn(strand_, [this](boost::asio::yield_context yield) {
boost::asio::spawn(strand_, [this](boost::asio::yield_context yield) {
boost::asio::steady_timer timer(yield.get_executor());
while (true) {
doWrite();
@@ -109,7 +108,7 @@ ClioNode
ClusterCommunicationService::selfData() const
{
ClioNode result{};
util::spawn(strand_, [this, &result](boost::asio::yield_context) { result = selfData_; });
boost::asio::spawn(strand_, [this, &result](boost::asio::yield_context) { result = selfData_; });
return result;
}
@@ -120,7 +119,7 @@ ClusterCommunicationService::clusterData() const
return std::unexpected{"Service is not healthy"};
}
std::vector<ClioNode> result;
util::spawn(strand_, [this, &result](boost::asio::yield_context) {
boost::asio::spawn(strand_, [this, &result](boost::asio::yield_context) {
result = otherNodesData_;
result.push_back(selfData_);
});

View File

@@ -23,7 +23,6 @@
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etl/CorruptionDetector.hpp"
#include "util/Spawn.hpp"
#include "util/log/Logger.hpp"
#include <boost/asio/executor_work_guard.hpp>
@@ -109,12 +108,14 @@ synchronous(FnType&& func)
using R = typename boost::result_of<FnType(boost::asio::yield_context)>::type;
if constexpr (!std::is_same_v<R, void>) {
R res;
util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func, &res](auto yield) { res = func(yield); });
boost::asio::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func, &res](auto yield) {
res = func(yield);
});
ctx.run();
return res;
} else {
util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func](auto yield) { func(yield); });
boost::asio::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func](auto yield) { func(yield); });
ctx.run();
}
}

View File

@@ -198,6 +198,39 @@ struct MPTHolderData {
ripple::AccountID holder;
};
/**
* @brief Check whether the supplied object is an offer.
*
* @param object The object to check
* @return true if the object is an offer; false otherwise
*/
template <typename T>
inline bool
isOffer(T const& object)
{
static constexpr short kOFFER_OFFSET = 0x006f;
static constexpr short kSHIFT = 8;
short offerBytes = (object[1] << kSHIFT) | object[2];
return offerBytes == kOFFER_OFFSET;
}
/**
* @brief Check whether the supplied hex represents an offer object.
*
* @param object The object to check
* @return true if the object is an offer; false otherwise
*/
template <typename T>
inline bool
isOfferHex(T const& object)
{
auto blob = ripple::strUnHex(4, object.begin(), object.begin() + 4);
if (blob)
return isOffer(*blob);
return false;
}
/**
* @brief Check whether the supplied object is a dir node.
*
@@ -208,10 +241,6 @@ template <typename T>
inline bool
isDirNode(T const& object)
{
static constexpr auto kMIN_SIZE_REQUIRED = 3;
if (std::size(object) < kMIN_SIZE_REQUIRED)
return false;
static constexpr short kDIR_NODE_SPACE_KEY = 0x0064;
short const spaceKey = (object.data()[1] << 8) | object.data()[2];
return spaceKey == kDIR_NODE_SPACE_KEY;
@@ -235,6 +264,23 @@ isBookDir(T const& key, R const& object)
return !sle[~ripple::sfOwner].has_value();
}
/**
* @brief Get the book out of an offer object.
*
* @param offer The offer to get the book for
* @return Book as ripple::uint256
*/
template <typename T>
inline ripple::uint256
getBook(T const& offer)
{
ripple::SerialIter it{offer.data(), offer.size()};
ripple::SLE const sle{it, {}};
ripple::uint256 book = sle.getFieldH256(ripple::sfBookDirectory);
return book;
}
/**
* @brief Get the book base.
*

View File

@@ -6,7 +6,7 @@ To support additional database types, you can create new classes that implement
## Data Model
The data model used by Clio to read and write ledger data is different from what `rippled` uses. `rippled` uses a novel data structure named [_SHAMap_](https://github.com/XRPLF/rippled/blob/develop/src/xrpld/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record.
The data model used by Clio to read and write ledger data is different from what `rippled` uses. `rippled` uses a novel data structure named [_SHAMap_](https://github.com/ripple/rippled/blob/master/src/ripple/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record.
`rippled` nodes can also generate a proof-tree by forming a subtree with all the path nodes and their neighbors, which can then be used to prove the existence of the leaf node data to other `rippled` nodes. In short, the main purpose of the SHAMap data structure is to facilitate the fast validation of data integrity between different decentralized `rippled` nodes.

View File

@@ -99,7 +99,7 @@ public:
connect() const;
/**
* @brief Connect to the specified keyspace asynchronously.
* @brief Connect to the the specified keyspace asynchronously.
*
* @param keyspace The keyspace to use
* @return A future
@@ -137,7 +137,7 @@ public:
disconnect() const;
/**
* @brief Reconnect to the specified keyspace asynchronously.
* @brief Reconnect to the the specified keyspace asynchronously.
*
* @param keyspace The keyspace to use
* @return A future

View File

@@ -30,8 +30,8 @@
#include <boost/asio.hpp>
#include <boost/asio/associated_executor.hpp>
#include <boost/asio/executor_work_guard.hpp>
#include <boost/asio/io_context.hpp>
#include <boost/asio/io_service.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
@@ -79,7 +79,7 @@ class DefaultExecutionStrategy {
std::condition_variable syncCv_;
boost::asio::io_context ioc_;
std::optional<boost::asio::executor_work_guard<boost::asio::io_context::executor_type>> work_;
std::optional<boost::asio::io_service::work> work_;
std::reference_wrapper<HandleType const> handle_;
std::thread thread_;
@@ -107,7 +107,7 @@ public:
: maxWriteRequestsOutstanding_{settings.maxWriteRequestsOutstanding}
, maxReadRequestsOutstanding_{settings.maxReadRequestsOutstanding}
, writeBatchSize_{settings.writeBatchSize}
, work_{boost::asio::make_work_guard(ioc_)}
, work_{ioc_}
, handle_{std::cref(handle)}
, thread_{[this]() { ioc_.run(); }}
, counters_{std::move(counters)}
@@ -334,7 +334,7 @@ public:
};
auto res = boost::asio::async_compose<CompletionTokenType, void(ResultOrErrorType)>(
std::move(init), token, boost::asio::get_associated_executor(token)
init, token, boost::asio::get_associated_executor(token)
);
numReadRequestsOutstanding_ -= numStatements;
@@ -387,7 +387,7 @@ public:
};
auto res = boost::asio::async_compose<CompletionTokenType, void(ResultOrErrorType)>(
std::move(init), token, boost::asio::get_associated_executor(token)
init, token, boost::asio::get_associated_executor(token)
);
--numReadRequestsOutstanding_;
@@ -456,7 +456,7 @@ public:
};
boost::asio::async_compose<CompletionTokenType, void()>(
std::move(init), token, boost::asio::get_associated_executor(token)
init, token, boost::asio::get_associated_executor(token)
);
numReadRequestsOutstanding_ -= statements.size();

View File

@@ -57,6 +57,7 @@
#include <string>
#include <thread>
#include <utility>
#include <variant>
#include <vector>
using namespace util::config;
@@ -277,46 +278,40 @@ LoadBalancer::forwardToRippled(
return std::unexpected{rpc::ClioError::RpcCommandIsMissing};
auto const cmd = boost::json::value_to<std::string>(request.at("command"));
if (forwardingCache_) {
if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
forwardingCounters_.cacheHit.get() += 1;
return std::move(cachedResponse).value();
if (forwardingCache_ and forwardingCache_->shouldCache(cmd)) {
bool servedFromCache = true;
auto updater = [this, &request, &clientIp, &servedFromCache, isAdmin](boost::asio::yield_context yield)
-> std::expected<util::ResponseExpirationCache::EntryData, util::ResponseExpirationCache::Error> {
servedFromCache = false;
auto result = forwardToRippledImpl(request, clientIp, isAdmin, yield);
if (result.has_value()) {
return util::ResponseExpirationCache::EntryData{
.lastUpdated = std::chrono::steady_clock::now(), .response = std::move(result).value()
};
}
return std::unexpected{
util::ResponseExpirationCache::Error{.status = rpc::Status{result.error()}, .warnings = {}}
};
};
auto result = forwardingCache_->getOrUpdate(
yield, cmd, std::move(updater), [](util::ResponseExpirationCache::EntryData const& entry) {
return not entry.response.contains("error");
}
);
if (servedFromCache) {
++forwardingCounters_.cacheHit.get();
}
}
forwardingCounters_.cacheMiss.get() += 1;
ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
auto numAttempts = 0u;
auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
std::optional<boost::json::object> response;
rpc::ClioError error = rpc::ClioError::EtlConnectionError;
while (numAttempts < sources_.size()) {
auto [res, duration] =
util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
if (res) {
forwardingCounters_.successDuration.get() += duration;
response = std::move(res).value();
break;
if (result.has_value()) {
return std::move(result).value();
}
forwardingCounters_.failDuration.get() += duration;
++forwardingCounters_.retries.get();
error = std::max(error, res.error()); // Choose the best result between all sources
sourceIdx = (sourceIdx + 1) % sources_.size();
++numAttempts;
auto const combinedError = result.error().status.code;
ASSERT(std::holds_alternative<rpc::ClioError>(combinedError), "There could be only ClioError here");
return std::unexpected{std::get<rpc::ClioError>(combinedError)};
}
if (response) {
if (forwardingCache_ and not response->contains("error"))
forwardingCache_->put(cmd, *response);
return std::move(response).value();
}
return std::unexpected{error};
return forwardToRippledImpl(request, clientIp, isAdmin, yield);
}
boost::json::value
@@ -407,4 +402,47 @@ LoadBalancer::chooseForwardingSource()
}
}
std::expected<boost::json::object, rpc::CombinedError>
LoadBalancer::forwardToRippledImpl(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
bool const isAdmin,
boost::asio::yield_context yield
)
{
++forwardingCounters_.cacheMiss.get();
ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
auto numAttempts = 0u;
auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
std::optional<boost::json::object> response;
rpc::ClioError error = rpc::ClioError::EtlConnectionError;
while (numAttempts < sources_.size()) {
auto [res, duration] =
util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
if (res) {
forwardingCounters_.successDuration.get() += duration;
response = std::move(res).value();
break;
}
forwardingCounters_.failDuration.get() += duration;
++forwardingCounters_.retries.get();
error = std::max(error, res.error()); // Choose the best result between all sources
sourceIdx = (sourceIdx + 1) % sources_.size();
++numAttempts;
}
if (response.has_value()) {
return std::move(response).value();
}
return std::unexpected{error};
}
} // namespace etl

View File

@@ -49,7 +49,6 @@
#include <concepts>
#include <cstdint>
#include <expected>
#include <functional>
#include <memory>
#include <optional>
#include <string>
@@ -282,6 +281,14 @@ private:
*/
void
chooseForwardingSource();
std::expected<boost::json::object, rpc::CombinedError>
forwardToRippledImpl(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
bool isAdmin,
boost::asio::yield_context yield
);
};
} // namespace etl

View File

@@ -18,7 +18,6 @@
//==============================================================================
#include "data/DBHelpers.hpp"
#include "util/Assert.hpp"
#include <ripple/protocol/STBase.h>
#include <ripple/protocol/STTx.h>
@@ -72,9 +71,6 @@ getMPTHolderFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
std::optional<MPTHolderData>
getMPTHolderFromObj(std::string const& key, std::string const& blob)
{
// https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033-multi-purpose-tokens#2121-mptoken-ledger-identifier
ASSERT(key.size() == ripple::uint256::size(), "The size of the key is expected to fit uint256 exactly");
ripple::STLedgerEntry const sle =
ripple::STLedgerEntry(ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data()));

View File

@@ -18,7 +18,6 @@
//==============================================================================
#include "data/DBHelpers.hpp"
#include "util/Assert.hpp"
#include <fmt/format.h>
#include <xrpl/basics/base_uint.h>
@@ -360,18 +359,14 @@ getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
std::vector<NFTsData>
getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string const& blob)
{
// https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0020-non-fungible-tokens#tokenpage-id-format
ASSERT(key.size() == ripple::uint256::size(), "The size of the key (token) is expected to fit uint256 exactly");
auto const sle =
std::vector<NFTsData> nfts;
ripple::STLedgerEntry const sle =
ripple::STLedgerEntry(ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data()));
if (sle.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE)
return {};
return nfts;
auto const owner = ripple::AccountID::fromVoid(key.data());
std::vector<NFTsData> nfts;
for (ripple::STObject const& node : sle.getFieldArray(ripple::sfNFTokens))
nfts.emplace_back(node.getFieldH256(ripple::sfNFTokenID), seq, owner, node.getFieldVL(ripple::sfURI));

View File

@@ -23,7 +23,6 @@
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/JS.hpp"
#include "util/Retry.hpp"
#include "util/Spawn.hpp"
#include "util/log/Logger.hpp"
#include "util/prometheus/Label.hpp"
#include "util/prometheus/Prometheus.hpp"
@@ -158,7 +157,7 @@ SubscriptionSource::stop(boost::asio::yield_context yield)
void
SubscriptionSource::subscribe()
{
util::spawn(strand_, [this, _ = boost::asio::make_work_guard(strand_)](boost::asio::yield_context yield) {
boost::asio::spawn(strand_, [this, _ = boost::asio::make_work_guard(strand_)](boost::asio::yield_context yield) {
if (auto connection = wsConnectionBuilder_.connect(yield); connection) {
wsConnection_ = std::move(connection).value();
} else {

View File

@@ -58,6 +58,7 @@
#include <string>
#include <thread>
#include <utility>
#include <variant>
#include <vector>
using namespace util::config;
@@ -283,46 +284,40 @@ LoadBalancer::forwardToRippled(
return std::unexpected{rpc::ClioError::RpcCommandIsMissing};
auto const cmd = boost::json::value_to<std::string>(request.at("command"));
if (forwardingCache_) {
if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
forwardingCounters_.cacheHit.get() += 1;
return std::move(cachedResponse).value();
if (forwardingCache_ and forwardingCache_->shouldCache(cmd)) {
bool servedFromCache = true;
auto updater = [this, &request, &clientIp, &servedFromCache, isAdmin](boost::asio::yield_context yield)
-> std::expected<util::ResponseExpirationCache::EntryData, util::ResponseExpirationCache::Error> {
servedFromCache = false;
auto result = forwardToRippledImpl(request, clientIp, isAdmin, yield);
if (result.has_value()) {
return util::ResponseExpirationCache::EntryData{
.lastUpdated = std::chrono::steady_clock::now(), .response = std::move(result).value()
};
}
return std::unexpected{
util::ResponseExpirationCache::Error{.status = rpc::Status{result.error()}, .warnings = {}}
};
};
auto result = forwardingCache_->getOrUpdate(
yield, cmd, std::move(updater), [](util::ResponseExpirationCache::EntryData const& entry) {
return not entry.response.contains("error");
}
);
if (servedFromCache) {
++forwardingCounters_.cacheHit.get();
}
}
forwardingCounters_.cacheMiss.get() += 1;
ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
auto numAttempts = 0u;
auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
std::optional<boost::json::object> response;
rpc::ClioError error = rpc::ClioError::EtlConnectionError;
while (numAttempts < sources_.size()) {
auto [res, duration] =
util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
if (res) {
forwardingCounters_.successDuration.get() += duration;
response = std::move(res).value();
break;
if (result.has_value()) {
return std::move(result).value();
}
forwardingCounters_.failDuration.get() += duration;
++forwardingCounters_.retries.get();
error = std::max(error, res.error()); // Choose the best result between all sources
sourceIdx = (sourceIdx + 1) % sources_.size();
++numAttempts;
auto const combinedError = result.error().status.code;
ASSERT(std::holds_alternative<rpc::ClioError>(combinedError), "There could be only ClioError here");
return std::unexpected{std::get<rpc::ClioError>(combinedError)};
}
if (response) {
if (forwardingCache_ and not response->contains("error"))
forwardingCache_->put(cmd, *response);
return std::move(response).value();
}
return std::unexpected{error};
return forwardToRippledImpl(request, clientIp, isAdmin, yield);
}
boost::json::value
@@ -413,4 +408,47 @@ LoadBalancer::chooseForwardingSource()
}
}
std::expected<boost::json::object, rpc::CombinedError>
LoadBalancer::forwardToRippledImpl(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
bool isAdmin,
boost::asio::yield_context yield
)
{
++forwardingCounters_.cacheMiss.get();
ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
std::size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
auto numAttempts = 0u;
auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
std::optional<boost::json::object> response;
rpc::ClioError error = rpc::ClioError::EtlConnectionError;
while (numAttempts < sources_.size()) {
auto [res, duration] =
util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
if (res) {
forwardingCounters_.successDuration.get() += duration;
response = std::move(res).value();
break;
}
forwardingCounters_.failDuration.get() += duration;
++forwardingCounters_.retries.get();
error = std::max(error, res.error()); // Choose the best result between all sources
sourceIdx = (sourceIdx + 1) % sources_.size();
++numAttempts;
}
if (response.has_value()) {
return std::move(response).value();
}
return std::unexpected{error};
}
} // namespace etlng

View File

@@ -282,6 +282,14 @@ private:
*/
void
chooseForwardingSource();
std::expected<boost::json::object, rpc::CombinedError>
forwardToRippledImpl(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
bool isAdmin,
boost::asio::yield_context yield
);
};
} // namespace etlng

View File

@@ -155,10 +155,6 @@ TaskManager::spawnLoader(TaskQueue& queue)
<< " tps[" << txnCount / seconds << "], ops[" << objCount / seconds << "]";
monitor_.get().notifySequenceLoaded(data->seq);
} else {
// TODO (https://github.com/XRPLF/clio/issues/1852) this is probably better done with a timeout (on
// coroutine) so that the thread itself is not blocked
queue.awaitTask();
}
}
@@ -183,7 +179,6 @@ TaskManager::stop()
for (auto& loader : loaders_)
loader.abort();
queue_.stop();
wait();
}

View File

@@ -70,13 +70,6 @@ public:
~TaskManager() override;
TaskManager(TaskManager const&) = delete;
TaskManager(TaskManager&&) = delete;
TaskManager&
operator=(TaskManager const&) = delete;
TaskManager&
operator=(TaskManager&&) = delete;
void
run(std::size_t numExtractors) override;

View File

@@ -20,14 +20,10 @@
#pragma once
#include "etlng/Models.hpp"
#include "util/Assert.hpp"
#include "util/Mutex.hpp"
#include <atomic>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
#include <mutex>
#include <optional>
#include <queue>
#include <utility>
@@ -48,6 +44,9 @@ struct ReverseOrderComparator {
* @note This may be a candidate for future improvements if performance proves to be poor (e.g. use a lock free queue)
*/
class TaskQueue {
std::size_t limit_;
std::uint32_t increment_;
struct Data {
std::uint32_t expectedSequence;
std::priority_queue<model::LedgerData, std::vector<model::LedgerData>, ReverseOrderComparator> forwardLoadQueue;
@@ -57,13 +56,8 @@ class TaskQueue {
}
};
std::size_t limit_;
std::uint32_t increment_;
util::Mutex<Data> data_;
std::condition_variable cv_;
std::atomic_bool stopping_ = false;
public:
struct Settings {
std::uint32_t startSeq = 0u; // sequence to start from (for dequeue)
@@ -73,19 +67,13 @@ public:
/**
* @brief Construct a new priority queue
* @param settings Settings for the queue, including starting sequence, increment value, and optional limit
* @note If limit is not set, the queue will have no limit
* @param limit The limit of items allowed simultaneously in the queue
*/
explicit TaskQueue(Settings settings)
: limit_(settings.limit.value_or(0uz)), increment_(settings.increment), data_(settings.startSeq)
{
}
~TaskQueue()
{
ASSERT(stopping_, "stop() must be called before destroying the TaskQueue");
}
/**
* @brief Enqueue a new item onto the queue if space is available
* @note This function blocks until the item is attempted to be added to the queue
@@ -100,8 +88,6 @@ public:
if (limit_ == 0uz or lock->forwardLoadQueue.size() < limit_) {
lock->forwardLoadQueue.push(std::move(item));
cv_.notify_all();
return true;
}
@@ -139,32 +125,6 @@ public:
{
return data_.lock()->forwardLoadQueue.empty();
}
/**
* @brief Awaits for the queue to become non-empty
* @note This function blocks until there is a task or the queue is being destroyed
*/
void
awaitTask()
{
if (stopping_)
return;
auto lock = data_.lock<std::unique_lock>();
cv_.wait(lock, [&] { return stopping_ or not lock->forwardLoadQueue.empty(); });
}
/**
* @brief Notify the queue that it's no longer needed
* @note This must be called before the queue is destroyed
*/
void
stop()
{
// unblock all waiters
stopping_ = true;
cv_.notify_all();
}
};
} // namespace etlng::impl

View File

@@ -25,7 +25,6 @@
#include "util/TerminationHandler.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
#include "util/prometheus/Prometheus.hpp"
#include <cstdlib>
#include <exception>
@@ -53,8 +52,6 @@ try {
if (not app::parseConfig(run.configPath))
return EXIT_FAILURE;
ClioConfigDefinition const& gClioConfig = getClioConfig();
PrometheusService::init(gClioConfig);
if (auto const initSuccess = util::LogService::init(gClioConfig); not initSuccess) {
std::cerr << initSuccess.error() << std::endl;
return EXIT_FAILURE;
@@ -66,11 +63,11 @@ try {
if (not app::parseConfig(migrate.configPath))
return EXIT_FAILURE;
if (auto const initSuccess = util::LogService::init(getClioConfig()); not initSuccess) {
if (auto const initSuccess = util::LogService::init(gClioConfig); not initSuccess) {
std::cerr << initSuccess.error() << std::endl;
return EXIT_FAILURE;
}
app::MigratorApplication migrator{getClioConfig(), migrate.subCmd};
app::MigratorApplication migrator{gClioConfig, migrate.subCmd};
return migrator.run();
}
);

View File

@@ -32,7 +32,7 @@ namespace migration {
*/
struct MigrationManagerInterface : virtual public MigrationInspectorInterface {
/**
* @brief Run the migration according to the given migrator's name
* @brief Run the the migration according to the given migrator's name
*/
virtual void
runMigration(std::string const&) = 0;

View File

@@ -56,7 +56,7 @@ public:
}
/**
* @brief Run the migration according to the given migrator's name
* @brief Run the the migration according to the given migrator's name
*
* @param name The name of the migrator
*/

View File

@@ -2,7 +2,6 @@
add_library(clio_rpc_center)
target_sources(clio_rpc_center PRIVATE RPCCenter.cpp)
target_include_directories(clio_rpc_center PUBLIC "${CMAKE_SOURCE_DIR}/src")
target_link_libraries(clio_rpc_center PUBLIC clio_options)
add_library(clio_rpc)

View File

@@ -157,55 +157,48 @@ public:
return forwardingProxy_.forward(ctx);
}
if (not ctx.isAdmin and responseCache_) {
if (auto res = responseCache_->get(ctx.method); res.has_value())
return Result{std::move(res).value()};
}
if (not ctx.isAdmin and responseCache_ and responseCache_->shouldCache(ctx.method)) {
auto updater = [this, &ctx](boost::asio::yield_context)
-> std::expected<util::ResponseExpirationCache::EntryData, util::ResponseExpirationCache::Error> {
auto result = buildResponseImpl(ctx);
if (backend_->isTooBusy()) {
LOG(log_.error()) << "Database is too busy. Rejecting request";
notifyTooBusy(); // TODO: should we add ctx.method if we have it?
return Result{Status{RippledError::rpcTOO_BUSY}};
}
auto const extracted =
[&result]() -> std::expected<boost::json::object, util::ResponseExpirationCache::Error> {
if (result.response.has_value()) {
return std::move(result.response).value();
}
return std::unexpected{util::ResponseExpirationCache::Error{
.status = std::move(result.response).error(), .warnings = std::move(result.warnings)
}};
}();
auto const method = handlerProvider_->getHandler(ctx.method);
if (!method) {
notifyUnknownCommand();
return Result{Status{RippledError::rpcUNKNOWN_COMMAND}};
}
try {
LOG(perfLog_.debug()) << ctx.tag() << " start executing rpc `" << ctx.method << '`';
auto const context = Context{
.yield = ctx.yield,
.session = ctx.session,
.isAdmin = ctx.isAdmin,
.clientIp = ctx.clientIp,
.apiVersion = ctx.apiVersion
if (extracted.has_value()) {
return util::ResponseExpirationCache::EntryData{
.lastUpdated = std::chrono::steady_clock::now(), .response = std::move(extracted).value()
};
}
return std::unexpected{std::move(extracted).error()};
};
auto v = (*method).process(ctx.params, context);
LOG(perfLog_.debug()) << ctx.tag() << " finish executing rpc `" << ctx.method << '`';
if (not v) {
notifyErrored(ctx.method);
} else if (not ctx.isAdmin and responseCache_) {
responseCache_->put(ctx.method, v.result->as_object());
auto result = responseCache_->getOrUpdate(
ctx.yield,
ctx.method,
std::move(updater),
[&ctx](util::ResponseExpirationCache::EntryData const& entry) {
return not ctx.isAdmin and not entry.response.contains("error");
}
);
if (result.has_value()) {
return Result{std::move(result).value()};
}
return Result{std::move(v)};
} catch (data::DatabaseTimeout const& t) {
LOG(log_.error()) << "Database timeout";
notifyTooBusy();
return Result{Status{RippledError::rpcTOO_BUSY}};
} catch (std::exception const& ex) {
LOG(log_.error()) << ctx.tag() << "Caught exception: " << ex.what();
notifyInternalError();
return Result{Status{RippledError::rpcINTERNAL}};
auto error = std::move(result).error();
Result errorResult{std::move(error.status)};
errorResult.warnings = std::move(error.warnings);
return errorResult;
}
return buildResponseImpl(ctx);
}
/**

View File

@@ -19,9 +19,7 @@
#pragma once
#include "util/Assert.hpp"
#include "util/Mutex.hpp"
#include "util/Spawn.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
#include "util/prometheus/Counter.hpp"
@@ -129,7 +127,7 @@ public:
// Each time we enqueue a job, we want to post a symmetrical job that will dequeue and run the job at the front
// of the job queue.
util::spawn(
boost::asio::spawn(
ioc_,
[this, func = std::forward<FnType>(func), start = std::chrono::system_clock::now()](auto yield) mutable {
auto const run = std::chrono::system_clock::now();

View File

@@ -69,10 +69,9 @@ concept SomeProcessor = (SomeRequirement<T> or SomeModifier<T>);
* @brief A process function that expects both some Input and a Context.
*/
template <typename T>
concept SomeContextProcessWithInput =
requires(T a, typename T::Input const& in, typename T::Output out, Context const& ctx) {
{ a.process(in, ctx) } -> std::same_as<HandlerReturnType<decltype(out)>>;
};
concept SomeContextProcessWithInput = requires(T a, typename T::Input in, typename T::Output out, Context const& ctx) {
{ a.process(in, ctx) } -> std::same_as<HandlerReturnType<decltype(out)>>;
};
/**
* @brief A process function that expects no Input but does take a Context.

View File

@@ -77,7 +77,7 @@ toIso8601(ripple::NetClock::time_point tp)
namespace rpc {
AMMInfoHandler::Result
AMMInfoHandler::process(AMMInfoHandler::Input const& input, Context const& ctx) const
AMMInfoHandler::process(AMMInfoHandler::Input input, Context const& ctx) const
{
using namespace ripple;

View File

@@ -111,7 +111,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
/**

View File

@@ -81,7 +81,7 @@ AccountChannelsHandler::addChannel(std::vector<ChannelResponse>& jsonChannels, r
}
AccountChannelsHandler::Result
AccountChannelsHandler::process(AccountChannelsHandler::Input const& input, Context const& ctx) const
AccountChannelsHandler::process(AccountChannelsHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountChannel's ledger range must be available");

View File

@@ -145,7 +145,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
static void

View File

@@ -43,7 +43,7 @@
namespace rpc {
AccountCurrenciesHandler::Result
AccountCurrenciesHandler::process(AccountCurrenciesHandler::Input const& input, Context const& ctx) const
AccountCurrenciesHandler::process(AccountCurrenciesHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountCurrencies' ledger range must be available");

View File

@@ -110,7 +110,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
/**

View File

@@ -50,7 +50,7 @@
namespace rpc {
AccountInfoHandler::Result
AccountInfoHandler::process(AccountInfoHandler::Input const& input, Context const& ctx) const
AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx) const
{
using namespace data;

View File

@@ -159,7 +159,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
/**

View File

@@ -130,7 +130,7 @@ AccountLinesHandler::addLine(
}
AccountLinesHandler::Result
AccountLinesHandler::process(AccountLinesHandler::Input const& input, Context const& ctx) const
AccountLinesHandler::process(AccountLinesHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountLines' ledger range must be available");

View File

@@ -160,7 +160,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
static void

View File

@@ -49,7 +49,7 @@
namespace rpc {
AccountNFTsHandler::Result
AccountNFTsHandler::process(AccountNFTsHandler::Input const& input, Context const& ctx) const
AccountNFTsHandler::process(AccountNFTsHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountNFT's ledger range must be available");

View File

@@ -118,7 +118,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
/**

View File

@@ -49,7 +49,7 @@
namespace rpc {
AccountObjectsHandler::Result
AccountObjectsHandler::process(AccountObjectsHandler::Input const& input, Context const& ctx) const
AccountObjectsHandler::process(AccountObjectsHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountObject's ledger range must be available");

View File

@@ -134,7 +134,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
/**

View File

@@ -65,7 +65,7 @@ AccountOffersHandler::addOffer(std::vector<Offer>& offers, ripple::SLE const& of
};
AccountOffersHandler::Result
AccountOffersHandler::process(AccountOffersHandler::Input const& input, Context const& ctx) const
AccountOffersHandler::process(AccountOffersHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountOffer's ledger range must be available");

View File

@@ -135,7 +135,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
static void

View File

@@ -52,7 +52,7 @@ namespace rpc {
// TODO: this is currently very similar to nft_history but its own copy for time
// being. we should aim to reuse common logic in some way in the future.
AccountTxHandler::Result
AccountTxHandler::process(AccountTxHandler::Input const& input, Context const& ctx) const
AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "AccountTX's ledger range must be available");

View File

@@ -178,7 +178,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
/**

View File

@@ -41,7 +41,7 @@
namespace rpc {
BookChangesHandler::Result
BookChangesHandler::process(BookChangesHandler::Input const& input, Context const& ctx) const
BookChangesHandler::process(BookChangesHandler::Input input, Context const& ctx) const
{
auto const range = sharedPtrBackend_->fetchLedgerRange();
ASSERT(range.has_value(), "BookChanges' ledger range must be available");

View File

@@ -104,7 +104,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
/**

View File

@@ -42,7 +42,7 @@
namespace rpc {
BookOffersHandler::Result
BookOffersHandler::process(Input const& input, Context const& ctx) const
BookOffersHandler::process(Input input, Context const& ctx) const
{
auto bookMaybe = parseBook(input.paysCurrency, input.paysID, input.getsCurrency, input.getsID, input.domain);
if (!bookMaybe.has_value())

View File

@@ -175,7 +175,7 @@ public:
* @return The result of the operation
*/
Result
process(Input const& input, Context const& ctx) const;
process(Input input, Context const& ctx) const;
private:
/**

Some files were not shown because too many files have changed in this diff Show More