ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-04 03:45:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

style: Add prettier pre-commit hook (#2031)

Browse Source 
There are 2 things to know about prettier:
- it's quite pretty most of the time
- it's not configurable
This commit is contained in:
Ayaz Salikhov
2025-04-25 16:24:45 +01:00
committed by GitHub
parent 593d7298b1
commit 1e0a2f5162
45 changed files with 632 additions and 548 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
.clang-format
View File

@@ -1,5 +1,5 @@
---
Language: Cpp
Language: Cpp
AccessModifierOffset: -4
AlignAfterOpenBracket: BlockIndent
AlignConsecutiveAssignments: false
@@ -22,31 +22,31 @@ BreakBeforeBinaryOperators: false
BreakBeforeBraces: WebKit
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: true
ColumnLimit: 120
CommentPragmas: '^ IWYU pragma:'
ColumnLimit: 120
CommentPragmas: "^ IWYU pragma:"
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4
ContinuationIndentWidth: 4
Cpp11BracedListStyle: true
DerivePointerAlignment: false
DisableFormat: false
DisableFormat: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: true
ForEachMacros: [ Q_FOREACH, BOOST_FOREACH ]
ForEachMacros: [Q_FOREACH, BOOST_FOREACH]
IncludeBlocks: Regroup
IncludeCategories:
- Regex: '^".*"$'
Priority: 1
- Regex: '^<.*\.(h|hpp)>$'
Priority: 2
- Regex: '^<.*>$'
Priority: 3
- Regex: '.*'
Priority: 4
IncludeIsMainRegex: '$'
- Regex: '^".*"$'
Priority: 1
- Regex: '^<.*\.(h|hpp)>$'
Priority: 2
- Regex: "^<.*>$"
Priority: 3
- Regex: ".*"
Priority: 4
IncludeIsMainRegex: "$"
IndentCaseLabels: true
IndentFunctionDeclarationAfterType: false
IndentWidth: 4
IndentWidth: 4
IndentWrappedFunctionNames: false
IndentRequiresClause: true
RequiresClausePosition: OwnLine
@@ -63,18 +63,18 @@ PenaltyExcessCharacter: 1000000
PenaltyReturnTypeOnItsOwnLine: 200
PointerAlignment: Left
QualifierAlignment: Right
ReflowComments: true
SortIncludes: true
ReflowComments: true
SortIncludes: true
SpaceAfterCStyleCast: false
SpaceBeforeAssignmentOperators: true
SpaceBeforeParens: ControlStatements
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 2
SpacesInAngles: false
SpacesInAngles: false
SpacesInContainerLiterals: true
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Cpp11
TabWidth: 8
UseTab: Never
Standard: Cpp11
TabWidth: 8
UseTab: Never

22
.clang-tidy
View File

@@ -1,5 +1,5 @@
---
Checks: '-*,
Checks: "-*,
bugprone-argument-comment,
bugprone-assert-side-effect,
bugprone-bad-signal-to-kill-thread,
@@ -146,7 +146,7 @@ Checks: '-*,
readability-static-definition-in-anonymous-namespace,
readability-suspicious-call-argument,
readability-use-std-min-max
'
"
CheckOptions:
readability-braces-around-statements.ShortStatementLines: 2
@@ -158,21 +158,21 @@ CheckOptions:
readability-identifier-naming.EnumConstantCase: CamelCase
readability-identifier-naming.ScopedEnumConstantCase: CamelCase
readability-identifier-naming.GlobalConstantCase: UPPER_CASE
readability-identifier-naming.GlobalConstantPrefix: 'k'
readability-identifier-naming.GlobalConstantPrefix: "k"
readability-identifier-naming.GlobalVariableCase: CamelCase
readability-identifier-naming.GlobalVariablePrefix: 'g'
readability-identifier-naming.GlobalVariablePrefix: "g"
readability-identifier-naming.ConstexprFunctionCase: camelBack
readability-identifier-naming.ConstexprMethodCase: camelBack
readability-identifier-naming.ClassMethodCase: camelBack
readability-identifier-naming.ClassMemberCase: camelBack
readability-identifier-naming.ClassConstantCase: UPPER_CASE
readability-identifier-naming.ClassConstantPrefix: 'k'
readability-identifier-naming.ClassConstantPrefix: "k"
readability-identifier-naming.StaticConstantCase: UPPER_CASE
readability-identifier-naming.StaticConstantPrefix: 'k'
readability-identifier-naming.StaticConstantPrefix: "k"
readability-identifier-naming.StaticVariableCase: UPPER_CASE
readability-identifier-naming.StaticVariablePrefix: 'k'
readability-identifier-naming.StaticVariablePrefix: "k"
readability-identifier-naming.ConstexprVariableCase: UPPER_CASE
readability-identifier-naming.ConstexprVariablePrefix: 'k'
readability-identifier-naming.ConstexprVariablePrefix: "k"
readability-identifier-naming.LocalConstantCase: camelBack
readability-identifier-naming.LocalVariableCase: camelBack
readability-identifier-naming.TemplateParameterCase: CamelCase
@@ -181,11 +181,11 @@ CheckOptions:
readability-identifier-naming.MemberCase: camelBack
readability-identifier-naming.PrivateMemberSuffix: _
readability-identifier-naming.ProtectedMemberSuffix: _
readability-identifier-naming.PublicMemberSuffix: ''
readability-identifier-naming.FunctionIgnoredRegexp: '.*tag_invoke.*'
readability-identifier-naming.PublicMemberSuffix: ""
readability-identifier-naming.FunctionIgnoredRegexp: ".*tag_invoke.*"
bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true
bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc
misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*;.*ranges_lower_bound\.h;time.h;stdlib.h;__chrono/.*;fmt/chrono.h;boost/uuid/uuid_hash.hpp'
HeaderFilterRegex: '^.*/(src|tests)/.*\.(h|hpp)$'
WarningsAsErrors: '*'
WarningsAsErrors: "*"

252
.cmake-format.yaml
View File

@@ -1,222 +1,222 @@
_help_parse: Options affecting listfile parsing
parse:
_help_additional_commands:
- Specify structure for custom cmake functions
- Specify structure for custom cmake functions
additional_commands:
foo:
flags:
- BAR
- BAZ
- BAR
- BAZ
kwargs:
HEADERS: '*'
SOURCES: '*'
DEPENDS: '*'
HEADERS: "*"
SOURCES: "*"
DEPENDS: "*"
_help_override_spec:
- Override configurations per-command where available
- Override configurations per-command where available
override_spec: {}
_help_vartags:
- Specify variable tags.
- Specify variable tags.
vartags: []
_help_proptags:
- Specify property tags.
- Specify property tags.
proptags: []
_help_format: Options affecting formatting.
format:
_help_disable:
- Disable formatting entirely, making cmake-format a no-op
- Disable formatting entirely, making cmake-format a no-op
disable: false
_help_line_width:
- How wide to allow formatted cmake files
- How wide to allow formatted cmake files
line_width: 120
_help_tab_size:
- How many spaces to tab for indent
- How many spaces to tab for indent
tab_size: 2
_help_use_tabchars:
- If true, lines are indented using tab characters (utf-8
- 0x09) instead of <tab_size> space characters (utf-8 0x20).
- In cases where the layout would require a fractional tab
- character, the behavior of the fractional indentation is
- governed by <fractional_tab_policy>
- If true, lines are indented using tab characters (utf-8
- 0x09) instead of <tab_size> space characters (utf-8 0x20).
- In cases where the layout would require a fractional tab
- character, the behavior of the fractional indentation is
- governed by <fractional_tab_policy>
use_tabchars: false
_help_fractional_tab_policy:
- If <use_tabchars> is True, then the value of this variable
- indicates how fractional indentions are handled during
- whitespace replacement. If set to 'use-space', fractional
- indentation is left as spaces (utf-8 0x20). If set to
- '`round-up` fractional indentation is replaced with a single'
- tab character (utf-8 0x09) effectively shifting the column
- to the next tabstop
- If <use_tabchars> is True, then the value of this variable
- indicates how fractional indentions are handled during
- whitespace replacement. If set to 'use-space', fractional
- indentation is left as spaces (utf-8 0x20). If set to
- "`round-up` fractional indentation is replaced with a single"
- tab character (utf-8 0x09) effectively shifting the column
- to the next tabstop
fractional_tab_policy: use-space
_help_max_subgroups_hwrap:
- If an argument group contains more than this many sub-groups
- (parg or kwarg groups) then force it to a vertical layout.
- If an argument group contains more than this many sub-groups
- (parg or kwarg groups) then force it to a vertical layout.
max_subgroups_hwrap: 4
_help_max_pargs_hwrap:
- If a positional argument group contains more than this many
- arguments, then force it to a vertical layout.
- If a positional argument group contains more than this many
- arguments, then force it to a vertical layout.
max_pargs_hwrap: 6
_help_max_rows_cmdline:
- If a cmdline positional group consumes more than this many
- lines without nesting, then invalidate the layout (and nest)
- If a cmdline positional group consumes more than this many
- lines without nesting, then invalidate the layout (and nest)
max_rows_cmdline: 2
_help_separate_ctrl_name_with_space:
- If true, separate flow control names from their parentheses
- with a space
- If true, separate flow control names from their parentheses
- with a space
separate_ctrl_name_with_space: true
_help_separate_fn_name_with_space:
- If true, separate function names from parentheses with a
- space
- If true, separate function names from parentheses with a
- space
separate_fn_name_with_space: false
_help_dangle_parens:
- If a statement is wrapped to more than one line, than dangle
- the closing parenthesis on its own line.
- If a statement is wrapped to more than one line, than dangle
- the closing parenthesis on its own line.
dangle_parens: true
_help_dangle_align:
- If the trailing parenthesis must be 'dangled' on its on
- 'line, then align it to this reference: `prefix`: the start'
- 'of the statement, `prefix-indent`: the start of the'
- 'statement, plus one indentation level, `child`: align to'
- the column of the arguments
- If the trailing parenthesis must be 'dangled' on its on
- "line, then align it to this reference: `prefix`: the start"
- "of the statement, `prefix-indent`: the start of the"
- "statement, plus one indentation level, `child`: align to"
- the column of the arguments
dangle_align: prefix
_help_min_prefix_chars:
- If the statement spelling length (including space and
- parenthesis) is smaller than this amount, then force reject
- nested layouts.
- If the statement spelling length (including space and
- parenthesis) is smaller than this amount, then force reject
- nested layouts.
min_prefix_chars: 4
_help_max_prefix_chars:
- If the statement spelling length (including space and
- parenthesis) is larger than the tab width by more than this
- amount, then force reject un-nested layouts.
- If the statement spelling length (including space and
- parenthesis) is larger than the tab width by more than this
- amount, then force reject un-nested layouts.
max_prefix_chars: 10
_help_max_lines_hwrap:
- If a candidate layout is wrapped horizontally but it exceeds
- this many lines, then reject the layout.
- If a candidate layout is wrapped horizontally but it exceeds
- this many lines, then reject the layout.
max_lines_hwrap: 2
_help_line_ending:
- What style line endings to use in the output.
- What style line endings to use in the output.
line_ending: unix
_help_command_case:
- Format command names consistently as 'lower' or 'upper' case
- Format command names consistently as 'lower' or 'upper' case
command_case: canonical
_help_keyword_case:
- Format keywords consistently as 'lower' or 'upper' case
- Format keywords consistently as 'lower' or 'upper' case
keyword_case: unchanged
_help_always_wrap:
- A list of command names which should always be wrapped
- A list of command names which should always be wrapped
always_wrap: []
_help_enable_sort:
- If true, the argument lists which are known to be sortable
- will be sorted lexicographicall
- If true, the argument lists which are known to be sortable
- will be sorted lexicographicall
enable_sort: true
_help_autosort:
- If true, the parsers may infer whether or not an argument
- list is sortable (without annotation).
- If true, the parsers may infer whether or not an argument
- list is sortable (without annotation).
autosort: true
_help_require_valid_layout:
- By default, if cmake-format cannot successfully fit
- everything into the desired linewidth it will apply the
- last, most agressive attempt that it made. If this flag is
- True, however, cmake-format will print error, exit with non-
- zero status code, and write-out nothing
- By default, if cmake-format cannot successfully fit
- everything into the desired linewidth it will apply the
- last, most agressive attempt that it made. If this flag is
- True, however, cmake-format will print error, exit with non-
- zero status code, and write-out nothing
require_valid_layout: false
_help_layout_passes:
- A dictionary mapping layout nodes to a list of wrap
- decisions. See the documentation for more information.
- A dictionary mapping layout nodes to a list of wrap
- decisions. See the documentation for more information.
layout_passes: {}
_help_markup: Options affecting comment reflow and formatting.
markup:
_help_bullet_char:
- What character to use for bulleted lists
bullet_char: '*'
- What character to use for bulleted lists
bullet_char: "*"
_help_enum_char:
- What character to use as punctuation after numerals in an
- enumerated list
- What character to use as punctuation after numerals in an
- enumerated list
enum_char: .
_help_first_comment_is_literal:
- If comment markup is enabled, don't reflow the first comment
- block in each listfile. Use this to preserve formatting of
- your copyright/license statements.
- If comment markup is enabled, don't reflow the first comment
- block in each listfile. Use this to preserve formatting of
- your copyright/license statements.
first_comment_is_literal: false
_help_literal_comment_pattern:
- If comment markup is enabled, don't reflow any comment block
- which matches this (regex) pattern. Default is `None`
- (disabled).
- If comment markup is enabled, don't reflow any comment block
- which matches this (regex) pattern. Default is `None`
- (disabled).
literal_comment_pattern: null
_help_fence_pattern:
- Regular expression to match preformat fences in comments
- default= ``r'^\s*([`~]{3}[`~]*)(.*)$'``
- Regular expression to match preformat fences in comments
- default= ``r'^\s*([`~]{3}[`~]*)(.*)$'``
fence_pattern: ^\s*([`~]{3}[`~]*)(.*)$
_help_ruler_pattern:
- Regular expression to match rulers in comments default=
- '``r''^\s*[^\w\s]{3}.*[^\w\s]{3}$''``'
- Regular expression to match rulers in comments default=
- '``r''^\s*[^\w\s]{3}.*[^\w\s]{3}$''``'
ruler_pattern: ^\s*[^\w\s]{3}.*[^\w\s]{3}$
_help_explicit_trailing_pattern:
- If a comment line matches starts with this pattern then it
- is explicitly a trailing comment for the preceeding
- argument. Default is '#<'
explicit_trailing_pattern: '#<'
- If a comment line matches starts with this pattern then it
- is explicitly a trailing comment for the preceeding
- argument. Default is '#<'
explicit_trailing_pattern: "#<"
_help_hashruler_min_length:
- If a comment line starts with at least this many consecutive
- hash characters, then don't lstrip() them off. This allows
- for lazy hash rulers where the first hash char is not
- separated by space
- If a comment line starts with at least this many consecutive
- hash characters, then don't lstrip() them off. This allows
- for lazy hash rulers where the first hash char is not
- separated by space
hashruler_min_length: 10
_help_canonicalize_hashrulers:
- If true, then insert a space between the first hash char and
- remaining hash chars in a hash ruler, and normalize its
- length to fill the column
- If true, then insert a space between the first hash char and
- remaining hash chars in a hash ruler, and normalize its
- length to fill the column
canonicalize_hashrulers: true
_help_enable_markup:
- enable comment markup parsing and reflow
- enable comment markup parsing and reflow
enable_markup: true
_help_lint: Options affecting the linter
lint:
_help_disabled_codes:
- a list of lint codes to disable
- a list of lint codes to disable
disabled_codes: []
_help_function_pattern:
- regular expression pattern describing valid function names
function_pattern: '[0-9a-z_]+'
- regular expression pattern describing valid function names
function_pattern: "[0-9a-z_]+"
_help_macro_pattern:
- regular expression pattern describing valid macro names
macro_pattern: '[0-9A-Z_]+'
- regular expression pattern describing valid macro names
macro_pattern: "[0-9A-Z_]+"
_help_global_var_pattern:
- regular expression pattern describing valid names for
- variables with global (cache) scope
global_var_pattern: '[A-Z][0-9A-Z_]+'
- regular expression pattern describing valid names for
- variables with global (cache) scope
global_var_pattern: "[A-Z][0-9A-Z_]+"
_help_internal_var_pattern:
- regular expression pattern describing valid names for
- variables with global scope (but internal semantic)
- regular expression pattern describing valid names for
- variables with global scope (but internal semantic)
internal_var_pattern: _[A-Z][0-9A-Z_]+
_help_local_var_pattern:
- regular expression pattern describing valid names for
- variables with local scope
local_var_pattern: '[a-z][a-z0-9_]+'
- regular expression pattern describing valid names for
- variables with local scope
local_var_pattern: "[a-z][a-z0-9_]+"
_help_private_var_pattern:
- regular expression pattern describing valid names for
- privatedirectory variables
- regular expression pattern describing valid names for
- privatedirectory variables
private_var_pattern: _[0-9a-z_]+
_help_public_var_pattern:
- regular expression pattern describing valid names for public
- directory variables
public_var_pattern: '[A-Z][0-9A-Z_]+'
- regular expression pattern describing valid names for public
- directory variables
public_var_pattern: "[A-Z][0-9A-Z_]+"
_help_argument_var_pattern:
- regular expression pattern describing valid names for
- function/macro arguments and loop variables.
argument_var_pattern: '[a-z][a-z0-9_]+'
- regular expression pattern describing valid names for
- function/macro arguments and loop variables.
argument_var_pattern: "[a-z][a-z0-9_]+"
_help_keyword_pattern:
- regular expression pattern describing valid names for
- keywords used in functions or macros
keyword_pattern: '[A-Z][0-9A-Z_]+'
- regular expression pattern describing valid names for
- keywords used in functions or macros
keyword_pattern: "[A-Z][0-9A-Z_]+"
_help_max_conditionals_custom_parser:
- In the heuristic for C0201, how many conditionals to match
- within a loop in before considering the loop a parser.
- In the heuristic for C0201, how many conditionals to match
- within a loop in before considering the loop a parser.
max_conditionals_custom_parser: 2
_help_min_statement_spacing:
- Require at least this many newlines between statements
- Require at least this many newlines between statements
min_statement_spacing: 1
_help_max_statement_spacing:
- Require no more than this many newlines between statements
- Require no more than this many newlines between statements
max_statement_spacing: 2
max_returns: 6
max_branches: 12
@@ -226,20 +226,20 @@ lint:
_help_encode: Options affecting file encoding
encode:
_help_emit_byteorder_mark:
- If true, emit the unicode byte-order mark (BOM) at the start
- of the file
- If true, emit the unicode byte-order mark (BOM) at the start
- of the file
emit_byteorder_mark: false
_help_input_encoding:
- Specify the encoding of the input file. Defaults to utf-8
- Specify the encoding of the input file. Defaults to utf-8
input_encoding: utf-8
_help_output_encoding:
- Specify the encoding of the output file. Defaults to utf-8.
- Note that cmake only claims to support utf-8 so be careful
- when using anything else
- Specify the encoding of the output file. Defaults to utf-8.
- Note that cmake only claims to support utf-8 so be careful
- when using anything else
output_encoding: utf-8
_help_misc: Miscellaneous configurations options.
misc:
_help_per_command:
- A dictionary containing any per-command configuration
- overrides. Currently only `command_case` is supported.
- A dictionary containing any per-command configuration
- overrides. Currently only `command_case` is supported.
per_command: {}

9
.github/ISSUE_TEMPLATE/bug_report.md vendored
View File

@@ -3,29 +3,34 @@ name: Bug report
about: Create a report to help us improve
title: "[Title with short description] (Version: [Clio version])"
labels: bug
assignees: ''
assignees: ""
---
<!-- Please search existing issues to avoid creating duplicates. -->
<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
## Issue Description
<!-- Provide a summary for your issue/bug. -->
## Steps to Reproduce
<!-- List in detail the exact steps to reproduce the unexpected behavior of the software. -->
## Expected Result
<!-- Explain in detail what behavior you expected to happen. -->
## Actual Result
<!-- Explain in detail what behavior actually happened. -->
## Environment
<!-- Please describe your environment setup (such as Ubuntu 20.04.2 with Boost 1.82). -->
<!-- Please use the version returned by './clio_server --version' as the version number -->
## Supporting Files
<!-- If you have supporting files such as a log, feel free to post a link here using Github Gist. -->
<!-- Consider adding configuration files with private information removed via Github Gist. -->

7
.github/ISSUE_TEMPLATE/feature_request.md vendored
View File

@@ -3,21 +3,24 @@ name: Feature request
about: Suggest an idea for this project
title: "[Title with short description] (Version: [Clio version])"
labels: enhancement
assignees: ''
assignees: ""
---
<!-- Please search existing issues to avoid creating duplicates. -->
<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
## Summary
<!-- Provide a summary to the feature request -->
## Motivation
<!-- Why do we need this feature? -->
## Solution
<!-- What is the solution? -->
## Paths Not Taken
<!-- What other alternatives have been considered? -->

5
.github/ISSUE_TEMPLATE/question.md vendored
View File

@@ -3,8 +3,7 @@ name: Question
about: A question in form of an issue
title: "[Title with short description] (Version: Clio version)"
labels: question
assignees: ''
assignees: ""
---
<!-- Please search existing issues to avoid creating duplicates. -->
@@ -12,7 +11,9 @@ assignees: ''
<!-- Kindly refrain from posting any credentials or sensitive information in this issue -->
## Question
<!-- Your question -->
## Paths Not Taken
<!-- If applicable, what other alternatives have been considered? -->

2
.github/actions/build_clio/action.yml vendored
View File

@@ -7,7 +7,7 @@ inputs:
substract_threads:
description: An option for the action get_number_of_threads. See get_number_of_threads
required: true
default: '0'
default: "0"
runs:
using: composite
steps:

76
.github/actions/build_docker_image/action.yml vendored
View File

@@ -22,46 +22,46 @@ inputs:
runs:
using: composite
steps:
- name: Login to DockerHub
if: ${{ inputs.push_image == 'true' }}
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #v3.4.0
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_PW }}
- name: Login to DockerHub
if: ${{ inputs.push_image == 'true' }}
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #v3.4.0
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_PW }}
- name: Login to GitHub Container Registry
if: ${{ inputs.push_image == 'true' }}
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #v3.4.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ env.GITHUB_TOKEN }}
- name: Login to GitHub Container Registry
if: ${{ inputs.push_image == 'true' }}
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #v3.4.0
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ env.GITHUB_TOKEN }}
- uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 #v3.6.0
with:
cache-image: false
- uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 #v3.10.0
- uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 #v3.6.0
with:
cache-image: false
- uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 #v3.10.0
- uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 #v5.7.0
id: meta
with:
images: ${{ inputs.image_name }}
tags: ${{ inputs.tags }}
- uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 #v5.7.0
id: meta
with:
images: ${{ inputs.image_name }}
tags: ${{ inputs.tags }}
- name: Build and push
uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 #v6.16.0
with:
context: ${{ inputs.directory }}
platforms: ${{ inputs.platforms }}
push: ${{ inputs.push_image == 'true' }}
tags: ${{ steps.meta.outputs.tags }}
- name: Build and push
uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 #v6.16.0
with:
context: ${{ inputs.directory }}
platforms: ${{ inputs.platforms }}
push: ${{ inputs.push_image == 'true' }}
tags: ${{ steps.meta.outputs.tags }}
- name: Update DockerHub description
if: ${{ inputs.push_image == 'true' }}
uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 #v4.0.2
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_PW }}
repository: ${{ inputs.image_name }}
short-description: ${{ inputs.description }}
readme-filepath: ${{ inputs.directory }}/README.md
- name: Update DockerHub description
if: ${{ inputs.push_image == 'true' }}
uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 #v4.0.2
with:
username: ${{ env.DOCKERHUB_USER }}
password: ${{ env.DOCKERHUB_PW }}
repository: ${{ inputs.image_name }}
short-description: ${{ inputs.description }}
readme-filepath: ${{ inputs.directory }}/README.md

22
.github/actions/create_issue/action.yml vendored
View File

@@ -10,11 +10,11 @@ inputs:
labels:
description: Comma-separated list of labels
required: true
default: 'bug'
default: "bug"
assignees:
description: Comma-separated list of assignees
required: true
default: 'godexsoft,kuznetsss,PeterChen13579'
default: "godexsoft,kuznetsss,PeterChen13579"
outputs:
created_issue_id:
description: Created issue id
@@ -22,12 +22,12 @@ outputs:
runs:
using: composite
steps:
- name: Create an issue
id: create_issue
shell: bash
run: |
echo -e '${{ inputs.body }}' > issue.md
gh issue create --assignee '${{ inputs.assignees }}' --label '${{ inputs.labels }}' --title '${{ inputs.title }}' --body-file ./issue.md > create_issue.log
created_issue=$(cat create_issue.log | sed 's|.*/||')
echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
rm create_issue.log issue.md
- name: Create an issue
id: create_issue
shell: bash
run: |
echo -e '${{ inputs.body }}' > issue.md
gh issue create --assignee '${{ inputs.assignees }}' --label '${{ inputs.labels }}' --title '${{ inputs.title }}' --body-file ./issue.md > create_issue.log
created_issue=$(cat create_issue.log | sed 's|.*/||')
echo "created_issue=$created_issue" >> $GITHUB_OUTPUT
rm create_issue.log issue.md

12
.github/actions/generate/action.yml vendored
View File

@@ -7,27 +7,27 @@ inputs:
conan_cache_hit:
description: Whether conan cache has been downloaded
required: true
default: 'false'
default: "false"
build_type:
description: Build type for third-party libraries and clio. Could be 'Release', 'Debug'
required: true
default: 'Release'
default: "Release"
build_integration_tests:
description: Whether to build integration tests
required: true
default: 'true'
default: "true"
code_coverage:
description: Whether conan's coverage option should be on or not
required: true
default: 'false'
default: "false"
static:
description: Whether Clio is to be statically linked
required: true
default: 'false'
default: "false"
sanitizer:
description: Sanitizer to use
required: true
default: 'false' # false, tsan, asan or ubsan
default: "false" # false, tsan, asan or ubsan
runs:
using: composite
steps:

2
.github/actions/get_number_of_threads/action.yml vendored
View File

@@ -4,7 +4,7 @@ inputs:
substract_threads:
description: How many threads to substract from the calculated number
required: true
default: '0'
default: "0"
outputs:
threads_number:
description: Number of threads to use

38
.github/actions/prepare_runner/action.yml vendored
View File

@@ -11,34 +11,34 @@ runs:
if: ${{ runner.os == 'macOS' }}
shell: bash
run: |
brew install llvm@14 pkg-config ninja bison ccache jq gh conan@1 ca-certificates
echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
brew install llvm@14 pkg-config ninja bison ccache jq gh conan@1 ca-certificates
echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
- name: Install CMake 3.31.6 on mac
if: ${{ runner.os == 'macOS' }}
shell: bash
run: |
# Uninstall any existing cmake
brew uninstall cmake --ignore-dependencies || true
# Uninstall any existing cmake
brew uninstall cmake --ignore-dependencies || true
# Download specific cmake formula
FORMULA_URL="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb"
FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4"
# Download specific cmake formula
FORMULA_URL="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb"
FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4"
mkdir -p /tmp/homebrew-formula
curl -s -L $FORMULA_URL -o /tmp/homebrew-formula/cmake.rb
mkdir -p /tmp/homebrew-formula
curl -s -L $FORMULA_URL -o /tmp/homebrew-formula/cmake.rb
# Verify the downloaded formula
ACTUAL_SHA256=$(shasum -a 256 /tmp/homebrew-formula/cmake.rb | cut -d ' ' -f 1)
if [ "$ACTUAL_SHA256" != "$FORMULA_EXPECTED_SHA256" ]; then
echo "Error: Formula checksum mismatch"
echo "Expected: $FORMULA_EXPECTED_SHA256"
echo "Actual: $ACTUAL_SHA256"
exit 1
fi
# Verify the downloaded formula
ACTUAL_SHA256=$(shasum -a 256 /tmp/homebrew-formula/cmake.rb | cut -d ' ' -f 1)
if [ "$ACTUAL_SHA256" != "$FORMULA_EXPECTED_SHA256" ]; then
echo "Error: Formula checksum mismatch"
echo "Expected: $FORMULA_EXPECTED_SHA256"
echo "Actual: $ACTUAL_SHA256"
exit 1
fi
# Install cmake from the specific formula with force flag
brew install --force /tmp/homebrew-formula/cmake.rb
# Install cmake from the specific formula with force flag
brew install --force /tmp/homebrew-formula/cmake.rb
- name: Fix git permissions on Linux
if: ${{ runner.os == 'Linux' }}

2
.github/actions/restore_cache/action.yml vendored
View File

@@ -17,7 +17,7 @@ inputs:
code_coverage:
description: Whether code coverage is on
required: true
default: 'false'
default: "false"
outputs:
conan_hash:
description: Hash to use as a part of conan cache key

2
.github/actions/save_cache/action.yml vendored
View File

@@ -28,7 +28,7 @@ inputs:
code_coverage:
description: Whether code coverage is on
required: true
default: 'false'
default: "false"
runs:
using: composite
steps:

2
.github/workflows/build_impl.yml vendored
View File

@@ -70,7 +70,7 @@ on:
description: Sanitizer to use
required: false
type: string
default: 'false'
default: "false"
jobs:
build:

8
.github/workflows/check_libxrpl.yml vendored
View File

@@ -13,7 +13,7 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-depth: 0
- name: Update libXRPL version requirement
shell: bash
@@ -23,7 +23,7 @@ jobs:
- name: Prepare runner
uses: ./.github/actions/prepare_runner
with:
disable_ccache: true
disable_ccache: true
- name: Setup conan
uses: ./.github/actions/setup_conan
@@ -83,8 +83,8 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
with:
labels: 'compatibility,bug'
title: 'Proposed libXRPL check failed'
labels: "compatibility,bug"
title: "Proposed libXRPL check failed"
body: >
Clio build or tests failed against `libXRPL ${{ github.event.client_payload.version }}`.

2
.github/workflows/clang-tidy.yml vendored
View File

@@ -83,7 +83,7 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
with:
title: 'Clang-tidy found bugs in code 🐛'
title: "Clang-tidy found bugs in code 🐛"
body: >
Clang-tidy found issues in the code:

18
.github/workflows/nightly.yml vendored
View File

@@ -1,12 +1,12 @@
name: Nightly release
on:
schedule:
- cron: '0 8 * * 1-5'
- cron: "0 8 * * 1-5"
workflow_dispatch:
pull_request:
paths:
- '.github/workflows/nightly.yml'
- '.github/workflows/build_clio_docker_image.yml'
- ".github/workflows/nightly.yml"
- ".github/workflows/build_clio_docker_image.yml"
jobs:
build:
@@ -91,8 +91,8 @@ jobs:
with:
name: clio_integration_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
# To be enabled back once docker in mac runner arrives
# https://github.com/XRPLF/clio/issues/1400
# To be enabled back once docker in mac runner arrives
# https://github.com/XRPLF/clio/issues/1400
- name: Run clio_integration_tests
if: matrix.integration_tests
run: |
@@ -121,6 +121,8 @@ jobs:
run: |
cp ${{ github.workspace }}/.github/workflows/nightly_notes.md "${RUNNER_TEMP}/nightly_notes.md"
cd nightly_release
echo '' >> "${RUNNER_TEMP}/nightly_notes.md"
echo '```' >> "${RUNNER_TEMP}/nightly_notes.md"
for d in $(ls); do
archive_name=$(ls $d)
mv ${d}/${archive_name} ./
@@ -149,8 +151,8 @@ jobs:
secrets: inherit
with:
tags: |
type=raw,value=nightly
type=raw,value=${{ github.sha }}
type=raw,value=nightly
type=raw,value=${{ github.sha }}
artifact_name: clio_server_Linux_Release_gcc
strip_binary: true
publish_image: ${{ github.event_name != 'pull_request' }}
@@ -170,7 +172,7 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
with:
title: 'Nightly release failed 🌙'
title: "Nightly release failed 🌙"
body: >
Nightly release failed:

1
.github/workflows/nightly_notes.md vendored
View File

@@ -3,4 +3,3 @@
Changelog (including previous releases): https://github.com/XRPLF/clio/commits/nightly
## SHA256 checksums
```

2
.github/workflows/sanitizers.yml vendored
View File

@@ -5,7 +5,7 @@ on:
workflow_dispatch:
pull_request:
paths:
- '.github/workflows/sanitizers.yml'
- ".github/workflows/sanitizers.yml"
jobs:
build:

10
.github/workflows/update_docker_ci.yml vendored
View File

@@ -2,14 +2,16 @@ name: Update CI docker image
on:
pull_request:
paths:
- 'docker/ci/**'
- 'docker/compilers/**'
- "docker/ci/**"
- "docker/compilers/**"
- .github/workflows/update_docker_ci.yml
push:
branches: [develop]
paths:
- 'docker/ci/**' # CI image must update when either its dockerfile changes
- 'docker/compilers/**' # or any compilers changed and were pushed by hand
# CI image must update when either its dockerfile changes
# or any compilers changed and were pushed by hand
- "docker/ci/**"
- "docker/compilers/**"
- .github/workflows/update_docker_ci.yml
workflow_dispatch:

9
.pre-commit-config.yaml
View File

@@ -12,7 +12,7 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
# `pre-commit sample-config` default hooks
# `pre-commit sample-config` default hooks
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
@@ -23,3 +23,10 @@ repos:
exclude: ^docs/doxygen-awesome-theme/
- id: trailing-whitespace
exclude: ^docs/doxygen-awesome-theme/
# Autoformat: YAML, JSON, Markdown, etc.
- repo: https://github.com/rbubley/mirrors-prettier
rev: v3.5.3
hooks:
- id: prettier
exclude: ^docs/doxygen-awesome-theme/

71
CONTRIBUTING.md
View File

@@ -1,7 +1,9 @@
# Contributing
Thank you for your interest in contributing to the `clio` project 🙏
To contribute, please:
1. Fork the repository under your own user.
2. Create a new branch on which to commit/push your changes.
3. Write and test your code.
@@ -14,26 +16,29 @@ To contribute, please:
> **Note:** Please read the [Style guide](#style-guide).
## Install git hooks
Please run the following command in order to use git hooks that are helpful for `clio` development.
``` bash
```bash
git config --local core.hooksPath .githooks
```
## Git hooks dependencies
The pre-commit hook requires `clang-format >= 19.0.0` and `cmake-format` to be installed on your machine.
`clang-format` can be installed using `brew` on macOS and default package manager on Linux.
`cmake-format` can be installed using `pip`.
The hook will also attempt to automatically use `doxygen` to verify that everything public in the codebase is covered by doc comments. If `doxygen` is not installed, the hook will raise a warning suggesting to install `doxygen` for future commits.
## Git commands
This sections offers a detailed look at the git commands you will need to use to get your PR submitted.
Please note that there are more than one way to do this and these commands are provided for your convenience.
At this point it's assumed that you have already finished working on your feature/bug.
> **Important:** Before you issue any of the commands below, please hit the `Sync fork` button and make sure your fork's `develop` branch is up-to-date with the main `clio` repository.
``` bash
```bash
# Create a backup of your branch
git branch <your feature branch>_bk
@@ -43,18 +48,20 @@ git pull origin develop
git checkout <your feature branch>
git rebase -i develop
```
For each commit in the list other than the first one, enter `s` to squash.
After this is done, you will have the opportunity to write a message for the squashed commit.
> **Hint:** Please use **imperative mood** in the commit message, and capitalize the first word.
``` bash
```bash
# You should now have a single commit on top of a commit in `develop`
git log
```
> **Note:** If there are merge conflicts, please resolve them now.
``` bash
```bash
# Use the same commit message as you did above
git commit -m 'Your message'
git rebase --continue
@@ -62,27 +69,30 @@ git rebase --continue
> **Important:** If you have no GPG keys set up, please follow [this tutorial](https://docs.github.com/en/authentication/managing-commit-signature-verification/adding-a-gpg-key-to-your-github-account)
``` bash
```bash
# Sign the commit with your GPG key, and push your changes
git commit --amend -S
git push --force
```
## Use ccache (optional)
Clio uses `ccache` to speed up compilation. If you want to use it, please make sure it is installed on your machine.
CMake will automatically detect it and use it if it is available.
## Opening a pull request
When a pull request is open CI will perform checks on the new code.
Title of the pull request and squashed commit should follow [conventional commits specification](https://www.conventionalcommits.org/en/v1.0.0/).
## Fixing issues found during code review
While your code is in review, it's possible that some changes will be requested by reviewer(s).
This section describes the process of adding your fixes.
We assume that you already made the required changes on your feature branch.
``` bash
```bash
# Add the changed code
git add <paths to add>
@@ -95,6 +105,7 @@ git push
```
## After code review
When your PR is approved and ready to merge, use `Squash and merge`.
The button for that is near the bottom of the PR's page on GitHub.
@@ -102,55 +113,63 @@ The button for that is near the bottom of the PR's page on GitHub.
> **Note:** See [issues](https://github.com/XRPLF/clio/issues) to find the `ISSUE_ID` for the feature/bug you were working on.
# Style guide
This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent.
## Formatting
Code must conform to `clang-format` version 19, unless the result would be unreasonably difficult to read or maintain.
In most cases the pre-commit hook will take care of formatting and will fix any issues automatically.
To manually format your code, use `clang-format -i <your changed files>` for C++ files and `cmake-format -i <your changed files>` for CMake files.
## Documentation
All public namespaces, classes and functions must be covered by doc (`doxygen`) comments. Everything that is not within a nested `impl` namespace is considered public.
> **Note:** Keep in mind that this is enforced by Clio's CI and your build will fail if newly added public code lacks documentation.
## Avoid
* Proliferation of nearly identical code.
* Proliferation of new files and classes unless it improves readability or/and compilation time.
* Unmanaged memory allocation and raw pointers.
* Macros (unless they add significant value.)
* Lambda patterns (unless these add significant value.)
* CPU or architecture-specific code unless there is a good reason to include it, and where it is used guard it with macros and provide explanatory comments.
* Importing new libraries unless there is a very good reason to do so.
- Proliferation of nearly identical code.
- Proliferation of new files and classes unless it improves readability or/and compilation time.
- Unmanaged memory allocation and raw pointers.
- Macros (unless they add significant value.)
- Lambda patterns (unless these add significant value.)
- CPU or architecture-specific code unless there is a good reason to include it, and where it is used guard it with macros and provide explanatory comments.
- Importing new libraries unless there is a very good reason to do so.
## Seek to
* Extend functionality of existing code rather than creating new code.
* Prefer readability over terseness where important logic is concerned.
* Inline functions that are not used or are not likely to be used elsewhere in the codebase.
* Use clear and self-explanatory names for functions, variables, structs and classes.
* Use TitleCase for classes, structs and filenames, camelCase for function and variable names, lower case for namespaces and folders.
* Provide as many comments as you feel that a competent programmer would need to understand what your code does.
- Extend functionality of existing code rather than creating new code.
- Prefer readability over terseness where important logic is concerned.
- Inline functions that are not used or are not likely to be used elsewhere in the codebase.
- Use clear and self-explanatory names for functions, variables, structs and classes.
- Use TitleCase for classes, structs and filenames, camelCase for function and variable names, lower case for namespaces and folders.
- Provide as many comments as you feel that a competent programmer would need to understand what your code does.
# Maintainers
Maintainers are ecosystem participants with elevated access to the repository. They are able to push new code, make decisions on when a release should be made, etc.
## Code Review
A PR must be reviewed and approved by at least one of the maintainers before it can be merged.
## Adding and Removing
New maintainers can be proposed by two existing maintainers, subject to a vote by a quorum of the existing maintainers. A minimum of 50% support and a 50% participation is required. In the event of a tie vote, the addition of the new maintainer will be rejected.
Existing maintainers can resign, or be subject to a vote for removal at the behest of two existing maintainers. A minimum of 60% agreement and 50% participation are required. The XRP Ledger Foundation will have the ability, for cause, to remove an existing maintainer without a vote.
## Existing Maintainers
* [godexsoft](https://github.com/godexsoft) (Ripple)
* [kuznetsss](https://github.com/kuznetsss) (Ripple)
* [legleux](https://github.com/legleux) (Ripple)
* [PeterChen13579](https://github.com/PeterChen13579) (Ripple)
- [godexsoft](https://github.com/godexsoft) (Ripple)
- [kuznetsss](https://github.com/kuznetsss) (Ripple)
- [legleux](https://github.com/legleux) (Ripple)
- [PeterChen13579](https://github.com/PeterChen13579) (Ripple)
## Honorable ex-Maintainers
* [cindyyan317](https://github.com/cindyyan317) (ex-Ripple)
* [cjcobb23](https://github.com/cjcobb23) (ex-Ripple)
* [natenichols](https://github.com/natenichols) (ex-Ripple)
- [cindyyan317](https://github.com/cindyyan317) (ex-Ripple)
- [cjcobb23](https://github.com/cjcobb23) (ex-Ripple)
- [natenichols](https://github.com/natenichols) (ex-Ripple)

2
README.md
View File

@@ -16,7 +16,7 @@ Multiple Clio nodes can share access to the same dataset, which allows for a hig
Clio offers the full `rippled` API, with the caveat that Clio by default only returns validated data. This means that `ledger_index` defaults to `validated` instead of `current` for all requests. Other non-validated data, such as information about queued transactions, is also not returned.
Clio retrieves data from a designated group of `rippled` nodes instead of connecting to the peer-to-peer network.
For requests that require access to the peer-to-peer network, such as `fee` or `submit`, Clio automatically forwards the request to a `rippled` node and propagates the response back to the client. To access non-validated data for *any* request, simply add `ledger_index: "current"` to the request, and Clio will forward the request to `rippled`.
For requests that require access to the peer-to-peer network, such as `fee` or `submit`, Clio automatically forwards the request to a `rippled` node and propagates the response back to the client. To access non-validated data for _any_ request, simply add `ledger_index: "current"` to the request, and Clio will forward the request to `rippled`.
> [!NOTE]
> Clio requires access to at least one `rippled` node, which can run on the same machine as Clio or separately.

1
docker/ci/README.md
View File

@@ -4,6 +4,7 @@ This image contains an environment to build [Clio](https://github.com/XRPLF/clio
It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but can also be used to compile Clio locally.
The image is based on Ubuntu 20.04 and contains:
- clang 16.0.6
- gcc 12.3
- doxygen 1.12

2
docker/clio/README.md
View File

@@ -12,12 +12,14 @@ Your configuration file should be mounted under the path `/opt/clio/etc/config.j
Clio repository provides an [example](https://github.com/XRPLF/clio/blob/develop/docs/examples/config/example-config.json) of the configuration file.
Config file recommendations:
- Set `log_to_console` to `false` if you want to avoid logs being written to `stdout`.
- Set `log_directory` to `/opt/clio/log` to store logs in a volume.
## Usage
The following command can be used to run Clio in docker (assuming server's port is `51233` in your config):
```bash
docker run -d -v <path to your config.json>:/opt/clio/etc/config.json -v <path to store logs>:/opt/clio/log -p 51233:51233 rippleci/clio
```

60
docs/build-clio.md
View File

@@ -11,7 +11,7 @@ Clio is built with [CMake](https://cmake.org/) and uses [Conan](https://conan.io
- [**Optional**] [CCache](https://ccache.dev/): speeds up compilation if you are going to compile Clio often
| Compiler | Version |
|-------------|---------|
| ----------- | ------- |
| GCC | 12.3 |
| Clang | 16 |
| Apple Clang | 15 |
@@ -104,10 +104,10 @@ To generate the API docs:
1. First, include `-o docs=True` in the conan install command. For example:
```sh
mkdir build && cd build
conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False -o docs=True
```
```sh
mkdir build && cd build
conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False -o docs=True
```
2. Once that has completed successfully, run the `cmake` command and add the `--target docs` option:
@@ -118,9 +118,9 @@ To generate the API docs:
3. Go to `build/docs/html` to view the generated files.
Open the `index.html` file in your browser to see the documentation pages.
Open the `index.html` file in your browser to see the documentation pages.
![API index page](./img/doxygen-docs-output.png "API index page")
![API index page](./img/doxygen-docs-output.png "API index page")
## Building Clio with Docker
@@ -148,39 +148,39 @@ Sometimes, during development, you need to build against a custom version of `li
1. First, pull/clone the appropriate `rippled` fork and switch to the branch you want to build. For example, the following example uses an in-development build with [XLS-33d Multi-Purpose Tokens](https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens):
```sh
git clone https://github.com/shawnxie999/rippled/
cd rippled
git switch mpt-1.1
```
```sh
git clone https://github.com/shawnxie999/rippled/
cd rippled
git switch mpt-1.1
```
2. Export a custom package to your local Conan store using a user/channel:
```sh
conan export . my/feature
```
```sh
conan export . my/feature
```
3. Patch your local Clio build to use the right package.
Edit `conanfile.py` (from the Clio repository root). Replace the `xrpl` requirement with the custom package version from the previous step. This must also include the current version number from your `rippled` branch. For example:
Edit `conanfile.py` (from the Clio repository root). Replace the `xrpl` requirement with the custom package version from the previous step. This must also include the current version number from your `rippled` branch. For example:
```py
# ... (excerpt from conanfile.py)
requires = [
'boost/1.82.0',
'cassandra-cpp-driver/2.17.0',
'fmt/10.1.1',
'protobuf/3.21.9',
'grpc/1.50.1',
'openssl/1.1.1u',
'xrpl/2.3.0-b1@my/feature', # Update this line
'libbacktrace/cci.20210118'
]
```
```py
# ... (excerpt from conanfile.py)
requires = [
'boost/1.82.0',
'cassandra-cpp-driver/2.17.0',
'fmt/10.1.1',
'protobuf/3.21.9',
'grpc/1.50.1',
'openssl/1.1.1u',
'xrpl/2.3.0-b1@my/feature', # Update this line
'libbacktrace/cci.20210118'
]
```
4. Build Clio as you would have before.
See [Building Clio](#building-clio) for details.
See [Building Clio](#building-clio) for details.
## Using `clang-tidy` for static analysis

2
docs/config-description.md
View File

@@ -381,7 +381,7 @@ This document provides a list of all available Clio configuration properties in
- **Type**: int
- **Default value**: `48`
- **Constraints**: The minimum value is `1`. The maximum value is `65535`.
- **Description**: Specifies how many markers are placed randomly within the cache. These markers define the positions on the ledger that will be loaded concurrently by the workers. The higher the number, the more places within the cache we potentially cover.
- **Description**: Specifies how many markers are placed randomly within the cache. These markers define the positions on the ledger that will be loaded concurrently by the workers. The higher the number, the more places within the cache we potentially cover.
### cache.num_cursors_from_diff

280
docs/examples/config/example-config.json
View File

@@ -2,146 +2,144 @@
* This is an example configuration file. Please do not use without modifying to suit your needs.
*/
{
"database": {
"type": "cassandra",
"cassandra": {
"contact_points": "127.0.0.1",
"port": 9042,
"keyspace": "clio",
"replication_factor": 1,
"table_prefix": "",
"max_write_requests_outstanding": 25000,
"max_read_requests_outstanding": 30000,
"threads": 8,
//
// Advanced options. USE AT OWN RISK:
// ---
"core_connections_per_host": 1, // Defaults to 1
"write_batch_size": 20 // Defaults to 20
//
// Below options will use defaults from cassandra driver if left unspecified.
// See https://docs.datastax.com/en/developer/cpp-driver/2.17/api/struct.CassCluster/ for details.
//
// "queue_size_io": 2
//
// ---
}
},
"allow_no_etl": false, // Allow Clio to run without valid ETL source, otherwise Clio will stop if ETL check fails
"etl_sources": [
{
"ip": "127.0.0.1",
"ws_port": "6005",
"grpc_port": "50051"
}
],
"forwarding": {
"cache_timeout": 0.250, // in seconds, could be 0, which means no cache
"request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds)
},
"rpc": {
"cache_timeout": 0.5 // in seconds, could be 0, which means no cache for rpc
},
"dos_guard": {
// Comma-separated list of IPs to exclude from rate limiting
"whitelist": [
"127.0.0.1"
],
//
// The below values are the default values and are only specified here
// for documentation purposes. The rate limiter currently limits
// connections and bandwidth per IP. The rate limiter looks at the raw
// IP of a client connection, and so requests routed through a load
// balancer will all have the same IP and be treated as a single client.
//
"max_fetches": 1000000, // Max bytes per IP per sweep interval
"max_connections": 20, // Max connections per IP
"max_requests": 20, // Max connections per IP per sweep interval
"sweep_interval": 1 // Time in seconds before resetting max_fetches and max_requests
},
"server": {
"ip": "0.0.0.0",
"port": 51233,
// Max number of requests to queue up before rejecting further requests.
// Defaults to 0, which disables the limit.
"max_queue_size": 500,
// If request contains header with authorization, Clio will check if it matches the prefix 'Password ' + this value's sha256 hash
// If matches, the request will be considered as admin request
"admin_password": "xrp",
// If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests
// It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time
"local_admin": false,
"processing_policy": "parallel", // Could be "sequent" or "parallel".
// For sequent policy request from one client connection will be processed one by one and the next one will not be read before
// the previous one is processed. For parallel policy Clio will take all requests and process them in parallel and
// send a reply for each request whenever it is ready.
"parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified.
// Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection.
"ws_max_sending_queue_size": 1500,
"__ng_web_server": false // Use ng web server. This is a temporary setting which will be deleted after switching to ng web server
},
// Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet.
"graceful_period": 10.0,
// Overrides log level on a per logging channel.
// Defaults to global "log_level" for each unspecified channel.
"log_channels": [
{
"channel": "Backend",
"log_level": "fatal"
},
{
"channel": "WebServer",
"log_level": "info"
},
{
"channel": "Subscriptions",
"log_level": "info"
},
{
"channel": "RPC",
"log_level": "error"
},
{
"channel": "ETL",
"log_level": "debug"
},
{
"channel": "Performance",
"log_level": "trace"
}
],
"cache": {
// Configure this to use either "num_diffs", "num_cursors_from_diff", or "num_cursors_from_account". By default, Clio uses "num_diffs".
"num_diffs": 32, // Generate the cursors from the latest ledger diff, then use the cursors to partition the ledger to load concurrently. The cursors number is affected by the busyness of the network.
// "num_cursors_from_diff": 3200, // Read the cursors from the diff table until we have enough cursors to partition the ledger to load concurrently.
// "num_cursors_from_account": 3200, // Read the cursors from the account table until we have enough cursors to partition the ledger to load concurrently.
"num_markers": 48, // The number of markers is the number of coroutines to load the cache concurrently.
"page_fetch_size": 512, // The number of rows to load for each page.
"load": "async" // "sync" to load cache synchronously or "async" to load cache asynchronously or "none"/"no" to turn off the cache.
},
"prometheus": {
"enabled": true,
"compress_reply": true
},
"log_level": "info",
// Log format (this is the default format)
"log_format": "%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%",
"log_to_console": true,
// Clio logs to file in the specified directory only if "log_directory" is set
// "log_directory": "./clio_log",
"log_rotation_size": 2048,
"log_directory_max_size": 51200,
"log_rotation_hour_interval": 12,
"log_tag_style": "uint",
"extractor_threads": 8,
"read_only": false,
// "start_sequence": [integer] the ledger index to start from,
// "finish_sequence": [integer] the ledger index to finish at,
// "ssl_cert_file" : "/full/path/to/cert.file",
// "ssl_key_file" : "/full/path/to/key.file"
"api_version": {
"min": 1, // Minimum API version supported (could be 1 or 2)
"max": 2, // Maximum API version supported (could be 1 or 2, but >= min)
"default": 1 // Clio behaves the same as rippled by default
"database": {
"type": "cassandra",
"cassandra": {
"contact_points": "127.0.0.1",
"port": 9042,
"keyspace": "clio",
"replication_factor": 1,
"table_prefix": "",
"max_write_requests_outstanding": 25000,
"max_read_requests_outstanding": 30000,
"threads": 8,
//
// Advanced options. USE AT OWN RISK:
// ---
"core_connections_per_host": 1, // Defaults to 1
"write_batch_size": 20 // Defaults to 20
//
// Below options will use defaults from cassandra driver if left unspecified.
// See https://docs.datastax.com/en/developer/cpp-driver/2.17/api/struct.CassCluster/ for details.
//
// "queue_size_io": 2
//
// ---
}
},
"allow_no_etl": false, // Allow Clio to run without valid ETL source, otherwise Clio will stop if ETL check fails
"etl_sources": [
{
"ip": "127.0.0.1",
"ws_port": "6005",
"grpc_port": "50051"
}
],
"forwarding": {
"cache_timeout": 0.25, // in seconds, could be 0, which means no cache
"request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds)
},
"rpc": {
"cache_timeout": 0.5 // in seconds, could be 0, which means no cache for rpc
},
"dos_guard": {
// Comma-separated list of IPs to exclude from rate limiting
"whitelist": ["127.0.0.1"],
//
// The below values are the default values and are only specified here
// for documentation purposes. The rate limiter currently limits
// connections and bandwidth per IP. The rate limiter looks at the raw
// IP of a client connection, and so requests routed through a load
// balancer will all have the same IP and be treated as a single client.
//
"max_fetches": 1000000, // Max bytes per IP per sweep interval
"max_connections": 20, // Max connections per IP
"max_requests": 20, // Max connections per IP per sweep interval
"sweep_interval": 1 // Time in seconds before resetting max_fetches and max_requests
},
"server": {
"ip": "0.0.0.0",
"port": 51233,
// Max number of requests to queue up before rejecting further requests.
// Defaults to 0, which disables the limit.
"max_queue_size": 500,
// If request contains header with authorization, Clio will check if it matches the prefix 'Password ' + this value's sha256 hash
// If matches, the request will be considered as admin request
"admin_password": "xrp",
// If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests
// It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time
"local_admin": false,
"processing_policy": "parallel", // Could be "sequent" or "parallel".
// For sequent policy request from one client connection will be processed one by one and the next one will not be read before
// the previous one is processed. For parallel policy Clio will take all requests and process them in parallel and
// send a reply for each request whenever it is ready.
"parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified.
// Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection.
"ws_max_sending_queue_size": 1500,
"__ng_web_server": false // Use ng web server. This is a temporary setting which will be deleted after switching to ng web server
},
// Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet.
"graceful_period": 10.0,
// Overrides log level on a per logging channel.
// Defaults to global "log_level" for each unspecified channel.
"log_channels": [
{
"channel": "Backend",
"log_level": "fatal"
},
{
"channel": "WebServer",
"log_level": "info"
},
{
"channel": "Subscriptions",
"log_level": "info"
},
{
"channel": "RPC",
"log_level": "error"
},
{
"channel": "ETL",
"log_level": "debug"
},
{
"channel": "Performance",
"log_level": "trace"
}
],
"cache": {
// Configure this to use either "num_diffs", "num_cursors_from_diff", or "num_cursors_from_account". By default, Clio uses "num_diffs".
"num_diffs": 32, // Generate the cursors from the latest ledger diff, then use the cursors to partition the ledger to load concurrently. The cursors number is affected by the busyness of the network.
// "num_cursors_from_diff": 3200, // Read the cursors from the diff table until we have enough cursors to partition the ledger to load concurrently.
// "num_cursors_from_account": 3200, // Read the cursors from the account table until we have enough cursors to partition the ledger to load concurrently.
"num_markers": 48, // The number of markers is the number of coroutines to load the cache concurrently.
"page_fetch_size": 512, // The number of rows to load for each page.
"load": "async" // "sync" to load cache synchronously or "async" to load cache asynchronously or "none"/"no" to turn off the cache.
},
"prometheus": {
"enabled": true,
"compress_reply": true
},
"log_level": "info",
// Log format (this is the default format)
"log_format": "%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%",
"log_to_console": true,
// Clio logs to file in the specified directory only if "log_directory" is set
// "log_directory": "./clio_log",
"log_rotation_size": 2048,
"log_directory_max_size": 51200,
"log_rotation_hour_interval": 12,
"log_tag_style": "uint",
"extractor_threads": 8,
"read_only": false,
// "start_sequence": [integer] the ledger index to start from,
// "finish_sequence": [integer] the ledger index to finish at,
// "ssl_cert_file" : "/full/path/to/cert.file",
// "ssl_key_file" : "/full/path/to/key.file"
"api_version": {
"min": 1, // Minimum API version supported (could be 1 or 2)
"max": 2, // Maximum API version supported (could be 1 or 2, but >= min)
"default": 1 // Clio behaves the same as rippled by default
}
}

5
docs/examples/infrastructure/README.md
View File

@@ -7,8 +7,9 @@
This directory contains an example of docker based infrastructure to collect and visualise metrics from clio.
The structure of the directory:
- `compose.yaml`
Docker-compose file with Prometheus and Grafana set up.
Docker-compose file with Prometheus and Grafana set up.
- `prometheus.yaml`
Defines metrics collection from Clio and Prometheus itself.
Demonstrates how to setup Clio target and Clio's admin authorisation in Prometheus.
@@ -24,6 +25,6 @@ The structure of the directory:
1. Make sure you have `docker` and `docker-compose` installed.
2. Run `docker-compose up -d` from this directory. It will start docker containers with Prometheus and Grafana.
3. Open [http://localhost:3000/dashboards](http://localhost:3000/dashboards). Grafana login `admin`, password `grafana`.
There will be preconfigured Clio dashboard.
There will be preconfigured Clio dashboard.
If Clio is not running yet launch Clio to see metrics. Some of the metrics may appear only after requests to Clio.

2
docs/examples/infrastructure/compose.yaml
View File

@@ -6,7 +6,7 @@ services:
volumes:
- ./prometheus.yaml:/etc/prometheus/prometheus.yml
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- "--config.file=/etc/prometheus/prometheus.yml"
grafana:
image: grafana/grafana
ports:

16
docs/examples/infrastructure/grafana/clio_dashboard.json
View File

@@ -80,9 +80,7 @@
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
@@ -161,9 +159,7 @@
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
@@ -246,9 +242,7 @@
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
@@ -331,9 +325,7 @@
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},

6
docs/examples/infrastructure/grafana/dashboard_local.yaml
View File

@@ -1,13 +1,13 @@
apiVersion: 1
providers:
- name: 'Clio dashboard'
- name: "Clio dashboard"
# <int> Org id. Default to 1
orgId: 1
# <string> name of the dashboard folder.
folder: ''
folder: ""
# <string> folder UID. will be automatically generated if not specified
folderUid: ''
folderUid: ""
# <string> provider type. Default to 'file'
type: file
# <bool> disable dashboard deletion

10
docs/examples/infrastructure/grafana/datasources.yaml
View File

@@ -1,8 +1,8 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
url: http://prometheus:9090
isDefault: true
access: proxy
- name: Prometheus
type: prometheus
url: http://prometheus:9090
isDefault: true
access: proxy

36
docs/examples/infrastructure/prometheus.yaml
View File

@@ -1,19 +1,19 @@
scrape_configs:
- job_name: clio
scrape_interval: 5s
scrape_timeout: 5s
authorization:
type: Password
# sha256sum from password `xrp`
# use echo -n 'your_password' | shasum -a 256 to get hash
credentials: 0e1dcf1ff020cceabf8f4a60a32e814b5b46ee0bb8cd4af5c814e4071bd86a18
static_configs:
- targets:
- host.docker.internal:51233
- job_name: prometheus
honor_timestamps: true
scrape_interval: 15s
scrape_timeout: 10s
static_configs:
- targets:
- localhost:9090
- job_name: clio
scrape_interval: 5s
scrape_timeout: 5s
authorization:
type: Password
# sha256sum from password `xrp`
# use echo -n 'your_password' | shasum -a 256 to get hash
credentials: 0e1dcf1ff020cceabf8f4a60a32e814b5b46ee0bb8cd4af5c814e4071bd86a18
static_configs:
- targets:
- host.docker.internal:51233
- job_name: prometheus
honor_timestamps: true
scrape_interval: 15s
scrape_timeout: 10s
static_configs:
- targets:
- localhost:9090

6
docs/logging.md
View File

@@ -8,7 +8,7 @@ The minimum level of severity at which the log message will be outputted by defa
## `log_format`
The format of log lines produced by Clio. Defaults to `"%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%"`.
The format of log lines produced by Clio. Defaults to `"%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%"`.
Each of the variables expands like so:
@@ -30,8 +30,8 @@ Each object is of this format:
```json
{
"channel": "Backend",
"log_level": "fatal"
"channel": "Backend",
"log_level": "fatal"
}
```

9
docs/run-clio.md
View File

@@ -3,10 +3,11 @@
## Prerequisites
- Access to a Cassandra cluster or ScyllaDB cluster. Can be local or remote.
> [!IMPORTANT]
> There are some key considerations when using **ScyllaDB**. By default, Scylla reserves all free RAM on a machine for itself. If you are running `rippled` or other services on the same machine, restrict its memory usage using the `--memory` argument.
>
> See [ScyllaDB in a Shared Environment](https://docs.scylladb.com/getting-started/scylla-in-a-shared-environment/) to learn more.
> [!IMPORTANT]
> There are some key considerations when using **ScyllaDB**. By default, Scylla reserves all free RAM on a machine for itself. If you are running `rippled` or other services on the same machine, restrict its memory usage using the `--memory` argument.
>
> See [ScyllaDB in a Shared Environment](https://docs.scylladb.com/getting-started/scylla-in-a-shared-environment/) to learn more.
- Access to one or more `rippled` nodes. Can be local or remote.

18
docs/trouble_shooting.md
View File

@@ -1,44 +1,60 @@
# Troubleshooting Guide
This guide will help you troubleshoot common issues of Clio.
## Can't connect to DB
If you see the error log message `Could not connect to Cassandra: No hosts available`, this means that Clio can't connect to the database. Check the following:
- Make sure the database is running at the specified address and port.
- Make sure the database is accessible from the machine where Clio is running.
You can use [cqlsh](https://pypi.org/project/cqlsh/) to check the connection to the database.
You can use [cqlsh](https://pypi.org/project/cqlsh/) to check the connection to the database.
If you would like to run a local ScyllaDB, you can call:
```sh
docker run --rm -p 9042:9042 --name clio-scylla -d scylladb/scylla
```
## Check the server status of Clio
To check if Clio is syncing with rippled:
```sh
curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep seq
```
If Clio is syncing with rippled, the `seq` value will be increasing.
## Clio fails to start
If you see the error log message `Failed to fetch ETL state from...`, this means the configured rippled node is not reachable. Check the following:
- Make sure the rippled node is running at the specified address and port.
- Make sure the rippled node is accessible from the machine where Clio is running.
If you would like to run Clio without an avaliable rippled node, you can add below setting to Clio's configuration file:
```
"allow_no_etl": true
```
## Clio is not added to secure_gateway in rippled's config
If you see the warning message `AsyncCallData is_unlimited is false.`, this means that Clio is not added to the `secure_gateway` of `port_grpc` session in the rippled configuration file. It will slow down the sync process. Please add Clio's IP to the `secure_gateway` in the rippled configuration file for both grpc and ws port.
## Clio is slow
To speed up the response time, Clio has a cache inside. However, cache can take time to warm up. If you see slow response time, you can firstly check if cache is still loading.
You can check the cache status by calling:
```sh
curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_full
curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_enabled
```
If `is_full` is false, it means the cache is still loading. Normally, the Clio can respond quicker if cache finishs loading. If `is_enabled` is false, it means the cache is disabled in the configuration file or there is data corruption in the database.
## Receive error message `Too many requests`
If client sees the error message `Too many requests`, this means that the client is blocked by Clio's DosGuard protection. You may want to add the client's IP to the whitelist in the configuration file, Or update other your DosGuard settings.

14
src/data/README.md
View File

@@ -6,7 +6,7 @@ To support additional database types, you can create new classes that implement
## Data Model
The data model used by Clio to read and write ledger data is different from what `rippled` uses. `rippled` uses a novel data structure named [*SHAMap*](https://github.com/ripple/rippled/blob/master/src/ripple/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record.
The data model used by Clio to read and write ledger data is different from what `rippled` uses. `rippled` uses a novel data structure named [_SHAMap_](https://github.com/ripple/rippled/blob/master/src/ripple/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record.
`rippled` nodes can also generate a proof-tree by forming a subtree with all the path nodes and their neighbors, which can then be used to prove the existence of the leaf node data to other `rippled` nodes. In short, the main purpose of the SHAMap data structure is to facilitate the fast validation of data integrity between different decentralized `rippled` nodes.
@@ -170,7 +170,7 @@ CREATE TABLE clio.successor (
This table is the important backbone of how histories of ledger objects are stored in Cassandra. The `successor` table stores the object index of all ledger objects that were validated on the XRP network along with the ledger sequence that the object was updated on.
As each key is ordered by the sequence, which is achieved by tracing through the table with a specific sequence number, Clio can recreate a Linked List data structure that represents all the existing ledger objects at that ledger sequence. The special values of `0x00...00` and `0xFF...FF` are used to label the *head* and *tail* of the Linked List in the successor table.
As each key is ordered by the sequence, which is achieved by tracing through the table with a specific sequence number, Clio can recreate a Linked List data structure that represents all the existing ledger objects at that ledger sequence. The special values of `0x00...00` and `0xFF...FF` are used to label the _head_ and _tail_ of the Linked List in the successor table.
The diagram below showcases how tracing through the same table, but with different sequence parameter filtering, can result in different Linked List data representing the corresponding past state of the ledger objects. A query like `SELECT * FROM successor WHERE key = ? AND seq <= n ORDER BY seq DESC LIMIT 1;` can effectively trace through the successor table and get the Linked List of a specific sequence `n`.
@@ -182,12 +182,12 @@ In each new ledger version with sequence `n`, a ledger object `v` can either be
For all three of these operations, the procedure to update the successor table can be broken down into two steps:
1. Trace through the Linked List of the previous sequence to find the ledger object `e` with the greatest object index smaller or equal than the `v`'s index. Save `e`'s `next` value (the index of the next ledger object) as `w`.
1. Trace through the Linked List of the previous sequence to find the ledger object `e` with the greatest object index smaller or equal than the `v`'s index. Save `e`'s `next` value (the index of the next ledger object) as `w`.
2. If `v` is...
1. Being **created**, add two new records of `seq=n` with one being `e` pointing to `v`, and `v` pointing to `w` (Linked List insertion operation).
2. Being **modified**, do nothing.
3. Being **deleted**, add a record of `seq=n` with `e` pointing to `v`'s `next` value (Linked List deletion operation).
2. If `v` is...
1. Being **created**, add two new records of `seq=n` with one being `e` pointing to `v`, and `v` pointing to `w` (Linked List insertion operation).
2. Being **modified**, do nothing.
3. Being **deleted**, add a record of `seq=n` with `e` pointing to `v`'s `next` value (Linked List deletion operation).
## NFT data model

20
src/etl/README.md
View File

@@ -23,17 +23,17 @@ For example, if segment **0x08581464C55B0B2C8C4FA27FA8DE0ED695D3BE019E7BE0969C92
Because of the nature of the Linked List, the cursors are crucial to balancing the workload of each coroutine. There are 3 types of cursor generation that can be used:
- **cache.num_diffs**: Cursors will be generated by the changed objects in the latest `cache.num_diffs` number of ledgers. The default value is 32. In *mainnet*, this type works well because the network is fairly busy and the number of changed objects in each ledger is relatively stable. Thus, we are able to get enough cursors after removing the deleted objects on *mainnet*.
For other networks, like the *devnet*, the number of changed objects in each ledger is not stable. When the network is silent, one coroutine may load a large number of objects while the other coroutines are idle. Below is a comparison of the number of cursors and loading time on *devnet*:
- **cache.num_diffs**: Cursors will be generated by the changed objects in the latest `cache.num_diffs` number of ledgers. The default value is 32. In _mainnet_, this type works well because the network is fairly busy and the number of changed objects in each ledger is relatively stable. Thus, we are able to get enough cursors after removing the deleted objects on _mainnet_.
For other networks, like the _devnet_, the number of changed objects in each ledger is not stable. When the network is silent, one coroutine may load a large number of objects while the other coroutines are idle. Below is a comparison of the number of cursors and loading time on _devnet_:
| Cursors | Loading time /seconds |
| ------- | --------------------- |
| 11 | 2072 |
| 33 | 983 |
| 120 | 953 |
| 200 | 843 |
| 250 | 816 |
| 500 | 792 |
| Cursors | Loading time /seconds |
| ------- | --------------------- |
| 11 | 2072 |
| 33 | 983 |
| 120 | 953 |
| 200 | 843 |
| 250 | 816 |
| 500 | 792 |
- **cache.num_cursors_from_diff**: Cursors will be generated by the changed objects in the recent ledgers. The generator will keep reading the previous ledger until we have `cache.num_cursors_from_diff` cursors. This type is the evolved version of `cache.num_diffs`. It removes the network busyness factor and only considers the number of cursors. The cache loading can be well tuned by this configuration.

33
src/migration/README.md
View File

@@ -1,35 +1,26 @@
# Clio Migration
Clio maintains the off-chain data of XRPL and multiple indexes tables to powering complex queries. To simplify the creation of index tables, this migration framework handles the process of database change and facilitates the migration of historical data seamlessly.
## Command Line Usage
Clio provides a migration command-line tool to migrate data in database.
> Note: We need a **configuration file** to run the migration tool. This configuration file has the same format as the configuration file of the Clio server, ensuring consistency and ease of use. It reads the database configuration from the same session as the server's configuration, eliminating the need for separate setup or additional configuration files. Be aware that migration-specific configuration is under `.migration` session.
### To query migration status:
./clio_server --migrate status ~/config/migrator.json
This command returns the current migration status of each migrator. The example output:
Current Migration Status:
Migrator: ExampleMigrator - Feature v1, Clio v3 - not migrated
### To start a migration:
./clio_server --migrate ExampleMigrator ~/config/migrator.json
Migration will run if the migrator has not been migrated. The migrator will be marked as migrated after the migration is completed.
## How to write a migrator
@@ -52,10 +43,10 @@ It contains:
> **Note** Each migrator is designed to work with a specific database.
- Register your migrator in MigrationManager. Currently we only support Cassandra/ScyllaDB. Migrator needs to be registered in `CassandraSupportedMigrators`.
- Register your migrator in MigrationManager. Currently we only support Cassandra/ScyllaDB. Migrator needs to be registered in `CassandraSupportedMigrators`.
## How to use full table scanner (Only for Cassandra/ScyllaDB)
Sometimes migrator isn't able to query the historical data by table's partition key. For example, migrator of transactions needs the historical transaction data without knowing each transaction hash. Full table scanner can help to get all the rows in parallel.
Most indexes are based on either ledger states or transactions. We provide the `objects` and `transactions` scanner. Developers only need to implement the callback function to receive the historical data. Please find the examples in `tests/integration/migration/cassandra/ExampleTransactionsMigrator.cpp` and `tests/integration/migration/cassandra/ExampleObjectsMigrator.cpp`.
@@ -65,15 +56,16 @@ Most indexes are based on either ledger states or transactions. We provide the `
## How to write a full table scan adapter (Only for Cassandra/ScyllaDB)
If you need to do full scan against other table, you can follow below steps:
- Describe the table which needs full scan in a struct. It has to satisfy the `TableSpec`(cassandra/Spec.hpp) concept, containing static member:
- Tuple type `Row`, it's the type of each field in a row. The order of types should match what database will return in a row. Key types should come first, followed by other field types sorted in alphabetical order.
- `kPARTITION_KEY`, it's the name of the partition key of the table.
- `kTABLE_NAME`
- Tuple type `Row`, it's the type of each field in a row. The order of types should match what database will return in a row. Key types should come first, followed by other field types sorted in alphabetical order.
- `kPARTITION_KEY`, it's the name of the partition key of the table.
- `kTABLE_NAME`
- Inherent from `FullTableScannerAdapterBase`.
- Implement `onRowRead`, its parameter is the `Row` we defined. It's the callback function when a row is read.
Please take ObjectsAdapter/TransactionsAdapter as example.
## Examples:
@@ -82,13 +74,16 @@ We have some example migrators under `tests/integration/migration/cassandra` fol
- ExampleDropTableMigrator
This migrator drops `diff` table.
This migrator drops `diff` table.
- ExampleLedgerMigrator
This migrator shows how to migrate data when we don't need to do full table scan. This migrator creates an index table `ledger_example` which maintains the map of ledger sequence and its account hash.
This migrator shows how to migrate data when we don't need to do full table scan. This migrator creates an index table `ledger_example` which maintains the map of ledger sequence and its account hash.
- ExampleObjectsMigrator
This migrator shows how to migrate ledger states related data. It uses `ObjectsScanner` to proceed the full scan in parallel. It counts the number of ACCOUNT_ROOT.
This migrator shows how to migrate ledger states related data. It uses `ObjectsScanner` to proceed the full scan in parallel. It counts the number of ACCOUNT_ROOT.
- ExampleTransactionsMigrator
This migrator shows how to migrate transactions related data. It uses `TransactionsScanner` to proceed the `transactions` table full scan in parallel. It creates an index table `tx_index_example` which tracks the transaction hash and its according transaction type.
This migrator shows how to migrate transactions related data. It uses `TransactionsScanner` to proceed the `transactions` table full scan in parallel. It creates an index table `tx_index_example` which tracks the transaction hash and its according transaction type.

35
src/util/async/README.md
View File

@@ -7,6 +7,7 @@ Clio uses threads intensively. Multiple parts of Clio were/are implemented by ru
On the other hand, Clio also uses `Boost.Asio` for more complex tasks such as networking, scheduling RPC handlers, and even interacting with the database is done via Asio’s coroutines.
There was a need for a simple yet powerful framework that will cover the following in a unified way:
- Exception/error handling and propagation
- Ability to return a value of any type as a result of a successful operation
- Cancellation (cooperative) of inflight operations
@@ -28,6 +29,7 @@ At the core of async framework are the execution contexts. Each execution contex
There are multiple execution contexts to choose from, each with their own pros and cons.
#### CoroExecutionContext
This context wraps a thread pool and executes blocks of code by means of `boost::asio::spawn` which spawns coroutines.
Deep inside the framework it hides `boost::asio::yield_context` and automatically switches coroutine contexts everytime user’s code is checking `isStopRequested()` on the `StopToken` given to the user-provided lambda.
@@ -37,71 +39,89 @@ The benefit is that both timers and async operations can work concurrently on a
Users of this execution context should take care to split their work in reasonably sized batches to avoid incurring a performance penalty caused by switching coroutine contexts too often. However if the batches are too time consuming it may lead to slower cooperative cancellation.
#### PoolExecutionContext
This context wraps a thread pool but executes blocks of code without using coroutines.
Note: A downside of this execution context is that if there is only 1 thread in the thread pool, timers can not execute while the thread is busy executing user-provided code. It's up to the user of this execution context to decide how to deal with this and whether it's important for their use case.
#### SyncExecutionContext
This is a fully synchronous execution context. It runs the scheduled operations right on the caller thread. By the time `execute([]{ … })` returns the Operation it’s guaranteed to be ready (i.e. value or error can be immediately queried with `.get()`).
In order to support scheduled operations and timeout-based cancellation, this context schedules all timers on the SystemExecutionContext instead.
#### SystemExecutionContext
This context of 1 thread is always readily available system-wide and can be used for
- fire and forget operations where it makes no sense to create an entirely new context for them
- as an external context for scheduling timers (used by SyncExecutionContext automatically)
### Strand
Any execution context provides a convenient `makeStrand` member function which will return a strand object for the execution context.
The strand can then be used with the same set of APIs that the execution context provides with the difference being that everything that is executed through a strand is guaranteed to be serially executed within the strand. This is a way to avoid the need for using a mutex or other explic synchronization mechanisms.
### Outcome
An outcome is like a `std::promise` to the operations that execute on the execution context.
The framework will hold onto the outcome object internally and the user of the framework will only receive an operation object that is like the `std::future` to the outcome.
The framework will set the final value or error through the outcome object so that the user can receive it on the operation side as a `std::expected`.
### Operation
There are several different operation types available. The one used will depend on the signature of the executable lambda passed by the user of this framework.
#### Stoppable and non-stoppable operations
Stoppable operations can be cooperatively stopped via a stop token that is passed to the user-provided function/lambda. A stoppable operation is returned to the user if they specify a stop token as the first argument of the function/lambda for execution.
Regular, non-stoppable operations, can not be stopped. A non-stoppable operation is returned to the user if they did not request a stop token as the first argument of the function/lambda for execution.
#### Scheduled operations
Scheduled operations are wrappers on top of Stoppable and regular Operations and provide the functionality of a timer that needs to run out before the given block of code will finally be executed on the Execution Context.
Scheduled operations can be aborted by calling
- `cancel` - will only cancel the timer. If the timer already fired this will have no effect
- `requestStop` - will stop the operation if it's already running or as soon as the timer runs out
- `abort` - will call `cancel` immediatelly followed by `requestStop`
### Error handling
By default, exceptions that happen during the execution of user-provided code are caught and returned in the error channel of `std::expected` as an instance of the `ExecutionError` struct. The user can then extract the error message by calling `what()` or directly accessing the `message` member.
### Returned value
If the user-provided lambda returns anything but `void`, the type and value will propagate through the operation object and can be received by calling `get` which will block until a value or an error is available.
The `wait` member function can be used when the user just wants to wait for the value to become available but not necessarily getting at the value just yet.
### Type erasure
On top of the templated execution contexts, outcomes, operations, strands and stop tokens this framework provides the type-erased wrappers with (mostly) the same interface.
#### AnyExecutionContext
This provides the same interface as any other execution context in this framework.
Note: the original context is taken in by reference.
See examples of use below.
#### AnyOperation<T>
Wraps any type of operations including regular, stoppable and scheduled.
Since this wrapper does not know which operation type it's wrapping it only provides an `abort` member function that will call the correct underlying functions depending on the real type of the operation. If `abort` is called on a regular (non-stoppable and not scheduled) operation, the call will result in an assertion failure.
## Examples
This section provides some examples. For more examples take a look at `ExecutionContextBenchmarks`, `AsyncExecutionContextTests` and `AnyExecutionContextTests`.
### Regular operation
#### Awaiting and reading values
```cpp
auto res = ctx.execute([]() { return 42; });
EXPECT_EQ(res.get().value(), 42);
@@ -114,8 +134,11 @@ ASSERT_EQ(value, 42);
```
### Stoppable operation
#### Requesting stoppage
The stop token can be used via the `isStopRequested()` member function:
```cpp
auto res = ctx.execute([](auto stopToken) {
while (not stopToken.isStopRequested())
@@ -128,6 +151,7 @@ res.requestStop();
```
Alternatively, the stop token is implicity convertible to `bool` so you can also use it like so:
```cpp
auto res = ctx.execute([](auto stopRequested) {
while (not stopRequested)
@@ -140,7 +164,9 @@ res.requestStop();
```
#### Automatic stoppage on timeout
By adding an optional timeout as the last arg to `execute` you can have the framework automatically call `requestStop()`:
```cpp
auto res = ctx.execute([](auto stopRequested) {
while (not stopRequested)
@@ -153,7 +179,9 @@ auto res = ctx.execute([](auto stopRequested) {
```
### Scheduled operation
#### Cancelling an outstanding operation
```cpp
auto res = ctx.scheduleAfter(
10ms, []([[maybe_unused]] auto stopRequested, auto cancelled) {
@@ -166,6 +194,7 @@ res.cancel(); // or .abort()
```
#### Get value after stopping
```cpp
auto res = ctx.scheduleAfter(1ms, [](auto stopRequested) {
while (not stopRequested)
@@ -178,6 +207,7 @@ res.requestStop();
```
#### Handling an exception
```cpp
auto res =
ctx.scheduleAfter(1s, []([[maybe_unused]] auto stopRequested, auto cancelled) {
@@ -192,9 +222,11 @@ EXPECT_TRUE(std::string{err}.ends_with("test"));
```
### Strand
The APIs are basically the same as with the parent `ExecutionContext`.
#### Computing a value on a strand
```cpp
auto strand = ctx.makeStrand();
auto res = strand.execute([] { return 42; });
@@ -203,7 +235,9 @@ EXPECT_EQ(res.get().value(), 42);
```
### Type erasure
#### Simple use
```cpp
auto ctx = CoroExecutionContext{4};
auto anyCtx = AnyExecutionContext{ctx};
@@ -215,6 +249,7 @@ auto op = anyCtx.execute([](auto stopToken) {
```
#### Aborting the operation
Erased operations only expose the `abort` member function that can be used to both cancel an outstanding and/or stop a running operation.
```cpp

2
src/util/newconfig/ConfigDescription.hpp
View File

@@ -246,7 +246,7 @@ private:
"from the position of markers concurrently. For more information, please read "
"[README.md](../src/etl/README.md)."},
KV{.key = "cache.num_markers",
.value = " Specifies how many markers are placed randomly within the cache. These markers define the "
.value = "Specifies how many markers are placed randomly within the cache. These markers define the "
"positions on the ledger that will be loaded concurrently by the workers. The higher the number, "
"the more places within the cache we potentially cover."},
KV{.key = "cache.num_cursors_from_diff",

3
tests/integration/README.md
View File

@@ -3,8 +3,11 @@
Tests that hit the real database are separate from the unit test suite found under `tests/unit`.
## Requirements
### Cassandra/ScyllaDB cluster
If you wish to test the backend component you will need to have access to a **local (127.0.0.1)** Cassandra cluster, opened at port **9042**. Please ensure that the cluster is successfully running before running these tests.
## Running
To run the DB tests, first build Clio as normal, then execute `./clio_integration_tests` to run all database tests.

2
tests/unit/README.md
View File

@@ -3,7 +3,9 @@
The correctness of new implementations can be verified via running unit tests. Below are the information on how to run unit tests.
## Running
To run the unit tests, first build Clio as normal, then execute `./clio_tests` to run all unit tests.
# Adding Unit Tests
To add unit tests, please create a separate file for the component you are trying to cover (unless it already exists) and use any other existing unit test file as an example.