diff --git a/.clang-format b/.clang-format index f27d31867..62454a78f 100644 --- a/.clang-format +++ b/.clang-format @@ -1,5 +1,5 @@ --- -Language: Cpp +Language: Cpp AccessModifierOffset: -4 AlignAfterOpenBracket: BlockIndent AlignConsecutiveAssignments: false @@ -22,31 +22,31 @@ BreakBeforeBinaryOperators: false BreakBeforeBraces: WebKit BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: true -ColumnLimit: 120 -CommentPragmas: '^ IWYU pragma:' +ColumnLimit: 120 +CommentPragmas: "^ IWYU pragma:" ConstructorInitializerAllOnOneLineOrOnePerLine: true ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false -DisableFormat: false +DisableFormat: false ExperimentalAutoDetectBinPacking: false FixNamespaceComments: true -ForEachMacros: [ Q_FOREACH, BOOST_FOREACH ] +ForEachMacros: [Q_FOREACH, BOOST_FOREACH] IncludeBlocks: Regroup IncludeCategories: - - Regex: '^".*"$' - Priority: 1 - - Regex: '^<.*\.(h|hpp)>$' - Priority: 2 - - Regex: '^<.*>$' - Priority: 3 - - Regex: '.*' - Priority: 4 -IncludeIsMainRegex: '$' + - Regex: '^".*"$' + Priority: 1 + - Regex: '^<.*\.(h|hpp)>$' + Priority: 2 + - Regex: "^<.*>$" + Priority: 3 + - Regex: ".*" + Priority: 4 +IncludeIsMainRegex: "$" IndentCaseLabels: true IndentFunctionDeclarationAfterType: false -IndentWidth: 4 +IndentWidth: 4 IndentWrappedFunctionNames: false IndentRequiresClause: true RequiresClausePosition: OwnLine @@ -63,18 +63,18 @@ PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left QualifierAlignment: Right -ReflowComments: true -SortIncludes: true +ReflowComments: true +SortIncludes: true SpaceAfterCStyleCast: false SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 2 -SpacesInAngles: false +SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false -Standard: Cpp11 -TabWidth: 8 -UseTab: Never +Standard: Cpp11 +TabWidth: 8 +UseTab: Never diff --git a/.clang-tidy b/.clang-tidy index 3bd741158..b8ebb841f 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -1,5 +1,5 @@ --- -Checks: '-*, +Checks: "-*, bugprone-argument-comment, bugprone-assert-side-effect, bugprone-bad-signal-to-kill-thread, @@ -146,7 +146,7 @@ Checks: '-*, readability-static-definition-in-anonymous-namespace, readability-suspicious-call-argument, readability-use-std-min-max - ' + " CheckOptions: readability-braces-around-statements.ShortStatementLines: 2 @@ -158,21 +158,21 @@ CheckOptions: readability-identifier-naming.EnumConstantCase: CamelCase readability-identifier-naming.ScopedEnumConstantCase: CamelCase readability-identifier-naming.GlobalConstantCase: UPPER_CASE - readability-identifier-naming.GlobalConstantPrefix: 'k' + readability-identifier-naming.GlobalConstantPrefix: "k" readability-identifier-naming.GlobalVariableCase: CamelCase - readability-identifier-naming.GlobalVariablePrefix: 'g' + readability-identifier-naming.GlobalVariablePrefix: "g" readability-identifier-naming.ConstexprFunctionCase: camelBack readability-identifier-naming.ConstexprMethodCase: camelBack readability-identifier-naming.ClassMethodCase: camelBack readability-identifier-naming.ClassMemberCase: camelBack readability-identifier-naming.ClassConstantCase: UPPER_CASE - readability-identifier-naming.ClassConstantPrefix: 'k' + readability-identifier-naming.ClassConstantPrefix: "k" readability-identifier-naming.StaticConstantCase: UPPER_CASE - readability-identifier-naming.StaticConstantPrefix: 'k' + readability-identifier-naming.StaticConstantPrefix: "k" readability-identifier-naming.StaticVariableCase: UPPER_CASE - readability-identifier-naming.StaticVariablePrefix: 'k' + readability-identifier-naming.StaticVariablePrefix: "k" readability-identifier-naming.ConstexprVariableCase: UPPER_CASE - readability-identifier-naming.ConstexprVariablePrefix: 'k' + readability-identifier-naming.ConstexprVariablePrefix: "k" readability-identifier-naming.LocalConstantCase: camelBack readability-identifier-naming.LocalVariableCase: camelBack readability-identifier-naming.TemplateParameterCase: CamelCase @@ -181,11 +181,11 @@ CheckOptions: readability-identifier-naming.MemberCase: camelBack readability-identifier-naming.PrivateMemberSuffix: _ readability-identifier-naming.ProtectedMemberSuffix: _ - readability-identifier-naming.PublicMemberSuffix: '' - readability-identifier-naming.FunctionIgnoredRegexp: '.*tag_invoke.*' + readability-identifier-naming.PublicMemberSuffix: "" + readability-identifier-naming.FunctionIgnoredRegexp: ".*tag_invoke.*" bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc - misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*;.*ranges_lower_bound\.h;time.h;stdlib.h' + misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*;.*ranges_lower_bound\.h;time.h;stdlib.h;__chrono/.*;fmt/chrono.h;boost/uuid/uuid_hash.hpp' HeaderFilterRegex: '^.*/(src|tests)/.*\.(h|hpp)$' -WarningsAsErrors: '*' +WarningsAsErrors: "*" diff --git a/.cmake-format.yaml b/.cmake-format.yaml index 35cc29c98..e3d1e9082 100644 --- a/.cmake-format.yaml +++ b/.cmake-format.yaml @@ -1,222 +1,222 @@ _help_parse: Options affecting listfile parsing parse: _help_additional_commands: - - Specify structure for custom cmake functions + - Specify structure for custom cmake functions additional_commands: foo: flags: - - BAR - - BAZ + - BAR + - BAZ kwargs: - HEADERS: '*' - SOURCES: '*' - DEPENDS: '*' + HEADERS: "*" + SOURCES: "*" + DEPENDS: "*" _help_override_spec: - - Override configurations per-command where available + - Override configurations per-command where available override_spec: {} _help_vartags: - - Specify variable tags. + - Specify variable tags. vartags: [] _help_proptags: - - Specify property tags. + - Specify property tags. proptags: [] _help_format: Options affecting formatting. format: _help_disable: - - Disable formatting entirely, making cmake-format a no-op + - Disable formatting entirely, making cmake-format a no-op disable: false _help_line_width: - - How wide to allow formatted cmake files + - How wide to allow formatted cmake files line_width: 120 _help_tab_size: - - How many spaces to tab for indent + - How many spaces to tab for indent tab_size: 2 _help_use_tabchars: - - If true, lines are indented using tab characters (utf-8 - - 0x09) instead of space characters (utf-8 0x20). - - In cases where the layout would require a fractional tab - - character, the behavior of the fractional indentation is - - governed by + - If true, lines are indented using tab characters (utf-8 + - 0x09) instead of space characters (utf-8 0x20). + - In cases where the layout would require a fractional tab + - character, the behavior of the fractional indentation is + - governed by use_tabchars: false _help_fractional_tab_policy: - - If is True, then the value of this variable - - indicates how fractional indentions are handled during - - whitespace replacement. If set to 'use-space', fractional - - indentation is left as spaces (utf-8 0x20). If set to - - '`round-up` fractional indentation is replaced with a single' - - tab character (utf-8 0x09) effectively shifting the column - - to the next tabstop + - If is True, then the value of this variable + - indicates how fractional indentions are handled during + - whitespace replacement. If set to 'use-space', fractional + - indentation is left as spaces (utf-8 0x20). If set to + - "`round-up` fractional indentation is replaced with a single" + - tab character (utf-8 0x09) effectively shifting the column + - to the next tabstop fractional_tab_policy: use-space _help_max_subgroups_hwrap: - - If an argument group contains more than this many sub-groups - - (parg or kwarg groups) then force it to a vertical layout. + - If an argument group contains more than this many sub-groups + - (parg or kwarg groups) then force it to a vertical layout. max_subgroups_hwrap: 4 _help_max_pargs_hwrap: - - If a positional argument group contains more than this many - - arguments, then force it to a vertical layout. + - If a positional argument group contains more than this many + - arguments, then force it to a vertical layout. max_pargs_hwrap: 6 _help_max_rows_cmdline: - - If a cmdline positional group consumes more than this many - - lines without nesting, then invalidate the layout (and nest) + - If a cmdline positional group consumes more than this many + - lines without nesting, then invalidate the layout (and nest) max_rows_cmdline: 2 _help_separate_ctrl_name_with_space: - - If true, separate flow control names from their parentheses - - with a space + - If true, separate flow control names from their parentheses + - with a space separate_ctrl_name_with_space: true _help_separate_fn_name_with_space: - - If true, separate function names from parentheses with a - - space + - If true, separate function names from parentheses with a + - space separate_fn_name_with_space: false _help_dangle_parens: - - If a statement is wrapped to more than one line, than dangle - - the closing parenthesis on its own line. + - If a statement is wrapped to more than one line, than dangle + - the closing parenthesis on its own line. dangle_parens: true _help_dangle_align: - - If the trailing parenthesis must be 'dangled' on its on - - 'line, then align it to this reference: `prefix`: the start' - - 'of the statement, `prefix-indent`: the start of the' - - 'statement, plus one indentation level, `child`: align to' - - the column of the arguments + - If the trailing parenthesis must be 'dangled' on its on + - "line, then align it to this reference: `prefix`: the start" + - "of the statement, `prefix-indent`: the start of the" + - "statement, plus one indentation level, `child`: align to" + - the column of the arguments dangle_align: prefix _help_min_prefix_chars: - - If the statement spelling length (including space and - - parenthesis) is smaller than this amount, then force reject - - nested layouts. + - If the statement spelling length (including space and + - parenthesis) is smaller than this amount, then force reject + - nested layouts. min_prefix_chars: 4 _help_max_prefix_chars: - - If the statement spelling length (including space and - - parenthesis) is larger than the tab width by more than this - - amount, then force reject un-nested layouts. + - If the statement spelling length (including space and + - parenthesis) is larger than the tab width by more than this + - amount, then force reject un-nested layouts. max_prefix_chars: 10 _help_max_lines_hwrap: - - If a candidate layout is wrapped horizontally but it exceeds - - this many lines, then reject the layout. + - If a candidate layout is wrapped horizontally but it exceeds + - this many lines, then reject the layout. max_lines_hwrap: 2 _help_line_ending: - - What style line endings to use in the output. + - What style line endings to use in the output. line_ending: unix _help_command_case: - - Format command names consistently as 'lower' or 'upper' case + - Format command names consistently as 'lower' or 'upper' case command_case: canonical _help_keyword_case: - - Format keywords consistently as 'lower' or 'upper' case + - Format keywords consistently as 'lower' or 'upper' case keyword_case: unchanged _help_always_wrap: - - A list of command names which should always be wrapped + - A list of command names which should always be wrapped always_wrap: [] _help_enable_sort: - - If true, the argument lists which are known to be sortable - - will be sorted lexicographicall + - If true, the argument lists which are known to be sortable + - will be sorted lexicographicall enable_sort: true _help_autosort: - - If true, the parsers may infer whether or not an argument - - list is sortable (without annotation). + - If true, the parsers may infer whether or not an argument + - list is sortable (without annotation). autosort: true _help_require_valid_layout: - - By default, if cmake-format cannot successfully fit - - everything into the desired linewidth it will apply the - - last, most agressive attempt that it made. If this flag is - - True, however, cmake-format will print error, exit with non- - - zero status code, and write-out nothing + - By default, if cmake-format cannot successfully fit + - everything into the desired linewidth it will apply the + - last, most aggressive attempt that it made. If this flag is + - True, however, cmake-format will print error, exit with non- + - zero status code, and write-out nothing require_valid_layout: false _help_layout_passes: - - A dictionary mapping layout nodes to a list of wrap - - decisions. See the documentation for more information. + - A dictionary mapping layout nodes to a list of wrap + - decisions. See the documentation for more information. layout_passes: {} _help_markup: Options affecting comment reflow and formatting. markup: _help_bullet_char: - - What character to use for bulleted lists - bullet_char: '*' + - What character to use for bulleted lists + bullet_char: "*" _help_enum_char: - - What character to use as punctuation after numerals in an - - enumerated list + - What character to use as punctuation after numerals in an + - enumerated list enum_char: . _help_first_comment_is_literal: - - If comment markup is enabled, don't reflow the first comment - - block in each listfile. Use this to preserve formatting of - - your copyright/license statements. + - If comment markup is enabled, don't reflow the first comment + - block in each listfile. Use this to preserve formatting of + - your copyright/license statements. first_comment_is_literal: false _help_literal_comment_pattern: - - If comment markup is enabled, don't reflow any comment block - - which matches this (regex) pattern. Default is `None` - - (disabled). + - If comment markup is enabled, don't reflow any comment block + - which matches this (regex) pattern. Default is `None` + - (disabled). literal_comment_pattern: null _help_fence_pattern: - - Regular expression to match preformat fences in comments - - default= ``r'^\s*([`~]{3}[`~]*)(.*)$'`` + - Regular expression to match preformat fences in comments + - default= ``r'^\s*([`~]{3}[`~]*)(.*)$'`` fence_pattern: ^\s*([`~]{3}[`~]*)(.*)$ _help_ruler_pattern: - - Regular expression to match rulers in comments default= - - '``r''^\s*[^\w\s]{3}.*[^\w\s]{3}$''``' + - Regular expression to match rulers in comments default= + - '``r''^\s*[^\w\s]{3}.*[^\w\s]{3}$''``' ruler_pattern: ^\s*[^\w\s]{3}.*[^\w\s]{3}$ _help_explicit_trailing_pattern: - - If a comment line matches starts with this pattern then it - - is explicitly a trailing comment for the preceeding - - argument. Default is '#<' - explicit_trailing_pattern: '#<' + - If a comment line matches starts with this pattern then it + - is explicitly a trailing comment for the preceding + - argument. Default is '#<' + explicit_trailing_pattern: "#<" _help_hashruler_min_length: - - If a comment line starts with at least this many consecutive - - hash characters, then don't lstrip() them off. This allows - - for lazy hash rulers where the first hash char is not - - separated by space + - If a comment line starts with at least this many consecutive + - hash characters, then don't lstrip() them off. This allows + - for lazy hash rulers where the first hash char is not + - separated by space hashruler_min_length: 10 _help_canonicalize_hashrulers: - - If true, then insert a space between the first hash char and - - remaining hash chars in a hash ruler, and normalize its - - length to fill the column + - If true, then insert a space between the first hash char and + - remaining hash chars in a hash ruler, and normalize its + - length to fill the column canonicalize_hashrulers: true _help_enable_markup: - - enable comment markup parsing and reflow + - enable comment markup parsing and reflow enable_markup: true _help_lint: Options affecting the linter lint: _help_disabled_codes: - - a list of lint codes to disable + - a list of lint codes to disable disabled_codes: [] _help_function_pattern: - - regular expression pattern describing valid function names - function_pattern: '[0-9a-z_]+' + - regular expression pattern describing valid function names + function_pattern: "[0-9a-z_]+" _help_macro_pattern: - - regular expression pattern describing valid macro names - macro_pattern: '[0-9A-Z_]+' + - regular expression pattern describing valid macro names + macro_pattern: "[0-9A-Z_]+" _help_global_var_pattern: - - regular expression pattern describing valid names for - - variables with global (cache) scope - global_var_pattern: '[A-Z][0-9A-Z_]+' + - regular expression pattern describing valid names for + - variables with global (cache) scope + global_var_pattern: "[A-Z][0-9A-Z_]+" _help_internal_var_pattern: - - regular expression pattern describing valid names for - - variables with global scope (but internal semantic) + - regular expression pattern describing valid names for + - variables with global scope (but internal semantic) internal_var_pattern: _[A-Z][0-9A-Z_]+ _help_local_var_pattern: - - regular expression pattern describing valid names for - - variables with local scope - local_var_pattern: '[a-z][a-z0-9_]+' + - regular expression pattern describing valid names for + - variables with local scope + local_var_pattern: "[a-z][a-z0-9_]+" _help_private_var_pattern: - - regular expression pattern describing valid names for - - privatedirectory variables + - regular expression pattern describing valid names for + - privatedirectory variables private_var_pattern: _[0-9a-z_]+ _help_public_var_pattern: - - regular expression pattern describing valid names for public - - directory variables - public_var_pattern: '[A-Z][0-9A-Z_]+' + - regular expression pattern describing valid names for public + - directory variables + public_var_pattern: "[A-Z][0-9A-Z_]+" _help_argument_var_pattern: - - regular expression pattern describing valid names for - - function/macro arguments and loop variables. - argument_var_pattern: '[a-z][a-z0-9_]+' + - regular expression pattern describing valid names for + - function/macro arguments and loop variables. + argument_var_pattern: "[a-z][a-z0-9_]+" _help_keyword_pattern: - - regular expression pattern describing valid names for - - keywords used in functions or macros - keyword_pattern: '[A-Z][0-9A-Z_]+' + - regular expression pattern describing valid names for + - keywords used in functions or macros + keyword_pattern: "[A-Z][0-9A-Z_]+" _help_max_conditionals_custom_parser: - - In the heuristic for C0201, how many conditionals to match - - within a loop in before considering the loop a parser. + - In the heuristic for C0201, how many conditionals to match + - within a loop in before considering the loop a parser. max_conditionals_custom_parser: 2 _help_min_statement_spacing: - - Require at least this many newlines between statements + - Require at least this many newlines between statements min_statement_spacing: 1 _help_max_statement_spacing: - - Require no more than this many newlines between statements + - Require no more than this many newlines between statements max_statement_spacing: 2 max_returns: 6 max_branches: 12 @@ -226,20 +226,20 @@ lint: _help_encode: Options affecting file encoding encode: _help_emit_byteorder_mark: - - If true, emit the unicode byte-order mark (BOM) at the start - - of the file + - If true, emit the unicode byte-order mark (BOM) at the start + - of the file emit_byteorder_mark: false _help_input_encoding: - - Specify the encoding of the input file. Defaults to utf-8 + - Specify the encoding of the input file. Defaults to utf-8 input_encoding: utf-8 _help_output_encoding: - - Specify the encoding of the output file. Defaults to utf-8. - - Note that cmake only claims to support utf-8 so be careful - - when using anything else + - Specify the encoding of the output file. Defaults to utf-8. + - Note that cmake only claims to support utf-8 so be careful + - when using anything else output_encoding: utf-8 _help_misc: Miscellaneous configurations options. misc: _help_per_command: - - A dictionary containing any per-command configuration - - overrides. Currently only `command_case` is supported. + - A dictionary containing any per-command configuration + - overrides. Currently only `command_case` is supported. per_command: {} diff --git a/.codecov.yml b/.codecov.yml index 3fc7bfb18..5fffb1c2a 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -9,3 +9,14 @@ coverage: default: target: 20% # Need to bump this number https://docs.codecov.com/docs/commit-status#patch-status threshold: 2% + +# `codecov/codecov-action` reruns `gcovr` if build files present +# That's why we run it in a separate workflow +# This ignore list is not currently used +# +# More info: https://github.com/XRPLF/clio/pull/2066 +ignore: + - "tests" + - "src/data/cassandra/" + - "src/data/CassandraBackend.hpp" + - "src/data/BackendFactory.*" diff --git a/.githooks/check-format b/.githooks/check-format deleted file mode 100755 index 3ffd85816..000000000 --- a/.githooks/check-format +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -# Note: This script is intended to be run from the root of the repository. -# -# This script checks the format of the code and cmake files. -# In many cases it will automatically fix the issues and abort the commit. - -no_formatted_directories_staged() { - staged_directories=$(git diff-index --cached --name-only HEAD | awk -F/ '{print $1}') - for sd in $staged_directories; do - if [[ "$sd" =~ ^(benchmark|cmake|src|tests)$ ]]; then - return 1 - fi - done - return 0 -} - -if no_formatted_directories_staged ; then - exit 0 -fi - -echo "+ Checking code format..." - -# paths to check and re-format -sources="src tests" -formatter="clang-format -i" -version=$($formatter --version | grep -o '[0-9\.]*') - -if [[ "19.0.0" > "$version" ]]; then - cat < /dev/null; then - cat <&1 | grep -q 'GNU' && echo true || echo false) - -if [[ "$GNU_SED" == "false" ]]; then # macOS sed - # make all includes to be <...> style - grep_code '#include ".*"' | xargs sed -i '' -E 's|#include "(.*)"|#include <\1>|g' - - # make local includes to be "..." style - main_src_dirs=$(find ./src -maxdepth 1 -type d -exec basename {} \; | tr '\n' '|' | sed 's/|$//' | sed 's/|/\\|/g') - grep_code "#include <\($main_src_dirs\)/.*>" | xargs sed -i '' -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g" -else - # make all includes to be <...> style - grep_code '#include ".*"' | xargs sed -i -E 's|#include "(.*)"|#include <\1>|g' - - # make local includes to be "..." style - main_src_dirs=$(find ./src -maxdepth 1 -type d -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g') - grep_code "#include <\($main_src_dirs\)/.*>" | xargs sed -i -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g" -fi - -cmake_dirs=$(echo cmake $sources) -cmake_files=$(find $cmake_dirs -type f \( -name "CMakeLists.txt" -o -name "*.cmake" \)) -cmake_files=$(echo $cmake_files ./CMakeLists.txt) - -first=$(git diff $sources $cmake_files) -find $sources -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.ipp' \) -print0 | xargs -0 $formatter -cmake-format -i $cmake_files -second=$(git diff $sources $cmake_files) -changes=$(diff <(echo "$first") <(echo "$second")) -changes_number=$(echo -n "$changes" | wc -l | sed -e 's/^[[:space:]]*//') - -if [ "$changes_number" != "0" ]; then - cat <<\EOF - - WARNING ------------------------------------------------------------------------------ - Automatically re-formatted code with 'clang-format' - commit was aborted. - Please manually add any updated files and commit again. ------------------------------------------------------------------------------ - -EOF - if [[ "$1" == "--diff" ]]; then - echo "$changes" - fi - exit 1 -fi diff --git a/.githooks/post-checkout b/.githooks/post-checkout deleted file mode 100755 index ca7fcb400..000000000 --- a/.githooks/post-checkout +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-checkout' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; } -git lfs post-checkout "$@" diff --git a/.githooks/post-commit b/.githooks/post-commit deleted file mode 100755 index 52b339cb3..000000000 --- a/.githooks/post-commit +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-commit' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; } -git lfs post-commit "$@" diff --git a/.githooks/post-merge b/.githooks/post-merge deleted file mode 100755 index a912e667a..000000000 --- a/.githooks/post-merge +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'post-merge' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; } -git lfs post-merge "$@" diff --git a/.githooks/pre-commit b/.githooks/pre-commit deleted file mode 100755 index 241ff9ae8..000000000 --- a/.githooks/pre-commit +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# This script is intended to be run from the root of the repository. - -source .githooks/check-format -source .githooks/check-docs - diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a42907dde..fdd170db0 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -3,29 +3,34 @@ name: Bug report about: Create a report to help us improve title: "[Title with short description] (Version: [Clio version])" labels: bug -assignees: '' - +assignees: "" --- ## Issue Description + ## Steps to Reproduce + ## Expected Result + ## Actual Result + ## Environment + ## Supporting Files + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index addc43740..0c0c4c3ca 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -3,21 +3,24 @@ name: Feature request about: Suggest an idea for this project title: "[Title with short description] (Version: [Clio version])" labels: enhancement -assignees: '' - +assignees: "" --- ## Summary + ## Motivation + ## Solution + ## Paths Not Taken + diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index c20e097c7..3320d8ad1 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -3,8 +3,7 @@ name: Question about: A question in form of an issue title: "[Title with short description] (Version: Clio version)" labels: question -assignees: '' - +assignees: "" --- @@ -12,7 +11,9 @@ assignees: '' ## Question + ## Paths Not Taken + diff --git a/.github/actions/build_clio/action.yml b/.github/actions/build_clio/action.yml index 2287ad22e..cc2381a4b 100644 --- a/.github/actions/build_clio/action.yml +++ b/.github/actions/build_clio/action.yml @@ -1,13 +1,15 @@ name: Build clio description: Build clio in build directory + inputs: - target: - description: Build target name + targets: + description: Space-separated build target names default: all - substract_threads: - description: An option for the action get_number_of_threads. See get_number_of_threads + subtract_threads: + description: An option for the action get_number_of_threads. See get_number_of_threads required: true - default: '0' + default: "0" + runs: using: composite steps: @@ -15,10 +17,13 @@ runs: uses: ./.github/actions/get_number_of_threads id: number_of_threads with: - substract_threads: ${{ inputs.substract_threads }} + subtract_threads: ${{ inputs.subtract_threads }} - - name: Build Clio + - name: Build targets shell: bash run: | cd build - cmake --build . --parallel ${{ steps.number_of_threads.outputs.threads_number }} --target ${{ inputs.target }} + cmake \ + --build . \ + --parallel "${{ steps.number_of_threads.outputs.threads_number }}" \ + --target ${{ inputs.targets }} diff --git a/.github/actions/build_docker_image/action.yml b/.github/actions/build_docker_image/action.yml index c3118a6d5..b0155cc9b 100644 --- a/.github/actions/build_docker_image/action.yml +++ b/.github/actions/build_docker_image/action.yml @@ -1,8 +1,12 @@ name: Build and push Docker image description: Build and push Docker image to DockerHub and GitHub Container Registry + inputs: - image_name: - description: Name of the image to build + images: + description: Name of the images to use as a base name + required: true + dockerhub_repo: + description: DockerHub repository name required: true push_image: description: Whether to push the image to the registry (true/false) @@ -19,48 +23,50 @@ inputs: description: description: Short description of the image required: true + runs: using: composite steps: - - name: Login to DockerHub - if: ${{ inputs.push_image == 'true' }} - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKERHUB_USER }} - password: ${{ env.DOCKERHUB_PW }} + - name: Login to DockerHub + if: ${{ inputs.push_image == 'true' }} + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + username: ${{ env.DOCKERHUB_USER }} + password: ${{ env.DOCKERHUB_PW }} - - name: Login to GitHub Container Registry - if: ${{ inputs.push_image == 'true' }} - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ env.GITHUB_TOKEN }} + - name: Login to GitHub Container Registry + if: ${{ inputs.push_image == 'true' }} + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ env.GITHUB_TOKEN }} - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 + - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + with: + cache-image: false + - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 - - uses: docker/metadata-action@v5 - id: meta - with: - images: ${{ inputs.image_name }} - tags: ${{ inputs.tags }} + - uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 + id: meta + with: + images: ${{ inputs.images }} + tags: ${{ inputs.tags }} - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: ${{ inputs.directory }} - platforms: ${{ inputs.platforms }} - push: ${{ inputs.push_image == 'true' }} - tags: ${{ steps.meta.outputs.tags }} - - - name: Update DockerHub description - if: ${{ inputs.push_image == 'true' }} - uses: peter-evans/dockerhub-description@v4 - with: - username: ${{ env.DOCKERHUB_USER }} - password: ${{ env.DOCKERHUB_PW }} - repository: ${{ inputs.image_name }} - short-description: ${{ inputs.description }} - readme-filepath: ${{ inputs.directory }}/README.md + - name: Build and push + uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0 + with: + context: ${{ inputs.directory }} + platforms: ${{ inputs.platforms }} + push: ${{ inputs.push_image == 'true' }} + tags: ${{ steps.meta.outputs.tags }} + - name: Update DockerHub description + if: ${{ inputs.push_image == 'true' }} + uses: peter-evans/dockerhub-description@432a30c9e07499fd01da9f8a49f0faf9e0ca5b77 # v4.0.2 + with: + username: ${{ env.DOCKERHUB_USER }} + password: ${{ env.DOCKERHUB_PW }} + repository: ${{ inputs.dockerhub_repo }} + short-description: ${{ inputs.description }} + readme-filepath: ${{ inputs.directory }}/README.md diff --git a/.github/actions/code_coverage/action.yml b/.github/actions/code_coverage/action.yml index 94dd0fceb..790372074 100644 --- a/.github/actions/code_coverage/action.yml +++ b/.github/actions/code_coverage/action.yml @@ -1,21 +1,26 @@ name: Generate code coverage report description: Run tests, generate code coverage report and upload it to codecov.io + runs: using: composite + steps: - name: Run tests shell: bash run: | build/clio_tests + # Please keep exclude list in sync with .codecov.yml - name: Run gcovr shell: bash run: | - gcovr -e tests \ + gcovr \ + -e tests \ -e src/data/cassandra \ -e src/data/CassandraBackend.hpp \ -e 'src/data/BackendFactory.*' \ - --xml build/coverage_report.xml -j8 --exclude-throw-branches + --xml build/coverage_report.xml \ + -j8 --exclude-throw-branches - name: Archive coverage report uses: actions/upload-artifact@v4 diff --git a/.github/actions/create_issue/action.yml b/.github/actions/create_issue/action.yml index 43d125593..9a572aa6f 100644 --- a/.github/actions/create_issue/action.yml +++ b/.github/actions/create_issue/action.yml @@ -1,5 +1,6 @@ name: Create an issue description: Create an issue + inputs: title: description: Issue title @@ -10,26 +11,31 @@ inputs: labels: description: Comma-separated list of labels required: true - default: 'bug' + default: "bug" assignees: description: Comma-separated list of assignees required: true - default: 'cindyyan317,godexsoft,kuznetsss' + default: "godexsoft,kuznetsss,PeterChen13579,mathbunnyru" + outputs: created_issue_id: description: Created issue id value: ${{ steps.create_issue.outputs.created_issue }} + runs: using: composite steps: - - name: Create an issue - id: create_issue - shell: bash - run: | - echo -e '${{ inputs.body }}' > issue.md - gh issue create --assignee '${{ inputs.assignees }}' --label '${{ inputs.labels }}' --title '${{ inputs.title }}' --body-file ./issue.md > create_issue.log - created_issue=$(cat create_issue.log | sed 's|.*/||') - echo "created_issue=$created_issue" >> $GITHUB_OUTPUT - rm create_issue.log issue.md - - + - name: Create an issue + id: create_issue + shell: bash + run: | + echo -e '${{ inputs.body }}' > issue.md + gh issue create \ + --assignee '${{ inputs.assignees }}' \ + --label '${{ inputs.labels }}' \ + --title '${{ inputs.title }}' \ + --body-file ./issue.md \ + > create_issue.log + created_issue="$(sed 's|.*/||' create_issue.log)" + echo "created_issue=$created_issue" >> $GITHUB_OUTPUT + rm create_issue.log issue.md diff --git a/.github/actions/generate/action.yml b/.github/actions/generate/action.yml index 1926cf135..3c2f91e45 100644 --- a/.github/actions/generate/action.yml +++ b/.github/actions/generate/action.yml @@ -1,5 +1,6 @@ name: Run conan and cmake description: Run conan and cmake + inputs: conan_profile: description: Conan profile name @@ -7,27 +8,37 @@ inputs: conan_cache_hit: description: Whether conan cache has been downloaded required: true - default: 'false' + default: "false" build_type: description: Build type for third-party libraries and clio. Could be 'Release', 'Debug' required: true - default: 'Release' + default: "Release" build_integration_tests: description: Whether to build integration tests required: true - default: 'true' + default: "true" code_coverage: description: Whether conan's coverage option should be on or not required: true - default: 'false' + default: "false" static: description: Whether Clio is to be statically linked required: true - default: 'false' + default: "false" sanitizer: description: Sanitizer to use required: true - default: 'false' # false, tsan, asan or ubsan + default: "false" + choices: + - "false" + - "tsan" + - "asan" + - "ubsan" + time_trace: + description: Whether to enable compiler trace reports + required: true + default: "false" + runs: using: composite steps: @@ -42,19 +53,36 @@ runs: CODE_COVERAGE: "${{ inputs.code_coverage == 'true' && 'True' || 'False' }}" STATIC_OPTION: "${{ inputs.static == 'true' && 'True' || 'False' }}" INTEGRATION_TESTS_OPTION: "${{ inputs.build_integration_tests == 'true' && 'True' || 'False' }}" + TIME_TRACE: "${{ inputs.time_trace == 'true' && 'True' || 'False' }}" run: | cd build - conan install .. -of . -b $BUILD_OPTION -s build_type=${{ inputs.build_type }} -o clio:static="${STATIC_OPTION}" -o clio:tests=True -o clio:integration_tests="${INTEGRATION_TESTS_OPTION}" -o clio:lint=False -o clio:coverage="${CODE_COVERAGE}" --profile ${{ inputs.conan_profile }} - + conan \ + install .. \ + -of . \ + -b $BUILD_OPTION \ + -s build_type="${{ inputs.build_type }}" \ + -o clio:static="${STATIC_OPTION}" \ + -o clio:tests=True \ + -o clio:integration_tests="${INTEGRATION_TESTS_OPTION}" \ + -o clio:lint=False \ + -o clio:coverage="${CODE_COVERAGE}" \ + -o clio:time_trace="${TIME_TRACE}" \ + --profile "${{ inputs.conan_profile }}" + - name: Run cmake shell: bash env: BUILD_TYPE: "${{ inputs.build_type }}" - SANITIZER_OPTION: | + SANITIZER_OPTION: |- ${{ inputs.sanitizer == 'tsan' && '-Dsan=thread' || inputs.sanitizer == 'ubsan' && '-Dsan=undefined' || inputs.sanitizer == 'asan' && '-Dsan=address' || '' }} run: | cd build - cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" ${SANITIZER_OPTION} .. -G Ninja + cmake \ + -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \ + -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ + "${SANITIZER_OPTION}" \ + .. \ + -G Ninja diff --git a/.github/actions/get_number_of_threads/action.yml b/.github/actions/get_number_of_threads/action.yml index 088371b9e..863988afa 100644 --- a/.github/actions/get_number_of_threads/action.yml +++ b/.github/actions/get_number_of_threads/action.yml @@ -1,14 +1,16 @@ name: Get number of threads description: Determines number of threads to use on macOS and Linux + inputs: - substract_threads: - description: How many threads to substract from the calculated number + subtract_threads: + description: How many threads to subtract from the calculated number required: true - default: '0' + default: "0" outputs: threads_number: description: Number of threads to use value: ${{ steps.number_of_threads_export.outputs.num }} + runs: using: composite steps: @@ -28,7 +30,7 @@ runs: id: number_of_threads_export shell: bash run: | - num_of_threads=${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }} - shift_by=${{ inputs.substract_threads }} - shifted=$((num_of_threads - shift_by)) + num_of_threads="${{ steps.mac_threads.outputs.num || steps.linux_threads.outputs.num }}" + shift_by="${{ inputs.subtract_threads }}" + shifted="$((num_of_threads - shift_by))" echo "num=$(( shifted > 1 ? shifted : 1 ))" >> $GITHUB_OUTPUT diff --git a/.github/actions/git_common_ancestor/action.yml b/.github/actions/git_common_ancestor/action.yml index 8461d1b3e..1e7f16b58 100644 --- a/.github/actions/git_common_ancestor/action.yml +++ b/.github/actions/git_common_ancestor/action.yml @@ -1,9 +1,11 @@ name: Git common ancestor description: Find the closest common commit + outputs: commit: description: Hash of commit value: ${{ steps.find_common_ancestor.outputs.commit }} + runs: using: composite steps: @@ -11,4 +13,4 @@ runs: id: find_common_ancestor shell: bash run: | - echo "commit=$(git merge-base --fork-point origin/develop)" >> $GITHUB_OUTPUT + echo "commit=\"$(git merge-base --fork-point origin/develop)\"" >> $GITHUB_OUTPUT diff --git a/.github/actions/prepare_runner/action.yml b/.github/actions/prepare_runner/action.yml index 73fcd22ef..5a2024838 100644 --- a/.github/actions/prepare_runner/action.yml +++ b/.github/actions/prepare_runner/action.yml @@ -1,9 +1,11 @@ name: Prepare runner description: Install packages, set environment variables, create directories + inputs: disable_ccache: description: Whether ccache should be disabled required: true + runs: using: composite steps: @@ -11,39 +13,42 @@ runs: if: ${{ runner.os == 'macOS' }} shell: bash run: | - brew install llvm@14 pkg-config ninja bison ccache jq gh conan@1 ca-certificates - echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH + brew install \ + bison \ + ca-certificates \ + ccache \ + clang-build-analyzer \ + conan@1 \ + gh \ + jq \ + llvm@14 \ + ninja \ + pkg-config + echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH - name: Install CMake 3.31.6 on mac if: ${{ runner.os == 'macOS' }} shell: bash run: | - # Uninstall any existing cmake - brew uninstall cmake --ignore-dependencies || true - - # Download specific cmake formula - FORMULA_URL="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb" - FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4" - - mkdir -p /tmp/homebrew-formula - curl -s -L $FORMULA_URL -o /tmp/homebrew-formula/cmake.rb - - # Verify the downloaded formula - ACTUAL_SHA256=$(shasum -a 256 /tmp/homebrew-formula/cmake.rb | cut -d ' ' -f 1) - if [ "$ACTUAL_SHA256" != "$FORMULA_EXPECTED_SHA256" ]; then - echo "Error: Formula checksum mismatch" - echo "Expected: $FORMULA_EXPECTED_SHA256" - echo "Actual: $ACTUAL_SHA256" - exit 1 - fi - - # Install cmake from the specific formula with force flag - brew install --force /tmp/homebrew-formula/cmake.rb + # Uninstall any existing cmake + brew uninstall cmake --ignore-dependencies || true + + # Download specific cmake formula + FORMULA_URL="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb" + FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4" + + mkdir -p /tmp/homebrew-formula + curl -s -L "$FORMULA_URL" -o /tmp/homebrew-formula/cmake.rb + + echo "$FORMULA_EXPECTED_SHA256 /tmp/homebrew-formula/cmake.rb" | shasum -a 256 -c + + # Install cmake from the specific formula with force flag + brew install --formula --force /tmp/homebrew-formula/cmake.rb - name: Fix git permissions on Linux if: ${{ runner.os == 'Linux' }} shell: bash - run: git config --global --add safe.directory $PWD + run: git config --global --add safe.directory "$PWD" - name: Set env variables for macOS if: ${{ runner.os == 'macOS' }} @@ -68,7 +73,5 @@ runs: - name: Create directories shell: bash run: | - mkdir -p $CCACHE_DIR - mkdir -p $CONAN_USER_HOME/.conan - - + mkdir -p "$CCACHE_DIR" + mkdir -p "$CONAN_USER_HOME/.conan" diff --git a/.github/actions/restore_cache/action.yml b/.github/actions/restore_cache/action.yml index fe2ebf35e..6032e7e86 100644 --- a/.github/actions/restore_cache/action.yml +++ b/.github/actions/restore_cache/action.yml @@ -1,5 +1,6 @@ name: Restore cache description: Find and restores conan and ccache cache + inputs: conan_dir: description: Path to .conan directory @@ -17,7 +18,7 @@ inputs: code_coverage: description: Whether code coverage is on required: true - default: 'false' + default: "false" outputs: conan_hash: description: Hash to use as a part of conan cache key @@ -28,6 +29,7 @@ outputs: ccache_cache_hit: description: True if ccache cache has been downloaded value: ${{ steps.ccache_cache.outputs.cache-hit }} + runs: using: composite steps: @@ -40,9 +42,9 @@ runs: shell: bash run: | conan info . -j info.json -o clio:tests=True - packages_info=$(cat info.json | jq '.[] | "\(.display_name): \(.id)"' | grep -v 'clio') + packages_info="$(cat info.json | jq '.[] | "\(.display_name): \(.id)"' | grep -v 'clio')" echo "$packages_info" - hash=$(echo "$packages_info" | shasum -a 256 | cut -d ' ' -f 1) + hash="$(echo "$packages_info" | shasum -a 256 | cut -d ' ' -f 1)" rm info.json echo "hash=$hash" >> $GITHUB_OUTPUT diff --git a/.github/actions/save_cache/action.yml b/.github/actions/save_cache/action.yml index 0511a7638..51e320abf 100644 --- a/.github/actions/save_cache/action.yml +++ b/.github/actions/save_cache/action.yml @@ -1,5 +1,6 @@ name: Save cache description: Save conan and ccache cache for develop branch + inputs: conan_dir: description: Path to .conan directory @@ -28,7 +29,8 @@ inputs: code_coverage: description: Whether code coverage is on required: true - default: 'false' + default: "false" + runs: using: composite steps: @@ -55,5 +57,3 @@ runs: with: path: ${{ inputs.ccache_dir }} key: clio-ccache-${{ runner.os }}-${{ inputs.build_type }}${{ inputs.code_coverage == 'true' && '-code_coverage' || '' }}-${{ inputs.conan_profile }}-develop-${{ steps.git_common_ancestor.outputs.commit }} - - diff --git a/.github/actions/setup_conan/action.yml b/.github/actions/setup_conan/action.yml index d77bfb90f..47f34a8f9 100644 --- a/.github/actions/setup_conan/action.yml +++ b/.github/actions/setup_conan/action.yml @@ -1,52 +1,33 @@ name: Setup conan description: Setup conan profile and artifactory + inputs: conan_profile: description: Conan profile name required: true -outputs: - conan_profile: - description: Created conan profile name - value: ${{ steps.conan_export_output.outputs.conan_profile }} + runs: using: composite steps: - - name: On mac + - name: Create conan profile on macOS if: ${{ runner.os == 'macOS' }} shell: bash env: - CONAN_PROFILE: apple_clang_16 - id: conan_setup_mac + CONAN_PROFILE: ${{ inputs.conan_profile }} run: | - echo "Creating $CONAN_PROFILE conan profile" - conan profile new $CONAN_PROFILE --detect --force - conan profile update settings.compiler.libcxx=libc++ $CONAN_PROFILE - conan profile update settings.compiler.cppstd=20 $CONAN_PROFILE - conan profile update env.CXXFLAGS=-DBOOST_ASIO_DISABLE_CONCEPTS $CONAN_PROFILE - conan profile update "conf.tools.build:cxxflags+=[\"-DBOOST_ASIO_DISABLE_CONCEPTS\"]" $CONAN_PROFILE - echo "created_conan_profile=$CONAN_PROFILE" >> $GITHUB_OUTPUT - - - name: On linux - if: ${{ runner.os == 'Linux' }} - shell: bash - id: conan_setup_linux - run: | - echo "created_conan_profile=${{ inputs.conan_profile }}" >> $GITHUB_OUTPUT - - - name: Export output variable - shell: bash - id: conan_export_output - run: | - echo "conan_profile=${{ steps.conan_setup_mac.outputs.created_conan_profile || steps.conan_setup_linux.outputs.created_conan_profile }}" >> $GITHUB_OUTPUT + echo "Creating \"$CONAN_PROFILE\" conan profile" + conan profile new "$CONAN_PROFILE" --detect --force + conan profile update settings.compiler.libcxx=libc++ "$CONAN_PROFILE" + conan profile update settings.compiler.cppstd=20 "$CONAN_PROFILE" + conan profile update env.CXXFLAGS=-DBOOST_ASIO_DISABLE_CONCEPTS "$CONAN_PROFILE" + conan profile update "conf.tools.build:cxxflags+=[\"-DBOOST_ASIO_DISABLE_CONCEPTS\"]" "$CONAN_PROFILE" - name: Add conan-non-prod artifactory shell: bash run: | - if [[ -z $(conan remote list | grep conan-non-prod) ]]; then + if [[ -z "$(conan remote list | grep conan-non-prod)" ]]; then echo "Adding conan-non-prod" conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod else echo "Conan-non-prod is available" fi - - diff --git a/.github/actions/test/Dockerfile b/.github/actions/test/Dockerfile deleted file mode 100644 index a2c10da44..000000000 --- a/.github/actions/test/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM cassandra:4.0.4 - -RUN apt-get update && apt-get install -y postgresql -COPY entrypoint.sh /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/test/entrypoint.sh b/.github/actions/test/entrypoint.sh deleted file mode 100755 index 86d5fadd8..000000000 --- a/.github/actions/test/entrypoint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -pg_ctlcluster 12 main start -su postgres -c"psql -c\"alter user postgres with password 'postgres'\"" -su cassandra -c "/opt/cassandra/bin/cassandra -R" -sleep 90 -chmod +x ./clio_tests -./clio_tests diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 090a77fd7..e6d1d3873 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,16 +1,157 @@ version: 2 updates: - - package-ecosystem: "github-actions" - directory: "/" + - package-ecosystem: github-actions + directory: / schedule: - interval: "weekly" - day: "monday" + interval: weekly + day: monday time: "04:00" - timezone: "Etc/GMT" + timezone: Etc/GMT reviewers: - - "cindyyan317" - - "godexsoft" - - "kuznetsss" + - XRPLF/clio-dev-team commit-message: - prefix: "[CI] " - target-branch: "develop" + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/build_clio/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/build_docker_image/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/code_coverage/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/create_issue/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/generate/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/get_number_of_threads/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/git_common_ancestor/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/prepare_runner/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/restore_cache/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/save_cache/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop + + - package-ecosystem: github-actions + directory: .github/actions/setup_conan/ + schedule: + interval: weekly + day: monday + time: "04:00" + timezone: Etc/GMT + reviewers: + - XRPLF/clio-dev-team + commit-message: + prefix: "ci: [DEPENDABOT] " + target-branch: develop diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 45d62bb7c..a26139996 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,170 +1,120 @@ name: Build + on: push: branches: [master, release/*, develop] pull_request: branches: [master, release/*, develop] + paths: + - .github/workflows/build.yml + + - .github/workflows/build_and_test.yml + - .github/workflows/build_impl.yml + - .github/workflows/test_impl.yml + - .github/workflows/upload_coverage_report.yml + + - ".github/actions/**" + - "!.github/actions/build_docker_image/**" + - "!.github/actions/create_issue/**" + + - CMakeLists.txt + - "cmake/**" + - "src/**" + - "tests/**" + + - docs/config-description.md workflow_dispatch: +concurrency: + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - check_format: - name: Check format - runs-on: ubuntu-latest - container: - image: rippleci/clio_ci:latest - steps: - - name: Fix git permissions on Linux - shell: bash - run: git config --global --add safe.directory $PWD + build-and-test: + name: Build and Test - - uses: actions/checkout@v4 - - name: Run formatters - id: run_formatters - run: | - ./.githooks/check-format --diff - shell: bash - - check_docs: - name: Check documentation - runs-on: ubuntu-latest - container: - image: rippleci/clio_ci:latest - steps: - - uses: actions/checkout@v4 - - name: Run linter - id: run_linter - run: | - ./.githooks/check-docs - shell: bash - - build: - name: Build - needs: - - check_format - - check_docs strategy: fail-fast: false matrix: + os: [heavy] + conan_profile: [gcc, clang] + build_type: [Release, Debug] + container: ['{ "image": "ghcr.io/xrplf/clio-ci:latest" }'] + static: [true] + include: - - os: heavy - conan_profile: gcc - build_type: Release - container: '{ "image": "rippleci/clio_ci:latest" }' - code_coverage: false - static: true - - os: heavy - conan_profile: gcc - build_type: Debug - container: '{ "image": "rippleci/clio_ci:latest" }' - code_coverage: true - static: true - - os: heavy - conan_profile: clang - build_type: Release - container: '{ "image": "rippleci/clio_ci:latest" }' - code_coverage: false - static: true - - os: heavy - conan_profile: clang - build_type: Debug - container: '{ "image": "rippleci/clio_ci:latest" }' - code_coverage: false - static: true - os: macos15 + conan_profile: default_apple_clang build_type: Release - code_coverage: false + container: "" static: false - uses: ./.github/workflows/build_impl.yml + + uses: ./.github/workflows/build_and_test.yml with: runs_on: ${{ matrix.os }} container: ${{ matrix.container }} conan_profile: ${{ matrix.conan_profile }} build_type: ${{ matrix.build_type }} - code_coverage: ${{ matrix.code_coverage }} static: ${{ matrix.static }} - unit_tests: true - integration_tests: true - clio_server: true + run_unit_tests: true + run_integration_tests: false + upload_clio_server: true - test: - name: Run Tests - needs: build - strategy: - fail-fast: false - matrix: - include: - - os: heavy - conan_profile: gcc - build_type: Release - container: - image: rippleci/clio_ci:latest - - os: heavy - conan_profile: clang - build_type: Release - container: - image: rippleci/clio_ci:latest - - os: heavy - conan_profile: clang - build_type: Debug - container: - image: rippleci/clio_ci:latest - - os: macos15 - conan_profile: apple_clang_16 - build_type: Release - runs-on: ${{ matrix.os }} - container: ${{ matrix.container }} + code_coverage: + name: Run Code Coverage - steps: - - name: Clean workdir - if: ${{ runner.os == 'macOS' }} - uses: kuznetsss/workspace-cleanup@1.0 - - - uses: actions/download-artifact@v4 - with: - name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }} - - - name: Run clio_tests - run: | - chmod +x ./clio_tests - ./clio_tests + uses: ./.github/workflows/build_impl.yml + with: + runs_on: heavy + container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }' + conan_profile: gcc + build_type: Debug + disable_cache: false + code_coverage: true + static: true + upload_clio_server: false + targets: all + sanitizer: "false" + analyze_build_time: false + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} check_config: name: Check Config Description - needs: build + needs: build-and-test runs-on: heavy - container: - image: rippleci/clio_ci:latest + container: + image: ghcr.io/xrplf/clio-ci:latest + steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: name: clio_server_Linux_Release_gcc + - name: Compare Config Description shell: bash run: | repoConfigFile=docs/config-description.md - if ! [ -f ${repoConfigFile} ]; then + if ! [ -f "${repoConfigFile}" ]; then echo "Config Description markdown file is missing in docs folder" exit 1 fi chmod +x ./clio_server configDescriptionFile=config_description_new.md - ./clio_server -d ${configDescriptionFile} + ./clio_server -d "${configDescriptionFile}" - configDescriptionHash=$(sha256sum ${configDescriptionFile} | cut -d' ' -f1) - repoConfigHash=$(sha256sum ${repoConfigFile} | cut -d' ' -f1) + configDescriptionHash=$(sha256sum "${configDescriptionFile}" | cut -d' ' -f1) + repoConfigHash=$(sha256sum "${repoConfigFile}" | cut -d' ' -f1) - if [ ${configDescriptionHash} != ${repoConfigHash} ]; then + if [ "${configDescriptionHash}" != "${repoConfigHash}" ]; then echo "Markdown file is not up to date" diff -u "${repoConfigFile}" "${configDescriptionFile}" - rm -f ${configDescriptionFile} + rm -f "${configDescriptionFile}" exit 1 fi - rm -f ${configDescriptionFile} + rm -f "${configDescriptionFile}" exit 0 - - - - - diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 000000000..6a73e62f6 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,92 @@ +name: Reusable build and test + +on: + workflow_call: + inputs: + runs_on: + description: Runner to run the job on + required: true + type: string + + container: + description: "The container object as a JSON string (leave empty to run natively)" + required: true + type: string + + conan_profile: + description: Conan profile to use + required: true + type: string + + build_type: + description: Build type + required: true + type: string + + disable_cache: + description: Whether ccache and conan cache should be disabled + required: false + type: boolean + default: false + + static: + description: Whether to build static binaries + required: true + type: boolean + default: true + + run_unit_tests: + description: Whether to run unit tests + required: true + type: boolean + + run_integration_tests: + description: Whether to run integration tests + required: true + type: boolean + default: false + + upload_clio_server: + description: Whether to upload clio_server + required: true + type: boolean + + targets: + description: Space-separated build target names + required: false + type: string + default: all + + sanitizer: + description: Sanitizer to use + required: false + type: string + default: "false" + +jobs: + build: + uses: ./.github/workflows/build_impl.yml + with: + runs_on: ${{ inputs.runs_on }} + container: ${{ inputs.container }} + conan_profile: ${{ inputs.conan_profile }} + build_type: ${{ inputs.build_type }} + disable_cache: ${{ inputs.disable_cache }} + code_coverage: false + static: ${{ inputs.static }} + upload_clio_server: ${{ inputs.upload_clio_server }} + targets: ${{ inputs.targets }} + sanitizer: ${{ inputs.sanitizer }} + analyze_build_time: false + + test: + needs: build + uses: ./.github/workflows/test_impl.yml + with: + runs_on: ${{ inputs.runs_on }} + container: ${{ inputs.container }} + conan_profile: ${{ inputs.conan_profile }} + build_type: ${{ inputs.build_type }} + run_unit_tests: ${{ inputs.run_unit_tests }} + run_integration_tests: ${{ inputs.run_integration_tests }} + sanitizer: ${{ inputs.sanitizer }} diff --git a/.github/workflows/build_clio_docker_image.yml b/.github/workflows/build_clio_docker_image.yml index 07c6ba321..d4a86d090 100644 --- a/.github/workflows/build_clio_docker_image.yml +++ b/.github/workflows/build_clio_docker_image.yml @@ -1,4 +1,5 @@ name: Build and publish Clio docker image + on: workflow_call: inputs: @@ -41,6 +42,7 @@ jobs: build_and_publish_image: name: Build and publish image runs-on: ubuntu-latest + steps: - uses: actions/checkout@v4 @@ -55,7 +57,7 @@ jobs: if: ${{ inputs.clio_server_binary_url != null }} shell: bash run: | - wget ${{inputs.clio_server_binary_url}} -P ./docker/clio/artifact/ + wget "${{inputs.clio_server_binary_url}}" -P ./docker/clio/artifact/ if [ "$(sha256sum ./docker/clio/clio_server | awk '{print $1}')" != "${{inputs.binary_sha256}}" ]; then echo "Binary sha256 sum doesn't match" exit 1 @@ -87,7 +89,10 @@ jobs: DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - image_name: rippleci/clio + images: | + rippleci/clio + ghcr.io/xrplf/clio + dockerhub_repo: rippleci/clio push_image: ${{ inputs.publish_image }} directory: docker/clio tags: ${{ inputs.tags }} diff --git a/.github/workflows/build_impl.yml b/.github/workflows/build_impl.yml index db4a7edd0..f3471ec79 100644 --- a/.github/workflows/build_impl.yml +++ b/.github/workflows/build_impl.yml @@ -1,4 +1,5 @@ name: Reusable build + on: workflow_call: inputs: @@ -6,13 +7,11 @@ on: description: Runner to run the job on required: true type: string - default: heavy container: description: "The container object as a JSON string (leave empty to run natively)" required: true type: string - default: "" conan_profile: description: Conan profile to use @@ -28,60 +27,51 @@ on: description: Whether ccache and conan cache should be disabled required: false type: boolean - default: false code_coverage: description: Whether to enable code coverage required: true type: boolean - default: false static: description: Whether to build static binaries required: true type: boolean - default: true - unit_tests: - description: Whether to run unit tests + upload_clio_server: + description: Whether to upload clio_server required: true type: boolean - default: false - integration_tests: - description: Whether to run integration tests + targets: + description: Space-separated build target names required: true - type: boolean - default: false - - clio_server: - description: Whether to build clio_server - required: true - type: boolean - default: true - - target: - description: Build target name - required: false type: string - default: all sanitizer: description: Sanitizer to use - required: false + required: true type: string - default: 'false' + + analyze_build_time: + description: Whether to enable build time analysis + required: true + type: boolean + + secrets: + CODECOV_TOKEN: + required: false jobs: build: name: Build ${{ inputs.container != '' && 'in container' || 'natively' }} - runs-on: ${{ inputs.runs_on }} + runs-on: ${{ inputs.runs_on }} container: ${{ inputs.container != '' && fromJson(inputs.container) || null }} steps: - name: Clean workdir if: ${{ runner.os == 'macOS' }} - uses: kuznetsss/workspace-cleanup@1.0 + uses: kuznetsss/workspace-cleanup@80b9863b45562c148927c3d53621ef354e5ae7ce # v1.0 - uses: actions/checkout@v4 with: @@ -94,7 +84,6 @@ jobs: - name: Setup conan uses: ./.github/actions/setup_conan - id: conan with: conan_profile: ${{ inputs.conan_profile }} @@ -104,7 +93,7 @@ jobs: id: restore_cache with: conan_dir: ${{ env.CONAN_USER_HOME }}/.conan - conan_profile: ${{ steps.conan.outputs.conan_profile }} + conan_profile: ${{ inputs.conan_profile }} ccache_dir: ${{ env.CCACHE_DIR }} build_type: ${{ inputs.build_type }} code_coverage: ${{ inputs.code_coverage }} @@ -112,17 +101,33 @@ jobs: - name: Run conan and cmake uses: ./.github/actions/generate with: - conan_profile: ${{ steps.conan.outputs.conan_profile }} + conan_profile: ${{ inputs.conan_profile }} conan_cache_hit: ${{ !inputs.disable_cache && steps.restore_cache.outputs.conan_cache_hit }} build_type: ${{ inputs.build_type }} code_coverage: ${{ inputs.code_coverage }} static: ${{ inputs.static }} sanitizer: ${{ inputs.sanitizer }} + time_trace: ${{ inputs.analyze_build_time }} - name: Build Clio uses: ./.github/actions/build_clio with: - target: ${{ inputs.target }} + targets: ${{ inputs.targets }} + + - name: Show build time analyze report + if: ${{ inputs.analyze_build_time }} + run: | + ClangBuildAnalyzer --all build/ build_time_report.bin + ClangBuildAnalyzer --analyze build_time_report.bin > build_time_report.txt + cat build_time_report.txt + shell: bash + + - name: Upload build time analyze report + if: ${{ inputs.analyze_build_time }} + uses: actions/upload-artifact@v4 + with: + name: build_time_report_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} + path: build_time_report.txt - name: Show ccache's statistics if: ${{ !inputs.disable_cache }} @@ -135,32 +140,32 @@ jobs: cat /tmp/ccache.stats - name: Strip unit_tests - if: ${{ inputs.unit_tests && !inputs.code_coverage && inputs.sanitizer == 'false' }} + if: inputs.sanitizer == 'false' && !inputs.code_coverage && !inputs.analyze_build_time run: strip build/clio_tests - + - name: Strip integration_tests - if: ${{ inputs.integration_tests && !inputs.code_coverage }} + if: inputs.sanitizer == 'false' && !inputs.code_coverage && !inputs.analyze_build_time run: strip build/clio_integration_tests - name: Upload clio_server - if: ${{ inputs.clio_server }} + if: inputs.upload_clio_server && !inputs.code_coverage && !inputs.analyze_build_time uses: actions/upload-artifact@v4 with: - name: clio_server_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }} + name: clio_server_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} path: build/clio_server - + - name: Upload clio_tests - if: ${{ inputs.unit_tests && !inputs.code_coverage }} + if: ${{ !inputs.code_coverage && !inputs.analyze_build_time }} uses: actions/upload-artifact@v4 with: - name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }} + name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} path: build/clio_tests - + - name: Upload clio_integration_tests - if: ${{ inputs.integration_tests && !inputs.code_coverage }} + if: ${{ !inputs.code_coverage && !inputs.analyze_build_time }} uses: actions/upload-artifact@v4 with: - name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ steps.conan.outputs.conan_profile }} + name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} path: build/clio_integration_tests - name: Save cache @@ -175,16 +180,25 @@ jobs: ccache_cache_miss_rate: ${{ steps.ccache_stats.outputs.miss_rate }} build_type: ${{ inputs.build_type }} code_coverage: ${{ inputs.code_coverage }} - conan_profile: ${{ steps.conan.outputs.conan_profile }} + conan_profile: ${{ inputs.conan_profile }} - # TODO: This is not a part of build process but it is the easiest way to do it here. - # It will be refactored in https://github.com/XRPLF/clio/issues/1075 + # This is run as part of the build job, because it requires the following: + # - source code + # - generated source code (Build.cpp) + # - conan packages + # - .gcno files in build directory + # + # It's all available in the build job, but not in the test job - name: Run code coverage if: ${{ inputs.code_coverage }} uses: ./.github/actions/code_coverage + # `codecov/codecov-action` will rerun `gcov` if it's available and build directory is present + # To prevent this from happening, we run this action in a separate workflow + # + # More info: https://github.com/XRPLF/clio/pull/2066 upload_coverage_report: - if: ${{ inputs.code_coverage }} + if: ${{ inputs.code_coverage }} name: Codecov needs: build uses: ./.github/workflows/upload_coverage_report.yml diff --git a/.github/workflows/check_libxrpl.yml b/.github/workflows/check_libxrpl.yml index 50f0eb46d..a3e5e8fb7 100644 --- a/.github/workflows/check_libxrpl.yml +++ b/.github/workflows/check_libxrpl.yml @@ -1,19 +1,28 @@ name: Check new libXRPL + on: repository_dispatch: types: [check_libxrpl] +concurrency: + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CONAN_PROFILE: gcc + jobs: build: name: Build Clio / `libXRPL ${{ github.event.client_payload.version }}` runs-on: [self-hosted, heavy] container: - image: rippleci/clio_ci:latest + image: ghcr.io/xrplf/clio-ci:latest steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 + fetch-depth: 0 - name: Update libXRPL version requirement shell: bash @@ -23,18 +32,17 @@ jobs: - name: Prepare runner uses: ./.github/actions/prepare_runner with: - disable_ccache: true + disable_ccache: true - name: Setup conan uses: ./.github/actions/setup_conan - id: conan with: - conan_profile: gcc + conan_profile: ${{ env.CONAN_PROFILE }} - name: Run conan and cmake uses: ./.github/actions/generate with: - conan_profile: ${{ steps.conan.outputs.conan_profile }} + conan_profile: ${{ env.CONAN_PROFILE }} conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }} build_type: Release @@ -55,7 +63,7 @@ jobs: needs: build runs-on: [self-hosted, heavy] container: - image: rippleci/clio_ci:latest + image: ghcr.io/xrplf/clio-ci:latest steps: - uses: actions/download-artifact@v4 @@ -72,9 +80,11 @@ jobs: needs: [build, run_tests] if: ${{ always() && contains(needs.*.result, 'failure') }} runs-on: ubuntu-latest + permissions: contents: write issues: write + steps: - uses: actions/checkout@v4 @@ -83,8 +93,8 @@ jobs: env: GH_TOKEN: ${{ github.token }} with: - labels: 'compatibility,bug' - title: 'Proposed libXRPL check failed' + labels: "compatibility,bug" + title: "Proposed libXRPL check failed" body: > Clio build or tests failed against `libXRPL ${{ github.event.client_payload.version }}`. diff --git a/.github/workflows/check_pr_title.yml b/.github/workflows/check_pr_title.yml index 5e6f86854..8818e7ab7 100644 --- a/.github/workflows/check_pr_title.yml +++ b/.github/workflows/check_pr_title.yml @@ -1,4 +1,5 @@ name: Check PR title + on: pull_request: types: [opened, edited, reopened, synchronize] @@ -7,12 +8,10 @@ on: jobs: check_title: runs-on: ubuntu-latest - # permissions: - # pull-requests: write + steps: - - uses: ytanikin/PRConventionalCommits@1.3.0 + - uses: ytanikin/pr-conventional-commits@8267db1bacc237419f9ed0228bb9d94e94271a1d # v1.4.1 with: task_types: '["build","feat","fix","docs","test","ci","style","refactor","perf","chore"]' add_label: false - # Turned off labelling because it leads to an error, see https://github.com/ytanikin/PRConventionalCommits/issues/19 - # custom_labels: '{"build":"build", "feat":"enhancement", "fix":"bug", "docs":"documentation", "test":"testability", "ci":"ci", "style":"refactoring", "refactor":"refactoring", "perf":"performance", "chore":"tooling"}' + custom_labels: '{"build":"build", "feat":"enhancement", "fix":"bug", "docs":"documentation", "test":"testability", "ci":"ci", "style":"refactoring", "refactor":"refactoring", "perf":"performance", "chore":"tooling"}' diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index e251aff1d..00646dbec 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -1,4 +1,5 @@ name: Clang-tidy check + on: schedule: - cron: "0 9 * * 1-5" @@ -6,15 +7,24 @@ on: pull_request: branches: [develop] paths: - - .clang_tidy - .github/workflows/clang-tidy.yml - workflow_call: + + - .clang_tidy + +concurrency: + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CONAN_PROFILE: clang jobs: clang_tidy: runs-on: heavy container: - image: rippleci/clio_ci:latest + image: ghcr.io/xrplf/clio-ci:latest + permissions: contents: write issues: write @@ -32,9 +42,8 @@ jobs: - name: Setup conan uses: ./.github/actions/setup_conan - id: conan with: - conan_profile: clang + conan_profile: ${{ env.CONAN_PROFILE }} - name: Restore cache uses: ./.github/actions/restore_cache @@ -42,12 +51,12 @@ jobs: with: conan_dir: ${{ env.CONAN_USER_HOME }}/.conan ccache_dir: ${{ env.CCACHE_DIR }} - conan_profile: ${{ steps.conan.outputs.conan_profile }} + conan_profile: ${{ env.CONAN_PROFILE }} - name: Run conan and cmake uses: ./.github/actions/generate with: - conan_profile: ${{ steps.conan.outputs.conan_profile }} + conan_profile: ${{ env.CONAN_PROFILE }} conan_cache_hit: ${{ steps.restore_cache.outputs.conan_cache_hit }} build_type: Release @@ -60,13 +69,14 @@ jobs: shell: bash id: run_clang_tidy run: | - run-clang-tidy-19 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt + run-clang-tidy-19 -p build -j "${{ steps.number_of_threads.outputs.threads_number }}" -fix -quiet 1>output.txt - - name: Check format + - name: Fix local includes and clang-format style if: ${{ steps.run_clang_tidy.outcome != 'success' }} - continue-on-error: true shell: bash - run: ./.githooks/check-format + run: | + pre-commit run --all-files fix-local-includes || true + pre-commit run --all-files clang-format || true - name: Print issues found if: ${{ steps.run_clang_tidy.outcome != 'success' }} @@ -77,20 +87,20 @@ jobs: rm output.txt - name: Create an issue - if: ${{ steps.run_clang_tidy.outcome != 'success' }} + if: ${{ steps.run_clang_tidy.outcome != 'success' && github.event_name != 'pull_request' }} id: create_issue uses: ./.github/actions/create_issue env: GH_TOKEN: ${{ github.token }} with: - title: 'Clang-tidy found bugs in code 🐛' + title: "Clang-tidy found bugs in code 🐛" body: > Clang-tidy found issues in the code: List of the issues found: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/ - - uses: crazy-max/ghaction-import-gpg@v6 - if: ${{ steps.run_clang_tidy.outcome != 'success' }} + - uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec # v6.3.0 + if: ${{ steps.run_clang_tidy.outcome != 'success' && github.event_name != 'pull_request' }} with: gpg_private_key: ${{ secrets.ACTIONS_GPG_PRIVATE_KEY }} passphrase: ${{ secrets.ACTIONS_GPG_PASSPHRASE }} @@ -98,8 +108,8 @@ jobs: git_commit_gpgsign: true - name: Create PR with fixes - if: ${{ steps.run_clang_tidy.outcome != 'success' }} - uses: peter-evans/create-pull-request@v7 + if: ${{ steps.run_clang_tidy.outcome != 'success' && github.event_name != 'pull_request' }} + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 env: GH_REPO: ${{ github.repository }} GH_TOKEN: ${{ github.token }} @@ -111,7 +121,7 @@ jobs: delete-branch: true title: "style: clang-tidy auto fixes" body: "Fixes #${{ steps.create_issue.outputs.created_issue_id }}. Please review and commit clang-tidy fixes." - reviewers: "cindyyan317,godexsoft,kuznetsss" + reviewers: "godexsoft,kuznetsss,PeterChen13579,mathbunnyru" - name: Fail the job if: ${{ steps.run_clang_tidy.outcome != 'success' }} diff --git a/.github/workflows/clang-tidy_on_fix_merged.yml b/.github/workflows/clang-tidy_on_fix_merged.yml index e78fef591..fc0b37bef 100644 --- a/.github/workflows/clang-tidy_on_fix_merged.yml +++ b/.github/workflows/clang-tidy_on_fix_merged.yml @@ -1,4 +1,5 @@ name: Restart clang-tidy workflow + on: push: branches: [develop] @@ -17,8 +18,8 @@ jobs: id: check shell: bash run: | - passed=$(if [[ $(git log -1 --pretty=format:%s | grep 'style: clang-tidy auto fixes') ]]; then echo 'true' ; else echo 'false' ; fi) - echo "passed=$passed" >> $GITHUB_OUTPUT + passed=$(if [[ "$(git log -1 --pretty=format:%s | grep 'style: clang-tidy auto fixes')" ]]; then echo 'true' ; else echo 'false' ; fi) + echo "passed=\"$passed\"" >> $GITHUB_OUTPUT - name: Run clang-tidy workflow if: ${{ contains(steps.check.outputs.passed, 'true') }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d4b9a80a3..b9dc988d0 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,4 +1,5 @@ name: Documentation + on: push: branches: [develop] @@ -10,7 +11,8 @@ permissions: id-token: write concurrency: - group: "pages" + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -21,18 +23,19 @@ jobs: runs-on: ubuntu-latest continue-on-error: true container: - image: rippleci/clio_ci:latest + image: ghcr.io/xrplf/clio-ci:latest + steps: - name: Checkout uses: actions/checkout@v4 - with: + with: lfs: true - name: Build docs run: | mkdir -p build_docs && cd build_docs cmake ../docs && cmake --build . --target docs - + - name: Setup Pages uses: actions/configure-pages@v5 @@ -41,7 +44,7 @@ jobs: with: path: build_docs/html name: docs-develop - + - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v4 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 12d215dca..387b83524 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,167 +1,123 @@ name: Nightly release + on: schedule: - - cron: '0 8 * * 1-5' + - cron: "0 8 * * 1-5" workflow_dispatch: pull_request: paths: - - '.github/workflows/nightly.yml' - - '.github/workflows/build_clio_docker_image.yml' + - .github/workflows/nightly.yml + + - .github/workflows/release_impl.yml + - .github/workflows/build_and_test.yml + - .github/workflows/build_impl.yml + - .github/workflows/build_clio_docker_image.yml + + - ".github/actions/**" + - "!.github/actions/code_coverage/**" + +concurrency: + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - build: - name: Build clio + build-and-test: + name: Build and Test + strategy: fail-fast: false matrix: include: - os: macos15 + conan_profile: default_apple_clang build_type: Release static: false - os: heavy + conan_profile: gcc build_type: Release static: true - container: '{ "image": "rippleci/clio_ci:latest" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }' - os: heavy + conan_profile: gcc build_type: Debug static: true - container: '{ "image": "rippleci/clio_ci:latest" }' + container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }' + + uses: ./.github/workflows/build_and_test.yml + with: + runs_on: ${{ matrix.os }} + container: ${{ matrix.container }} + conan_profile: ${{ matrix.conan_profile }} + build_type: ${{ matrix.build_type }} + static: ${{ matrix.static }} + run_unit_tests: true + run_integration_tests: true + upload_clio_server: true + disable_cache: true + + analyze_build_time: + name: Analyze Build Time + + strategy: + fail-fast: false + matrix: + include: + # TODO: Enable when we have at least ubuntu 22.04 + # as ClangBuildAnalyzer requires relatively modern glibc + # + # - os: heavy + # conan_profile: clang + # container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }' + # static: true + - os: macos15 + conan_profile: default_apple_clang + container: "" + static: false uses: ./.github/workflows/build_impl.yml with: runs_on: ${{ matrix.os }} container: ${{ matrix.container }} - conan_profile: gcc - build_type: ${{ matrix.build_type }} + conan_profile: ${{ matrix.conan_profile }} + build_type: Release + disable_cache: true code_coverage: false static: ${{ matrix.static }} - unit_tests: true - integration_tests: true - clio_server: true - disable_cache: true - - run_tests: - needs: build - strategy: - fail-fast: false - matrix: - include: - - os: macos15 - conan_profile: apple_clang_16 - build_type: Release - integration_tests: false - - os: heavy - conan_profile: gcc - build_type: Release - container: - image: rippleci/clio_ci:latest - integration_tests: true - - os: heavy - conan_profile: gcc - build_type: Debug - container: - image: rippleci/clio_ci:latest - integration_tests: true - runs-on: [self-hosted, "${{ matrix.os }}"] - container: ${{ matrix.container }} - - services: - scylladb: - image: ${{ (matrix.integration_tests) && 'scylladb/scylla' || '' }} - options: >- - --health-cmd "cqlsh -e 'describe cluster'" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - name: Clean workdir - if: ${{ runner.os == 'macOS' }} - uses: kuznetsss/workspace-cleanup@1.0 - - - uses: actions/download-artifact@v4 - with: - name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }} - - - name: Run clio_tests - run: | - chmod +x ./clio_tests - ./clio_tests - - - uses: actions/download-artifact@v4 - with: - name: clio_integration_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }} - - # To be enabled back once docker in mac runner arrives - # https://github.com/XRPLF/clio/issues/1400 - - name: Run clio_integration_tests - if: matrix.integration_tests - run: | - chmod +x ./clio_integration_tests - ./clio_integration_tests --backend_host=scylladb + upload_clio_server: false + targets: all + sanitizer: "false" + analyze_build_time: true nightly_release: - if: ${{ github.event_name != 'pull_request' }} - needs: run_tests - runs-on: ubuntu-latest - env: - GH_REPO: ${{ github.repository }} - GH_TOKEN: ${{ github.token }} - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - - uses: actions/download-artifact@v4 - with: - path: nightly_release - pattern: clio_server_* - - - name: Prepare files - shell: bash - run: | - cp ${{ github.workspace }}/.github/workflows/nightly_notes.md "${RUNNER_TEMP}/nightly_notes.md" - cd nightly_release - for d in $(ls); do - archive_name=$(ls $d) - mv ${d}/${archive_name} ./ - rm -r $d - sha256sum ./$archive_name > ./${archive_name}.sha256sum - cat ./$archive_name.sha256sum >> "${RUNNER_TEMP}/nightly_notes.md" - done - echo '```' >> "${RUNNER_TEMP}/nightly_notes.md" - - - name: Remove current nightly release and nightly tag - shell: bash - run: | - gh release delete nightly --yes || true - git push origin :nightly || true - - - name: Publish nightly release - shell: bash - run: | - gh release create nightly --prerelease --title "Clio development (nightly) build" \ - --target $GITHUB_SHA --notes-file "${RUNNER_TEMP}/nightly_notes.md" \ - ./nightly_release/clio_server* + needs: build-and-test + uses: ./.github/workflows/release_impl.yml + with: + overwrite_release: true + title: "Clio development (nightly) build" + version: nightly + notes_header_file: nightly_notes.md build_and_publish_docker_image: uses: ./.github/workflows/build_clio_docker_image.yml - needs: run_tests + needs: build-and-test secrets: inherit with: tags: | - type=raw,value=nightly - type=raw,value=${{ github.sha }} + type=raw,value=nightly + type=raw,value=${{ github.sha }} artifact_name: clio_server_Linux_Release_gcc strip_binary: true publish_image: ${{ github.event_name != 'pull_request' }} create_issue_on_failure: - needs: [build, run_tests, nightly_release, build_and_publish_docker_image] + needs: [build-and-test, nightly_release, build_and_publish_docker_image] if: ${{ always() && contains(needs.*.result, 'failure') && github.event_name != 'pull_request' }} runs-on: ubuntu-latest + permissions: contents: write issues: write + steps: - uses: actions/checkout@v4 @@ -170,7 +126,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} with: - title: 'Nightly release failed 🌙' + title: "Nightly release failed 🌙" body: > Nightly release failed: diff --git a/.github/workflows/nightly_notes.md b/.github/workflows/nightly_notes.md index 38deb4302..8f7777888 100644 --- a/.github/workflows/nightly_notes.md +++ b/.github/workflows/nightly_notes.md @@ -1,6 +1,7 @@ +# Release notes + > **Note:** Please remember that this is a development release and it is not recommended for production use. -Changelog (including previous releases): https://github.com/XRPLF/clio/commits/nightly +Changelog (including previous releases): ## SHA256 checksums -``` diff --git a/.github/workflows/pre-commit-autoupdate.yml b/.github/workflows/pre-commit-autoupdate.yml new file mode 100644 index 000000000..a45602896 --- /dev/null +++ b/.github/workflows/pre-commit-autoupdate.yml @@ -0,0 +1,39 @@ +name: Pre-commit auto-update + +on: + # every first day of the month + schedule: + - cron: "0 0 1 * *" + # on demand + workflow_dispatch: + +jobs: + auto-update: + runs-on: ubuntu-latest + + permissions: + contents: write + pull-requests: write + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: 3.x + + - run: pip install pre-commit + - run: pre-commit autoupdate --freeze + - run: pre-commit run --all-files || true + + - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + if: always() + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + with: + branch: update/pre-commit-hooks + title: Update pre-commit hooks + commit-message: "style: update pre-commit hooks" + body: Update versions of pre-commit hooks to latest version. + reviewers: "godexsoft,kuznetsss,PeterChen13579,mathbunnyru" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 000000000..aab132c9d --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,28 @@ +name: Run pre-commit hooks + +on: + pull_request: + push: + branches: + - develop + workflow_dispatch: + +jobs: + run-hooks: + runs-on: heavy + container: + image: ghcr.io/xrplf/clio-ci:latest + + steps: + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Prepare runner + uses: ./.github/actions/prepare_runner + with: + disable_ccache: true + + - name: Run pre-commit ✅ + run: pre-commit run --all-files diff --git a/.github/workflows/release_impl.yml b/.github/workflows/release_impl.yml new file mode 100644 index 000000000..08b8b666a --- /dev/null +++ b/.github/workflows/release_impl.yml @@ -0,0 +1,78 @@ +name: Make release + +on: + workflow_call: + inputs: + overwrite_release: + description: "Overwrite the current release and tag" + required: true + type: boolean + + title: + description: "Release title" + required: true + type: string + + version: + description: "Release version" + required: true + type: string + + notes_header_file: + description: "Release notes header file" + required: true + type: string + +jobs: + release: + runs-on: ubuntu-latest + env: + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ github.token }} + + permissions: + contents: write + + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + with: + path: release_artifacts + pattern: clio_server_* + + - name: Prepare files + shell: bash + working-directory: release_artifacts + run: | + cp ${{ github.workspace }}/.github/workflows/${{ inputs.notes_header_file }} "${RUNNER_TEMP}/release_notes.md" + echo '' >> "${RUNNER_TEMP}/release_notes.md" + echo '```' >> "${RUNNER_TEMP}/release_notes.md" + + for d in $(ls); do + archive_name=$(ls $d) + mv ${d}/${archive_name} ./ + rm -r $d + sha256sum ./$archive_name > ./${archive_name}.sha256sum + cat ./$archive_name.sha256sum >> "${RUNNER_TEMP}/release_notes.md" + done + + echo '```' >> "${RUNNER_TEMP}/release_notes.md" + + - name: Remove current release and tag + if: ${{ github.event_name != 'pull_request' && inputs.overwrite_release }} + shell: bash + run: | + gh release delete ${{ inputs.version }} --yes || true + git push origin :${{ inputs.version }} || true + + - name: Publish release + if: ${{ github.event_name != 'pull_request' }} + shell: bash + run: | + gh release create ${{ inputs.version }} \ + ${{ inputs.overwrite_release && '--prerelease' || '' }} \ + --title "${{ inputs.title }}" \ + --target $GITHUB_SHA \ + --notes-file "${RUNNER_TEMP}/release_notes.md" \ + ./release_artifacts/clio_server* diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index ce0f2bb33..2b7116dca 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -1,15 +1,37 @@ name: Run tests with sanitizers + on: schedule: - cron: "0 4 * * 1-5" workflow_dispatch: pull_request: paths: - - '.github/workflows/sanitizers.yml' + - .github/workflows/sanitizers.yml + + - .github/workflows/build_and_test.yml + - .github/workflows/build_impl.yml + - .github/workflows/test_impl.yml + + - ".github/actions/**" + - "!.github/actions/build_docker_image/**" + - "!.github/actions/create_issue/**" + - .github/scripts/execute-tests-under-sanitizer + + - CMakeLists.txt + - "cmake/**" + # We don't run sanitizer on code change, because it takes too long + # - "src/**" + # - "tests/**" + +concurrency: + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - build: - name: Build clio tests + build-and-test: + name: Build and Test + strategy: fail-fast: false matrix: @@ -18,89 +40,19 @@ jobs: compiler: gcc - sanitizer: asan compiler: gcc - # - sanitizer: ubsan # todo: enable when heavy runners are available - # compiler: gcc - uses: ./.github/workflows/build_impl.yml + - sanitizer: ubsan + compiler: gcc + + uses: ./.github/workflows/build_and_test.yml with: - runs_on: ubuntu-latest # todo: change to heavy - container: '{ "image": "rippleci/clio_ci:latest" }' + runs_on: heavy + container: '{ "image": "ghcr.io/xrplf/clio-ci:latest" }' disable_cache: true conan_profile: ${{ matrix.compiler }}.${{ matrix.sanitizer }} build_type: Release - code_coverage: false static: false - unit_tests: true - integration_tests: false - clio_server: false - target: clio_tests + run_unit_tests: true + run_integration_tests: false + upload_clio_server: false + targets: clio_tests clio_integration_tests sanitizer: ${{ matrix.sanitizer }} - - # consider combining this with the previous matrix instead - run_tests: - needs: build - strategy: - fail-fast: false - matrix: - include: - - sanitizer: tsan - compiler: gcc - - sanitizer: asan - compiler: gcc - # - sanitizer: ubsan # todo: enable when heavy runners are available - # compiler: gcc - runs-on: ubuntu-latest # todo: change to heavy - container: - image: rippleci/clio_ci:latest - permissions: - contents: write - issues: write - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: actions/download-artifact@v4 - with: - name: clio_tests_${{ runner.os }}_Release_${{ matrix.compiler }}.${{ matrix.sanitizer }} - - - name: Run clio_tests [${{ matrix.compiler }} / ${{ matrix.sanitizer }}] - shell: bash - run: | - chmod +x ./clio_tests - ./.github/scripts/execute-tests-under-sanitizer ./clio_tests - - - name: Check for sanitizer report - shell: bash - id: check_report - run: | - if ls .sanitizer-report/* 1> /dev/null 2>&1; then - echo "found_report=true" >> $GITHUB_OUTPUT - else - echo "found_report=false" >> $GITHUB_OUTPUT - fi - - - name: Upload report - if: ${{ steps.check_report.outputs.found_report == 'true' }} - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.compiler }}_${{ matrix.sanitizer }}_report - path: .sanitizer-report/* - include-hidden-files: true - - # - # todo: enable when we have fixed all currently existing issues from sanitizers - # - # - name: Create an issue - # if: ${{ steps.check_report.outputs.found_report == 'true' }} - # uses: ./.github/actions/create_issue - # env: - # GH_TOKEN: ${{ github.token }} - # with: - # labels: 'bug' - # title: '[${{ matrix.sanitizer }}/${{ matrix.compiler }}] reported issues' - # body: > - # Clio tests failed one or more sanitizer checks when built with ${{ matrix.compiler }}`. - - # Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/ - # Reports are available as artifacts. diff --git a/.github/workflows/test_impl.yml b/.github/workflows/test_impl.yml new file mode 100644 index 000000000..4fe50c200 --- /dev/null +++ b/.github/workflows/test_impl.yml @@ -0,0 +1,165 @@ +name: Reusable test + +on: + workflow_call: + inputs: + runs_on: + description: Runner to run the job on + required: true + type: string + + container: + description: "The container object as a JSON string (leave empty to run natively)" + required: true + type: string + + conan_profile: + description: Conan profile to use + required: true + type: string + + build_type: + description: Build type + required: true + type: string + + run_unit_tests: + description: Whether to run unit tests + required: true + type: boolean + + run_integration_tests: + description: Whether to run integration tests + required: true + type: boolean + + sanitizer: + description: Sanitizer to use + required: true + type: string + +jobs: + unit_tests: + name: Unit testing ${{ inputs.container != '' && 'in container' || 'natively' }} + runs-on: ${{ inputs.runs_on }} + container: ${{ inputs.container != '' && fromJson(inputs.container) || null }} + + if: inputs.run_unit_tests + + env: + # TODO: remove when we have fixed all currently existing issues from sanitizers + SANITIZER_IGNORE_ERRORS: ${{ inputs.sanitizer != 'false' && inputs.sanitizer != 'ubsan' }} + + steps: + - name: Clean workdir + if: ${{ runner.os == 'macOS' }} + uses: kuznetsss/workspace-cleanup@80b9863b45562c148927c3d53621ef354e5ae7ce # v1.0 + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/download-artifact@v4 + with: + name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} + + - name: Make clio_tests executable + shell: bash + run: chmod +x ./clio_tests + + - name: Run clio_tests (regular) + if: env.SANITIZER_IGNORE_ERRORS == 'false' + run: ./clio_tests + + - name: Run clio_tests (sanitizer errors ignored) + if: env.SANITIZER_IGNORE_ERRORS == 'true' + run: ./.github/scripts/execute-tests-under-sanitizer ./clio_tests + + - name: Check for sanitizer report + if: env.SANITIZER_IGNORE_ERRORS == 'true' + shell: bash + id: check_report + run: | + if ls .sanitizer-report/* 1> /dev/null 2>&1; then + echo "found_report=true" >> $GITHUB_OUTPUT + else + echo "found_report=false" >> $GITHUB_OUTPUT + fi + + - name: Upload sanitizer report + if: env.SANITIZER_IGNORE_ERRORS == 'true' && steps.check_report.outputs.found_report == 'true' + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.conan_profile }}_report + path: .sanitizer-report/* + include-hidden-files: true + + - name: Create an issue + if: false && env.SANITIZER_IGNORE_ERRORS == 'true' && steps.check_report.outputs.found_report == 'true' + uses: ./.github/actions/create_issue + env: + GH_TOKEN: ${{ github.token }} + with: + labels: "bug" + title: "[${{ inputs.conan_profile }}] reported issues" + body: > + Clio tests failed one or more sanitizer checks when built with ${{ inputs.conan_profile }}`. + + Workflow: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/ + Reports are available as artifacts. + + integration_tests: + name: Integration testing ${{ inputs.container != '' && 'in container' || 'natively' }} + runs-on: ${{ inputs.runs_on }} + container: ${{ inputs.container != '' && fromJson(inputs.container) || null }} + + if: inputs.run_integration_tests + + services: + scylladb: + image: ${{ inputs.container != '' && 'scylladb/scylla' || '' }} + options: >- + --health-cmd "cqlsh -e 'describe cluster'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - name: Clean workdir + if: ${{ runner.os == 'macOS' }} + uses: kuznetsss/workspace-cleanup@80b9863b45562c148927c3d53621ef354e5ae7ce # v1.0 + + - name: Spin up scylladb + if: ${{ runner.os == 'macOS' }} + timeout-minutes: 3 + run: | + docker rm --force scylladb || true + docker run \ + --detach \ + --name scylladb \ + --health-cmd "cqlsh -e 'describe cluster'" \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + --publish 9042:9042 \ + --memory 16G \ + scylladb/scylla + + until [ "$(docker inspect -f '{{.State.Health.Status}}' scylladb)" == "healthy" ]; do + sleep 5 + done + + - uses: actions/download-artifact@v4 + with: + name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }} + + - name: Run clio_integration_tests + run: | + chmod +x ./clio_integration_tests + ./clio_integration_tests ${{ runner.os != 'macOS' && '--backend_host=scylladb' || '' }} + + - name: Show docker logs and stop scylladb + if: ${{ always() && runner.os == 'macOS' }} + run: | + docker logs scylladb + docker rm --force scylladb || true diff --git a/.github/workflows/update_docker_ci.yml b/.github/workflows/update_docker_ci.yml index 5c53b6ab6..1050b1417 100644 --- a/.github/workflows/update_docker_ci.yml +++ b/.github/workflows/update_docker_ci.yml @@ -1,22 +1,37 @@ name: Update CI docker image + on: pull_request: paths: - - 'docker/ci/**' - - 'docker/compilers/**' - .github/workflows/update_docker_ci.yml + + - ".github/actions/build_docker_image/**" + + - "docker/ci/**" + - "docker/compilers/**" push: branches: [develop] paths: - - 'docker/ci/**' # CI image must update when either its dockerfile changes - - 'docker/compilers/**' # or any compilers changed and were pushed by hand - .github/workflows/update_docker_ci.yml + + - ".github/actions/build_docker_image/**" + + # CI image must update when either its Dockerfile changes + # or any compilers changed and were pushed by hand + - "docker/ci/**" + - "docker/compilers/**" workflow_dispatch: +concurrency: + # Only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build_and_push: name: Build and push docker image runs-on: [self-hosted, heavy] + steps: - uses: actions/checkout@v4 - uses: ./.github/actions/build_docker_image @@ -25,7 +40,10 @@ jobs: DOCKERHUB_PW: ${{ secrets.DOCKERHUB_PW }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - image_name: rippleci/clio_ci + images: | + rippleci/clio_ci + ghcr.io/xrplf/clio-ci + dockerhub_repo: rippleci/clio_ci push_image: ${{ github.event_name != 'pull_request' }} directory: docker/ci tags: | diff --git a/.github/workflows/upload_coverage_report.yml b/.github/workflows/upload_coverage_report.yml index 1f9c46c73..c7e58061c 100644 --- a/.github/workflows/upload_coverage_report.yml +++ b/.github/workflows/upload_coverage_report.yml @@ -1,4 +1,5 @@ name: Upload report + on: workflow_dispatch: workflow_call: @@ -10,6 +11,7 @@ jobs: upload_report: name: Upload report runs-on: ubuntu-latest + steps: - uses: actions/checkout@v4 with: @@ -23,13 +25,9 @@ jobs: - name: Upload coverage report if: ${{ hashFiles('build/coverage_report.xml') != '' }} - uses: wandalen/wretry.action@v3.7.3 + uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3 with: - action: codecov/codecov-action@v4 - with: | - files: build/coverage_report.xml - fail_ci_if_error: false - verbose: true - token: ${{ secrets.CODECOV_TOKEN }} - attempt_limit: 5 - attempt_delay: 10000 + files: build/coverage_report.xml + fail_ci_if_error: true + verbose: true + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.hadolint.yml b/.hadolint.yml new file mode 100644 index 000000000..1d964becc --- /dev/null +++ b/.hadolint.yml @@ -0,0 +1,8 @@ +--- +ignored: + - DL3003 + - DL3008 + - DL3013 + - DL3015 + - DL3027 + - DL3047 diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 000000000..4bce02df8 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,6 @@ +# Default state for all rules +default: true + +# MD013/line-length - Line length +MD013: + line_length: 1000 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..8686de35e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,109 @@ +--- +# pre-commit is a tool to perform a predefined set of tasks manually and/or +# automatically before git commits are made. +# +# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level +# +# Common tasks +# +# - Run on all files: pre-commit run --all-files +# - Register git hooks: pre-commit install --hook-type pre-commit --hook-type pre-push +# +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + # `pre-commit sample-config` default hooks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b # frozen: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: end-of-file-fixer + exclude: ^docs/doxygen-awesome-theme/ + - id: trailing-whitespace + exclude: ^docs/doxygen-awesome-theme/ + + # Autoformat: YAML, JSON, Markdown, etc. + - repo: https://github.com/rbubley/mirrors-prettier + rev: 787fb9f542b140ba0b2aced38e6a3e68021647a3 # frozen: v3.5.3 + hooks: + - id: prettier + exclude: ^docs/doxygen-awesome-theme/ + + - repo: https://github.com/igorshubovych/markdownlint-cli + rev: 586c3ea3f51230da42bab657c6a32e9e66c364f0 # frozen: v0.44.0 + hooks: + - id: markdownlint-fix + exclude: LICENSE.md + + - repo: https://github.com/hadolint/hadolint + rev: c3dc18df7a501f02a560a2cc7ba3c69a85ca01d3 # frozen: v2.13.1-beta + hooks: + - id: hadolint-docker + # hadolint-docker is a special hook that runs hadolint in a Docker container + # Docker is not installed in the environment where pre-commit is run + stages: [manual] + entry: hadolint/hadolint:v2.12.1-beta hadolint + + - repo: https://github.com/codespell-project/codespell + rev: 63c8f8312b7559622c0d82815639671ae42132ac # frozen: v2.4.1 + hooks: + - id: codespell + args: + [ + --write-changes, + --ignore-words=pre-commit-hooks/codespell_ignore.txt, + ] + + # Running fix-local-includes before clang-format + # to ensure that the include order is correct. + - repo: local + hooks: + - id: fix-local-includes + name: Fix Local Includes + entry: pre-commit-hooks/fix-local-includes.sh + types: [c++] + language: script + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: f9a52e87b6cdcb01b0a62b8611d9ba9f2dad0067 # frozen: v19.1.7 + hooks: + - id: clang-format + args: [--style=file] + types: [c++] + + - repo: https://github.com/cheshirekow/cmake-format-precommit + rev: e2c2116d86a80e72e7146a06e68b7c228afc6319 # frozen: v0.6.13 + hooks: + - id: cmake-format + additional_dependencies: [PyYAML] + + - repo: local + hooks: + - id: check-no-h-files + name: No .h files + entry: There should be no .h files in this repository + language: fail + files: \.h$ + + - repo: local + hooks: + - id: gofmt + name: Go Format + entry: pre-commit-hooks/run-go-fmt.sh + types: [go] + language: golang + description: "Runs `gofmt`, requires golang" + - id: check-docs + name: Check Doxygen Documentation + entry: pre-commit-hooks/check-doxygen-docs.sh + types: [text] + language: script + pass_filenames: false + - id: verify-commits + name: Verify Commits + entry: pre-commit-hooks/verify-commits.sh + always_run: true + stages: [pre-push] + language: script + pass_filenames: false diff --git a/CMakeLists.txt b/CMakeLists.txt index 27b1c3d55..0e2c62f9f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ option(packaging "Create distribution packages" FALSE) option(lint "Run clang-tidy checks during compilation" FALSE) option(static "Statically linked Clio" FALSE) option(snapshot "Build snapshot tool" FALSE) +option(time_trace "Build using -ftime-trace to create compiler trace reports" FALSE) # ========================================================================== # set(san "" CACHE STRING "Add sanitizer instrumentation") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7be9bc78f..24cb3f885 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,39 +1,57 @@ # Contributing + Thank you for your interest in contributing to the `clio` project 🙏 +## Workflow + To contribute, please: + 1. Fork the repository under your own user. 2. Create a new branch on which to commit/push your changes. 3. Write and test your code. 4. Ensure that your code compiles with the provided build engine and update the provided build engine as part of your PR where needed and where appropriate. 5. Where applicable, write test cases for your code and include those in the relevant subfolder under `tests`. -6. Ensure your code passes automated checks (e.g. clang-format) +6. Ensure your code passes [automated checks](#pre-commit-hooks) 7. Squash your commits (i.e. rebase) into as few commits as is reasonable to describe your changes at a high level (typically a single commit for a small change). See below for more details. 8. Open a PR to the main repository onto the _develop_ branch, and follow the provided template. > **Note:** Please read the [Style guide](#style-guide). -## Install git hooks -Please run the following command in order to use git hooks that are helpful for `clio` development. +### `git lfs` hooks -``` bash -git config --local core.hooksPath .githooks +Install `git lfs` hooks using the following command: + +```bash +git lfs install ``` -## Git hooks dependencies -The pre-commit hook requires `clang-format >= 19.0.0` and `cmake-format` to be installed on your machine. -`clang-format` can be installed using `brew` on macOS and default package manager on Linux. -`cmake-format` can be installed using `pip`. -The hook will also attempt to automatically use `doxygen` to verify that everything public in the codebase is covered by doc comments. If `doxygen` is not installed, the hook will raise a warning suggesting to install `doxygen` for future commits. +> **Note:** You need to install Git LFS hooks before installing `pre-commit` hooks. -## Git commands -This sections offers a detailed look at the git commands you will need to use to get your PR submitted. +### `pre-commit` hooks + +To ensure code quality and style, we use [`pre-commit`](https://pre-commit.com/). + +Run the following command to enable `pre-commit` hooks that help with Clio development: + +```bash +pip3 install pre-commit +pre-commit install --hook-type pre-commit --hook-type pre-push +``` + +`pre-commit` takes care of running each tool in [`.pre-commit-config.yaml`](https://github.com/XRPLF/clio/blob/develop/.pre-commit-config.yaml) in a separate environment. + +`pre-commit` also attempts to automatically use Doxygen to verify that everything public in the codebase has doc comments. +If Doxygen is not installed, the hook issues a warning and recommends installing Doxygen for future commits. + +### Git commands + +This sections offers a detailed look at the git commands you will need to use to get your PR submitted. Please note that there are more than one way to do this and these commands are provided for your convenience. At this point it's assumed that you have already finished working on your feature/bug. > **Important:** Before you issue any of the commands below, please hit the `Sync fork` button and make sure your fork's `develop` branch is up-to-date with the main `clio` repository. -``` bash +```bash # Create a backup of your branch git branch _bk @@ -43,18 +61,20 @@ git pull origin develop git checkout git rebase -i develop ``` + For each commit in the list other than the first one, enter `s` to squash. After this is done, you will have the opportunity to write a message for the squashed commit. > **Hint:** Please use **imperative mood** in the commit message, and capitalize the first word. -``` bash +```bash # You should now have a single commit on top of a commit in `develop` git log ``` + > **Note:** If there are merge conflicts, please resolve them now. -``` bash +```bash # Use the same commit message as you did above git commit -m 'Your message' git rebase --continue @@ -62,27 +82,30 @@ git rebase --continue > **Important:** If you have no GPG keys set up, please follow [this tutorial](https://docs.github.com/en/authentication/managing-commit-signature-verification/adding-a-gpg-key-to-your-github-account) -``` bash +```bash # Sign the commit with your GPG key, and push your changes git commit --amend -S git push --force ``` -## Use ccache (optional) +### Use ccache (optional) + Clio uses `ccache` to speed up compilation. If you want to use it, please make sure it is installed on your machine. CMake will automatically detect it and use it if it is available. -## Opening a pull request +### Opening a pull request + When a pull request is open CI will perform checks on the new code. Title of the pull request and squashed commit should follow [conventional commits specification](https://www.conventionalcommits.org/en/v1.0.0/). -## Fixing issues found during code review +### Fixing issues found during code review + While your code is in review, it's possible that some changes will be requested by reviewer(s). This section describes the process of adding your fixes. We assume that you already made the required changes on your feature branch. -``` bash +```bash # Add the changed code git add @@ -94,62 +117,72 @@ git commit -S -m "[FOLD] Your commit message" git push ``` -## After code review +### After code review + When your PR is approved and ready to merge, use `Squash and merge`. The button for that is near the bottom of the PR's page on GitHub. > **Important:** Please leave the automatically-generated mention/link to the PR in the subject line **and** in the description field add `"Fix #ISSUE_ID"` (replacing `ISSUE_ID` with yours) if the PR fixes an issue. > **Note:** See [issues](https://github.com/XRPLF/clio/issues) to find the `ISSUE_ID` for the feature/bug you were working on. -# Style guide +## Style guide + This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent. -## Formatting -Code must conform to `clang-format` version 19, unless the result would be unreasonably difficult to read or maintain. -In most cases the pre-commit hook will take care of formatting and will fix any issues automatically. -To manually format your code, use `clang-format -i ` for C++ files and `cmake-format -i ` for CMake files. +### Formatting + +Code must conform to `clang-format`, unless the result is unreasonably difficult to read or maintain. +In most cases the `pre-commit` hook takes care of formatting and fixes any issues automatically. +To manually format your code, run `pre-commit run clang-format --files ` for C++ files, and `pre-commit run cmake-format --files ` for CMake files. + +### Documentation -## Documentation All public namespaces, classes and functions must be covered by doc (`doxygen`) comments. Everything that is not within a nested `impl` namespace is considered public. > **Note:** Keep in mind that this is enforced by Clio's CI and your build will fail if newly added public code lacks documentation. -## Avoid -* Proliferation of nearly identical code. -* Proliferation of new files and classes unless it improves readability or/and compilation time. -* Unmanaged memory allocation and raw pointers. -* Macros (unless they add significant value.) -* Lambda patterns (unless these add significant value.) -* CPU or architecture-specific code unless there is a good reason to include it, and where it is used guard it with macros and provide explanatory comments. -* Importing new libraries unless there is a very good reason to do so. +### Avoid -## Seek to -* Extend functionality of existing code rather than creating new code. -* Prefer readability over terseness where important logic is concerned. -* Inline functions that are not used or are not likely to be used elsewhere in the codebase. -* Use clear and self-explanatory names for functions, variables, structs and classes. -* Use TitleCase for classes, structs and filenames, camelCase for function and variable names, lower case for namespaces and folders. -* Provide as many comments as you feel that a competent programmer would need to understand what your code does. +- Proliferation of nearly identical code. +- Proliferation of new files and classes unless it improves readability or/and compilation time. +- Unmanaged memory allocation and raw pointers. +- Macros (unless they add significant value.) +- Lambda patterns (unless these add significant value.) +- CPU or architecture-specific code unless there is a good reason to include it, and where it is used guard it with macros and provide explanatory comments. +- Importing new libraries unless there is a very good reason to do so. + +### Seek to + +- Extend functionality of existing code rather than creating new code. +- Prefer readability over terseness where important logic is concerned. +- Inline functions that are not used or are not likely to be used elsewhere in the codebase. +- Use clear and self-explanatory names for functions, variables, structs and classes. +- Use TitleCase for classes, structs and filenames, camelCase for function and variable names, lower case for namespaces and folders. +- Provide as many comments as you feel that a competent programmer would need to understand what your code does. + +## Maintainers -# Maintainers Maintainers are ecosystem participants with elevated access to the repository. They are able to push new code, make decisions on when a release should be made, etc. -## Code Review +### Code Review + A PR must be reviewed and approved by at least one of the maintainers before it can be merged. -## Adding and Removing +### Adding and Removing + New maintainers can be proposed by two existing maintainers, subject to a vote by a quorum of the existing maintainers. A minimum of 50% support and a 50% participation is required. In the event of a tie vote, the addition of the new maintainer will be rejected. Existing maintainers can resign, or be subject to a vote for removal at the behest of two existing maintainers. A minimum of 60% agreement and 50% participation are required. The XRP Ledger Foundation will have the ability, for cause, to remove an existing maintainer without a vote. -## Existing Maintainers +### Existing Maintainers -* [cindyyan317](https://github.com/cindyyan317) (Ripple) -* [godexsoft](https://github.com/godexsoft) (Ripple) -* [kuznetsss](https://github.com/kuznetsss) (Ripple) -* [legleux](https://github.com/legleux) (Ripple) +- [godexsoft](https://github.com/godexsoft) (Ripple) +- [kuznetsss](https://github.com/kuznetsss) (Ripple) +- [legleux](https://github.com/legleux) (Ripple) +- [PeterChen13579](https://github.com/PeterChen13579) (Ripple) -## Honorable ex-Maintainers +### Honorable ex-Maintainers -* [cjcobb23](https://github.com/cjcobb23) (ex-Ripple) -* [natenichols](https://github.com/natenichols) (ex-Ripple) +- [cindyyan317](https://github.com/cindyyan317) (ex-Ripple) +- [cjcobb23](https://github.com/cjcobb23) (ex-Ripple) +- [natenichols](https://github.com/natenichols) (ex-Ripple) diff --git a/LICENSE.md b/LICENSE.md index cc0b4ae66..b86919710 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,8 +1,7 @@ ISC License -Copyright (c) 2022, the clio developers +Copyright (c) 2022, the clio developers Permission to use, copy, modify, and distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - diff --git a/README.md b/README.md index efe087d1b..d7629461e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Clio +# Clio [![Build status](https://github.com/XRPLF/clio/actions/workflows/build.yml/badge.svg?branch=develop)](https://github.com/XRPLF/clio/actions/workflows/build.yml?query=branch%3Adevelop) [![Nightly release status](https://github.com/XRPLF/clio/actions/workflows/nightly.yml/badge.svg?branch=develop)](https://github.com/XRPLF/clio/actions/workflows/nightly.yml?query=branch%3Adevelop) @@ -16,9 +16,9 @@ Multiple Clio nodes can share access to the same dataset, which allows for a hig Clio offers the full `rippled` API, with the caveat that Clio by default only returns validated data. This means that `ledger_index` defaults to `validated` instead of `current` for all requests. Other non-validated data, such as information about queued transactions, is also not returned. Clio retrieves data from a designated group of `rippled` nodes instead of connecting to the peer-to-peer network. -For requests that require access to the peer-to-peer network, such as `fee` or `submit`, Clio automatically forwards the request to a `rippled` node and propagates the response back to the client. To access non-validated data for *any* request, simply add `ledger_index: "current"` to the request, and Clio will forward the request to `rippled`. +For requests that require access to the peer-to-peer network, such as `fee` or `submit`, Clio automatically forwards the request to a `rippled` node and propagates the response back to the client. To access non-validated data for _any_ request, simply add `ledger_index: "current"` to the request, and Clio will forward the request to `rippled`. -> [!NOTE] +> [!NOTE] > Clio requires access to at least one `rippled` node, which can run on the same machine as Clio or separately. ## 📚 Learn more about Clio diff --git a/cmake/Settings.cmake b/cmake/Settings.cmake index 6ed745272..85fe7df93 100644 --- a/cmake/Settings.cmake +++ b/cmake/Settings.cmake @@ -26,7 +26,7 @@ set(COMPILER_FLAGS # TODO: Address these and others in https://github.com/XRPLF/clio/issues/1273 ) -# TODO: reenable when we change CI #884 if (is_gcc AND NOT lint) list(APPEND COMPILER_FLAGS -Wduplicated-branches +# TODO: re-enable when we change CI #884 if (is_gcc AND NOT lint) list(APPEND COMPILER_FLAGS -Wduplicated-branches # -Wduplicated-cond -Wlogical-op -Wuseless-cast ) endif () if (is_clang) @@ -70,4 +70,12 @@ endif () # See https://github.com/cpp-best-practices/cppbestpractices/blob/master/02-Use_the_Tools_Available.md#gcc--clang for # the flags description +if (time_trace) + if (is_clang OR is_appleclang) + list(APPEND COMPILER_FLAGS -ftime-trace) + else () + message(FATAL_ERROR "Clang or AppleClang is required to use `-ftime-trace`") + endif () +endif () + target_compile_options(clio_options INTERFACE ${COMPILER_FLAGS}) diff --git a/conanfile.py b/conanfile.py index d6e6144ab..23984ddc9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,6 +1,7 @@ from conan import ConanFile from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout + class Clio(ConanFile): name = 'clio' license = 'ISC' @@ -20,6 +21,7 @@ class Clio(ConanFile): 'coverage': [True, False], # build for test coverage report; create custom target `clio_tests-ccov` 'lint': [True, False], # run clang-tidy checks during compilation 'snapshot': [True, False], # build export/import snapshot tool + 'time_trace': [True, False] # build using -ftime-trace to create compiler trace reports } requires = [ @@ -29,7 +31,7 @@ class Clio(ConanFile): 'protobuf/3.21.9', 'grpc/1.50.1', 'openssl/1.1.1v', - 'xrpl/2.4.0', + 'xrpl/2.5.0-b1', 'zlib/1.3.1', 'libbacktrace/cci.20210118' ] @@ -46,7 +48,8 @@ class Clio(ConanFile): 'lint': False, 'docs': False, 'snapshot': False, - + 'time_trace': False, + 'xrpl/*:tests': False, 'xrpl/*:rocksdb': False, 'cassandra-cpp-driver/*:shared': False, @@ -78,11 +81,12 @@ class Clio(ConanFile): def layout(self): cmake_layout(self) - # Fix this setting to follow the default introduced in Conan 1.48 + # Fix this setting to follow the default introduced in Conan 1.48 # to align with our build instructions. self.folders.generators = 'build/generators' generators = 'CMakeDeps' + def generate(self): tc = CMakeToolchain(self) tc.variables['verbose'] = self.options.verbose @@ -95,6 +99,7 @@ class Clio(ConanFile): tc.variables['packaging'] = self.options.packaging tc.variables['benchmark'] = self.options.benchmark tc.variables['snapshot'] = self.options.snapshot + tc.variables['time_trace'] = self.options.time_trace tc.generate() def build(self): diff --git a/docker/ci/dockerfile b/docker/ci/Dockerfile similarity index 78% rename from docker/ci/dockerfile rename to docker/ci/Dockerfile index 85d661bc2..21a4b11f6 100644 --- a/docker/ci/dockerfile +++ b/docker/ci/Dockerfile @@ -2,27 +2,36 @@ FROM rippleci/clio_clang:16 ARG DEBIAN_FRONTEND=noninteractive ARG TARGETARCH -SHELL ["/bin/bash", "-c"] +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Using root by default is not very secure but github checkout action doesn't work with any other user +# https://github.com/actions/checkout/issues/956 +# And Github Actions doc recommends using root +# https://docs.github.com/en/actions/sharing-automations/creating-actions/dockerfile-support-for-github-actions#user + +# hadolint ignore=DL3002 USER root WORKDIR /root ENV CCACHE_VERSION=4.10.2 \ LLVM_TOOLS_VERSION=19 \ GH_VERSION=2.40.0 \ - DOXYGEN_VERSION=1.12.0 - + DOXYGEN_VERSION=1.12.0 \ + CLANG_BUILD_ANALYZER_VERSION=1.6.0 \ + GIT_CLIFF_VERSION=2.8.0 + # Add repositories RUN apt-get -qq update \ && apt-get -qq install -y --no-install-recommends --no-install-suggests gnupg wget curl software-properties-common \ && echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-${LLVM_TOOLS_VERSION} main" >> /etc/apt/sources.list \ - && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - + && wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + apt-get clean && rm -rf /var/lib/apt/lists/* # Install packages RUN apt update -qq \ && apt install -y --no-install-recommends --no-install-suggests python3 python3-pip git git-lfs make ninja-build flex bison jq graphviz \ - clang-format-${LLVM_TOOLS_VERSION} clang-tidy-${LLVM_TOOLS_VERSION} clang-tools-${LLVM_TOOLS_VERSION} \ - && update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-${LLVM_TOOLS_VERSION} 100 \ - && pip3 install -q --upgrade --no-cache-dir pip && pip3 install -q --no-cache-dir conan==1.62 gcovr cmake cmake-format \ + clang-tidy-${LLVM_TOOLS_VERSION} clang-tools-${LLVM_TOOLS_VERSION} \ + && pip3 install -q --upgrade --no-cache-dir pip && pip3 install -q --no-cache-dir conan==1.62 gcovr cmake==3.31.6 pre-commit \ && apt-get clean && apt remove -y software-properties-common # Install gcc-12 and make ldconfig aware of the new libstdc++ location (for gcc) @@ -62,17 +71,25 @@ RUN wget "https://github.com/doxygen/doxygen/releases/download/Release_${DOXYGEN && cmake --build . --target install \ && rm -rf /tmp/* /var/tmp/* +# Install ClangBuildAnalyzer +RUN wget "https://github.com/aras-p/ClangBuildAnalyzer/releases/download/v${CLANG_BUILD_ANALYZER_VERSION}/ClangBuildAnalyzer-linux" \ + && chmod +x ClangBuildAnalyzer-linux \ + && mv ClangBuildAnalyzer-linux /usr/bin/ClangBuildAnalyzer \ + && rm -rf /tmp/* /var/tmp/* + +# Install git-cliff +RUN wget "https://github.com/orhun/git-cliff/releases/download/v${GIT_CLIFF_VERSION}/git-cliff-${GIT_CLIFF_VERSION}-x86_64-unknown-linux-musl.tar.gz" \ + && tar xf git-cliff-${GIT_CLIFF_VERSION}-x86_64-unknown-linux-musl.tar.gz \ + && mv git-cliff-${GIT_CLIFF_VERSION}/git-cliff /usr/bin/git-cliff \ + && rm -rf /tmp/* /var/tmp/* + # Install gh -RUN wget https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \ +RUN wget "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz" \ && tar xf gh_${GH_VERSION}_linux_${TARGETARCH}.tar.gz \ && mv gh_${GH_VERSION}_linux_${TARGETARCH}/bin/gh /usr/bin/gh \ && rm -rf /tmp/* /var/tmp/* WORKDIR /root -# Using root by default is not very secure but github checkout action doesn't work with any other user -# https://github.com/actions/checkout/issues/956 -# And Github Actions doc recommends using root -# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user # Setup conan RUN conan remote add --insert 0 conan-non-prod http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod @@ -95,7 +112,7 @@ RUN conan profile new clang --detect \ && conan profile update env.CC=/usr/bin/clang-16 clang \ && conan profile update env.CXX=/usr/bin/clang++-16 clang \ && conan profile update env.CXXFLAGS="-DBOOST_ASIO_DISABLE_CONCEPTS" clang \ - && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/clang-16\", \"cpp\": \"/usr/bin/clang++-16\"}" clang + && conan profile update "conf.tools.build:compiler_executables={\"c\": \"/usr/bin/clang-16\", \"cpp\": \"/usr/bin/clang++-16\"}" clang RUN echo "include(gcc)" >> .conan/profiles/default diff --git a/docker/ci/README.md b/docker/ci/README.md index cfc256f61..6dc311126 100644 --- a/docker/ci/README.md +++ b/docker/ci/README.md @@ -4,6 +4,7 @@ This image contains an environment to build [Clio](https://github.com/XRPLF/clio It is used in [Clio Github Actions](https://github.com/XRPLF/clio/actions) but can also be used to compile Clio locally. The image is based on Ubuntu 20.04 and contains: + - clang 16.0.6 - gcc 12.3 - doxygen 1.12 @@ -13,4 +14,4 @@ The image is based on Ubuntu 20.04 and contains: - and some other useful tools Conan is set up to build Clio without any additional steps. There are two preset conan profiles: `clang` and `gcc` to use corresponding compiler. By default conan is setup to use `gcc`. -Sanitizer builds for `ASAN`, `TSAN` and `UBSAN` are enabled via conan profiles for each of the supported compilers. These can be selected using the following pattern (all lowercase): `[compiler].[sanitizer]` (e.g. `--profile gcc.tsan`). +Sanitizer builds for `ASAN`, `TSAN` and `UBSAN` are enabled via conan profiles for each of the supported compilers. These can be selected using the following pattern (all lowercase): `[compiler].[sanitizer]` (e.g. `--profile gcc.tsan`). diff --git a/docker/clio/dockerfile b/docker/clio/Dockerfile similarity index 100% rename from docker/clio/dockerfile rename to docker/clio/Dockerfile diff --git a/docker/clio/README.md b/docker/clio/README.md index 3ec59f01b..a1d4a5145 100644 --- a/docker/clio/README.md +++ b/docker/clio/README.md @@ -12,12 +12,14 @@ Your configuration file should be mounted under the path `/opt/clio/etc/config.j Clio repository provides an [example](https://github.com/XRPLF/clio/blob/develop/docs/examples/config/example-config.json) of the configuration file. Config file recommendations: + - Set `log_to_console` to `false` if you want to avoid logs being written to `stdout`. - Set `log_directory` to `/opt/clio/log` to store logs in a volume. ## Usage The following command can be used to run Clio in docker (assuming server's port is `51233` in your config): + ```bash docker run -d -v :/opt/clio/etc/config.json -v :/opt/clio/log -p 51233:51233 rippleci/clio ``` diff --git a/docker/compilers/clang-16/dockerfile b/docker/compilers/clang-16/Dockerfile similarity index 91% rename from docker/compilers/clang-16/dockerfile rename to docker/compilers/clang-16/Dockerfile index 7972d60b5..e543d904a 100644 --- a/docker/compilers/clang-16/dockerfile +++ b/docker/compilers/clang-16/Dockerfile @@ -1,8 +1,10 @@ -FROM ubuntu:focal +FROM ubuntu:focal ARG DEBIAN_FRONTEND=noninteractive ARG TARGETARCH SHELL ["/bin/bash", "-c"] + +# hadolint ignore=DL3002 USER root WORKDIR /root diff --git a/docker/compilers/gcc-12/dockerfile b/docker/compilers/gcc-12/Dockerfile similarity index 98% rename from docker/compilers/gcc-12/dockerfile rename to docker/compilers/gcc-12/Dockerfile index fc2395857..07ac79bd7 100644 --- a/docker/compilers/gcc-12/dockerfile +++ b/docker/compilers/gcc-12/Dockerfile @@ -43,7 +43,7 @@ RUN /gcc-$GCC_VERSION/configure \ --disable-multilib \ --without-cuda-driver \ --enable-checking=release \ - && make -j`nproc` \ + && make -j "$(nproc)" \ && make install-strip DESTDIR=/gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION \ && mkdir -p /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64 \ && mv /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/lib64/libstdc++.so.6.0.30-gdb.py /gcc-$GCC_VERSION-$BUILD_VERSION-ubuntu-$UBUNTU_VERSION/usr/share/gdb/auto-load/usr/lib64/libstdc++.so.6.0.30-gdb.py @@ -63,7 +63,7 @@ COPY --from=build /gcc12.deb / # Make gcc-12 available but also leave gcc12.deb for others to copy if needed RUN apt update && apt-get install -y binutils libc6-dev \ - && dpkg -i /gcc12.deb + && dpkg -i /gcc12.deb RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 \ && update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 \ diff --git a/docker/develop/compose.yaml b/docker/develop/compose.yaml index 69376ccfd..4fb23b7c2 100644 --- a/docker/develop/compose.yaml +++ b/docker/develop/compose.yaml @@ -1,6 +1,6 @@ services: clio_develop: - image: rippleci/clio_ci:latest + image: ghcr.io/xrplf/clio-ci:latest volumes: - clio_develop_conan_data:/root/.conan/data - clio_develop_ccache:/root/.ccache diff --git a/docker/develop/run b/docker/develop/run index 4acb9f4f1..6874831e2 100755 --- a/docker/develop/run +++ b/docker/develop/run @@ -59,4 +59,3 @@ case $1 in esac popd > /dev/null - diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 2a6d6fa5f..598b4fd41 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -3,6 +3,8 @@ project(docs) include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/ClioVersion.cmake) +# cmake-format: off # Generate `docs` target for doxygen documentation # Note: use `cmake --build . --target docs` from your `build` directory to generate the documentation +# cmake-format: on include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/Docs.cmake) diff --git a/docs/Doxyfile b/docs/Doxyfile index 712bed580..f9d897d63 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -34,7 +34,7 @@ FULL_SIDEBAR = NO HTML_HEADER = ${SOURCE}/docs/doxygen-awesome-theme/header.html HTML_EXTRA_STYLESHEET = ${SOURCE}/docs/doxygen-awesome-theme/doxygen-awesome.css \ ${SOURCE}/docs/doxygen-awesome-theme/doxygen-awesome-sidebar-only.css \ - ${SOURCE}/docs/doxygen-awesome-theme/doxygen-awesome-sidebar-only-darkmode-toggle.css + ${SOURCE}/docs/doxygen-awesome-theme/doxygen-awesome-sidebar-only-darkmode-toggle.css HTML_EXTRA_FILES = ${SOURCE}/docs/doxygen-awesome-theme/doxygen-awesome-darkmode-toggle.js \ ${SOURCE}/docs/doxygen-awesome-theme/doxygen-awesome-interactive-toc.js diff --git a/docs/build-clio.md b/docs/build-clio.md index 6351b0345..5e725c42e 100644 --- a/docs/build-clio.md +++ b/docs/build-clio.md @@ -1,31 +1,32 @@ # How to build Clio -Clio is built with [CMake](https://cmake.org/) and uses [Conan](https://conan.io/) for managing dependencies. It is written in C++20 and therefore requires a modern compiler. +`Clio` is built with [CMake](https://cmake.org/) and uses [Conan](https://conan.io/) for managing dependencies. +`Clio` is written in C++23 and therefore requires a modern compiler. ## Minimum Requirements - [Python 3.7](https://www.python.org/downloads/) -- [Conan 1.55](https://conan.io/downloads.html) -- [CMake 3.20](https://cmake.org/download/) +- [Conan 1.55, <2.0](https://conan.io/downloads.html) +- [CMake 3.20, <4.0](https://cmake.org/download/) - [**Optional**] [GCovr](https://gcc.gnu.org/onlinedocs/gcc/Gcov.html): needed for code coverage generation - [**Optional**] [CCache](https://ccache.dev/): speeds up compilation if you are going to compile Clio often | Compiler | Version | -|-------------|---------| +| ----------- | ------- | | GCC | 12.3 | | Clang | 16 | | Apple Clang | 15 | ### Conan Configuration -Clio does not require anything other than `compiler.cppstd=20` in your (`~/.conan/profiles/default`) Conan profile. +Clio requires `compiler.cppstd=20` in your Conan profile (`~/.conan/profiles/default`). > [!NOTE] > Although Clio is built using C++23, it's required to set `compiler.cppstd=20` for the time being as some of Clio's dependencies are not yet capable of building under C++23. -> Mac example: +**Mac example**: -``` +```text [settings] os=Macos os_build=Macos @@ -40,9 +41,9 @@ compiler.cppstd=20 tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"] ``` -> Linux example: +**Linux example**: -``` +```text [settings] os=Linux os_build=Linux @@ -80,7 +81,8 @@ Navigate to Clio's root directory and run: ```sh mkdir build && cd build -conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False +conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True +# You can also add -GNinja to use Ninja build system instead of Make cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release .. cmake --build . --parallel 8 # or without the number if you feel extra adventurous ``` @@ -93,21 +95,23 @@ If successful, `conan install` will find the required packages and `cmake` will > [!TIP] > To generate a Code Coverage report, include `-o coverage=True` in the `conan install` command above, along with `-o tests=True` to enable tests. After running the `cmake` commands, execute `make clio_tests-ccov`. The coverage report will be found at `clio_tests-llvm-cov/index.html`. + + > [!NOTE] > If you've built Clio before and the build is now failing, it's likely due to updated dependencies. Try deleting the build folder and then rerunning the Conan and CMake commands mentioned above. ### Generating API docs for Clio -The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen on your system. +The API documentation for Clio is generated by [Doxygen](https://www.doxygen.nl/index.html). If you want to generate the API documentation when building Clio, make sure to install Doxygen 1.12.0 on your system. To generate the API docs: 1. First, include `-o docs=True` in the conan install command. For example: - ```sh - mkdir build && cd build - conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False -o docs=True - ``` + ```sh + mkdir build && cd build + conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o docs=True + ``` 2. Once that has completed successfully, run the `cmake` command and add the `--target docs` option: @@ -118,19 +122,19 @@ To generate the API docs: 3. Go to `build/docs/html` to view the generated files. - Open the `index.html` file in your browser to see the documentation pages. + Open the `index.html` file in your browser to see the documentation pages. - ![API index page](./img/doxygen-docs-output.png "API index page") + ![API index page](./img/doxygen-docs-output.png "API index page") ## Building Clio with Docker It is also possible to build Clio using [Docker](https://www.docker.com/) if you don't want to install all the dependencies on your machine. ```sh -docker run -it rippleci/clio_ci:latest +docker run -it ghcr.io/xrplf/clio-ci:latest git clone https://github.com/XRPLF/clio mkdir build && cd build -conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=False +conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True cmake -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release .. cmake --build . --parallel 8 # or without the number if you feel extra adventurous ``` @@ -146,54 +150,56 @@ If you wish to develop against a `rippled` instance running in standalone mode t Sometimes, during development, you need to build against a custom version of `libxrpl`. (For example, you may be developing compatibility for a proposed amendment that is not yet merged to the main `rippled` codebase.) To build Clio with compatibility for a custom fork or branch of `rippled`, follow these steps: -1. First, pull/clone the appropriate `rippled` fork and switch to the branch you want to build. For example, the following example uses an in-development build with [XLS-33d Multi-Purpose Tokens](https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens): +1. First, pull/clone the appropriate `rippled` fork and switch to the branch you want to build. + The following example uses an in-development build with [XLS-33d Multi-Purpose Tokens](https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033d-multi-purpose-tokens): - ```sh - git clone https://github.com/shawnxie999/rippled/ - cd rippled - git switch mpt-1.1 - ``` + ```sh + git clone https://github.com/shawnxie999/rippled/ + cd rippled + git switch mpt-1.1 + ``` 2. Export a custom package to your local Conan store using a user/channel: - ```sh - conan export . my/feature - ``` + ```sh + conan export . my/feature + ``` 3. Patch your local Clio build to use the right package. - Edit `conanfile.py` (from the Clio repository root). Replace the `xrpl` requirement with the custom package version from the previous step. This must also include the current version number from your `rippled` branch. For example: + Edit `conanfile.py` (from the Clio repository root). Replace the `xrpl` requirement with the custom package version from the previous step. This must also include the current version number from your `rippled` branch. For example: - ```py - # ... (excerpt from conanfile.py) - requires = [ - 'boost/1.82.0', - 'cassandra-cpp-driver/2.17.0', - 'fmt/10.1.1', - 'protobuf/3.21.9', - 'grpc/1.50.1', - 'openssl/1.1.1u', - 'xrpl/2.3.0-b1@my/feature', # Update this line - 'libbacktrace/cci.20210118' - ] - ``` + ```py + # ... (excerpt from conanfile.py) + requires = [ + 'boost/1.83.0', + 'cassandra-cpp-driver/2.17.0', + 'fmt/10.1.1', + 'protobuf/3.21.9', + 'grpc/1.50.1', + 'openssl/1.1.1v', + 'xrpl/2.3.0-b1@my/feature', # Update this line + 'zlib/1.3.1', + 'libbacktrace/cci.20210118' + ] + ``` 4. Build Clio as you would have before. - See [Building Clio](#building-clio) for details. + See [Building Clio](#building-clio) for details. ## Using `clang-tidy` for static analysis The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 19.0. -Clang-tidy can be run by Cmake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command: +Clang-tidy can be run by CMake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command: ```sh conan install .. --output-folder . --build missing --settings build_type=Release -o tests=True -o lint=True ``` -By default Cmake will try to find `clang-tidy` automatically in your system. -To force Cmake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example: +By default CMake will try to find `clang-tidy` automatically in your system. +To force CMake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example: ```sh export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@19/bin/clang-tidy diff --git a/docs/config-description.md b/docs/config-description.md index 5ef30b2f6..2d895563e 100644 --- a/docs/config-description.md +++ b/docs/config-description.md @@ -1,452 +1,592 @@ # Clio Config Description -This file lists all Clio Configuration definitions in detail. + +This document provides a list of all available Clio configuration properties in detail. + +> [!NOTE] +> Dot notation in configuration key names represents nested fields. For example, **database.scylladb** refers to the _scylladb_ field inside the _database_ object. If a key name includes "[]", it indicates that the nested field is an array (e.g., etl_sources.[]). ## Configuration Details -### Key: database.type -- **Required**: True -- **Type**: string -- **Default value**: cassandra -- **Constraints**: The value must be one of the following: `cassandra` - - **Description**: Type of database to use. We currently support Cassandra and Scylladb. We default to Scylladb. -### Key: database.cassandra.contact_points -- **Required**: True -- **Type**: string -- **Default value**: localhost -- **Constraints**: None - - **Description**: A list of IP addresses or hostnames of the initial nodes (Cassandra/Scylladb cluster nodes) that the client will connect to when establishing a connection with the database. If you're running locally, it should be 'localhost' or 127.0.0.1 -### Key: database.cassandra.secure_connect_bundle -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: Configuration file that contains the necessary security credentials and connection details for securely connecting to a Cassandra database cluster. -### Key: database.cassandra.port -- **Required**: False -- **Type**: int -- **Default value**: None -- **Constraints**: The minimum value is `1`. The maximum value is `65535 - - **Description**: Port number to connect to the database. -### Key: database.cassandra.keyspace -- **Required**: True -- **Type**: string -- **Default value**: clio -- **Constraints**: None - - **Description**: Keyspace to use for the database. -### Key: database.cassandra.replication_factor -- **Required**: True -- **Type**: int -- **Default value**: 3 -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Number of replicated nodes for Scylladb. Visit this link for more details : https://university.scylladb.com/courses/scylla-essentials-overview/lessons/high-availability/topic/fault-tolerance-replication-factor/ -### Key: database.cassandra.table_prefix -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: Prefix for Database table names. -### Key: database.cassandra.max_write_requests_outstanding -- **Required**: True -- **Type**: int -- **Default value**: 10000 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Maximum number of outstanding write requests. Write requests are api calls that write to database -### Key: database.cassandra.max_read_requests_outstanding -- **Required**: True -- **Type**: int -- **Default value**: 100000 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Maximum number of outstanding read requests, which reads from database -### Key: database.cassandra.threads -- **Required**: True -- **Type**: int -- **Default value**: The number of available CPU cores. -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Number of threads that will be used for database operations. -### Key: database.cassandra.core_connections_per_host -- **Required**: True -- **Type**: int -- **Default value**: 1 -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Number of core connections per host for Cassandra. -### Key: database.cassandra.queue_size_io -- **Required**: False -- **Type**: int -- **Default value**: None -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Queue size for I/O operations in Cassandra. -### Key: database.cassandra.write_batch_size -- **Required**: True -- **Type**: int -- **Default value**: 20 -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Batch size for write operations in Cassandra. -### Key: database.cassandra.connect_timeout -- **Required**: False -- **Type**: int -- **Default value**: None -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: The maximum amount of time in seconds the system will wait for a connection to be successfully established with the database. -### Key: database.cassandra.request_timeout -- **Required**: False -- **Type**: int -- **Default value**: None -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: The maximum amount of time in seconds the system will wait for a request to be fetched from database. -### Key: database.cassandra.username -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: The username used for authenticating with the database. -### Key: database.cassandra.password -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: The password used for authenticating with the database. -### Key: database.cassandra.certfile -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: The path to the SSL/TLS certificate file used to establish a secure connection between the client and the Cassandra database. -### Key: allow_no_etl -- **Required**: True -- **Type**: boolean -- **Default value**: True -- **Constraints**: None - - **Description**: If True, no ETL nodes will run with Clio. -### Key: etl_sources.[].ip -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: The value must be a valid IP address - - **Description**: IP address of the ETL source. -### Key: etl_sources.[].ws_port -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: The minimum value is `1`. The maximum value is `65535 - - **Description**: WebSocket port of the ETL source. -### Key: etl_sources.[].grpc_port -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: The minimum value is `1`. The maximum value is `65535 - - **Description**: gRPC port of the ETL source. -### Key: forwarding.cache_timeout -- **Required**: True -- **Type**: double -- **Default value**: 0 -- **Constraints**: The value must be a positive double number - - **Description**: Timeout duration for the forwarding cache used in Rippled communication. -### Key: forwarding.request_timeout -- **Required**: True -- **Type**: double -- **Default value**: 10 -- **Constraints**: The value must be a positive double number - - **Description**: Timeout duration for the forwarding request used in Rippled communication. -### Key: rpc.cache_timeout -- **Required**: True -- **Type**: double -- **Default value**: 0 -- **Constraints**: The value must be a positive double number - - **Description**: Timeout duration for RPC requests. -### Key: num_markers -- **Required**: False -- **Type**: int -- **Default value**: None -- **Constraints**: The minimum value is `1`. The maximum value is `256` - - **Description**: The number of markers is the number of coroutines to download the initial ledger -### Key: dos_guard.whitelist.[] -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: List of IP addresses to whitelist for DOS protection. -### Key: dos_guard.max_fetches -- **Required**: True -- **Type**: int -- **Default value**: 1000000 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Maximum number of fetch operations allowed by DOS guard. -### Key: dos_guard.max_connections -- **Required**: True -- **Type**: int -- **Default value**: 20 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Maximum number of concurrent connections allowed by DOS guard. -### Key: dos_guard.max_requests -- **Required**: True -- **Type**: int -- **Default value**: 20 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Maximum number of requests allowed by DOS guard. -### Key: dos_guard.sweep_interval -- **Required**: True -- **Type**: double -- **Default value**: 1 -- **Constraints**: The value must be a positive double number - - **Description**: Interval in seconds for DOS guard to sweep/clear its state. -### Key: workers -- **Required**: True -- **Type**: int -- **Default value**: The number of available CPU cores. -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Number of threads to process RPC requests. -### Key: server.ip -- **Required**: True -- **Type**: string -- **Default value**: None -- **Constraints**: The value must be a valid IP address - - **Description**: IP address of the Clio HTTP server. -### Key: server.port -- **Required**: True -- **Type**: int -- **Default value**: None -- **Constraints**: The minimum value is `1`. The maximum value is `65535 - - **Description**: Port number of the Clio HTTP server. -### Key: server.max_queue_size -- **Required**: True -- **Type**: int -- **Default value**: 0 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Maximum size of the server's request queue. Value of 0 is no limit. -### Key: server.local_admin -- **Required**: False -- **Type**: boolean -- **Default value**: None -- **Constraints**: None - - **Description**: Indicates if the server should run with admin privileges. Only one of local_admin or admin_password can be set. -### Key: server.admin_password -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: Password for Clio admin-only APIs. Only one of local_admin or admin_password can be set. -### Key: server.processing_policy -- **Required**: True -- **Type**: string -- **Default value**: parallel -- **Constraints**: The value must be one of the following: `parallel, sequent` - - **Description**: Could be "sequent" or "parallel". For the sequent policy, requests from a single client - connection are processed one by one, with the next request read only after the previous one is processed. For the parallel policy, Clio will accept - all requests and process them in parallel, sending a reply for each request as soon as it is ready. -### Key: server.parallel_requests_limit -- **Required**: False -- **Type**: int -- **Default value**: None -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Optional parameter, used only if processing_strategy `parallel`. It limits the number of requests for a single client connection that are processed in parallel. If not specified, the limit is infinite. -### Key: server.ws_max_sending_queue_size -- **Required**: True -- **Type**: int -- **Default value**: 1500 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Maximum size of the websocket sending queue. -### Key: prometheus.enabled -- **Required**: True -- **Type**: boolean -- **Default value**: False -- **Constraints**: None - - **Description**: Enable or disable Prometheus metrics. -### Key: prometheus.compress_reply -- **Required**: True -- **Type**: boolean -- **Default value**: False -- **Constraints**: None - - **Description**: Enable or disable compression of Prometheus responses. -### Key: io_threads -- **Required**: True -- **Type**: int -- **Default value**: 2 -- **Constraints**: The minimum value is `1`. The maximum value is `65535` - - **Description**: Number of I/O threads. Value cannot be less than 1 -### Key: subscription_workers -- **Required**: True -- **Type**: int -- **Default value**: 1 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: The number of worker threads or processes that are responsible for managing and processing subscription-based tasks from rippled -### Key: graceful_period -- **Required**: True -- **Type**: double -- **Default value**: 10 -- **Constraints**: The value must be a positive double number - - **Description**: Number of milliseconds server will wait to shutdown gracefully. -### Key: cache.num_diffs -- **Required**: True -- **Type**: int -- **Default value**: 32 -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Number of diffs to cache. For more info, consult readme.md in etc -### Key: cache.num_markers -- **Required**: True -- **Type**: int -- **Default value**: 48 -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Number of markers to cache. -### Key: cache.num_cursors_from_diff -- **Required**: True -- **Type**: int -- **Default value**: 0 -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Num of cursors that are different. -### Key: cache.num_cursors_from_account -- **Required**: True -- **Type**: int -- **Default value**: 0 -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Number of cursors from an account. -### Key: cache.page_fetch_size -- **Required**: True -- **Type**: int -- **Default value**: 512 -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Page fetch size for cache operations. -### Key: cache.load -- **Required**: True -- **Type**: string -- **Default value**: async -- **Constraints**: The value must be one of the following: `sync, async, none` - - **Description**: Cache loading strategy ('sync' or 'async'). -### Key: log_channels.[].channel -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: The value must be one of the following: `General, WebServer, Backend, RPC, ETL, Subscriptions, Performance, Migration` - - **Description**: Name of the log channel.'RPC', 'ETL', and 'Performance' -### Key: log_channels.[].log_level -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: The value must be one of the following: `trace, debug, info, warning, error, fatal, count` - - **Description**: Log level for the specific log channel.`warning`, `error`, `fatal` -### Key: log_level -- **Required**: True -- **Type**: string -- **Default value**: info -- **Constraints**: The value must be one of the following: `trace, debug, info, warning, error, fatal, count` - - **Description**: General logging level of Clio. This level will be applied to all log channels that do not have an explicitly defined logging level. -### Key: log_format -- **Required**: True -- **Type**: string -- **Default value**: %TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message% -- **Constraints**: None - - **Description**: Format string for log messages. -### Key: log_to_console -- **Required**: True -- **Type**: boolean -- **Default value**: True -- **Constraints**: None - - **Description**: Enable or disable logging to console. -### Key: log_directory -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: Directory path for log files. -### Key: log_rotation_size -- **Required**: True -- **Type**: int -- **Default value**: 2048 -- **Constraints**: The minimum value is `1`. The maximum value is `4294967295` - - **Description**: Log rotation size in megabytes. When the log file reaches this particular size, a new log file starts. -### Key: log_directory_max_size -- **Required**: True -- **Type**: int -- **Default value**: 51200 -- **Constraints**: The minimum value is `1`. The maximum value is `4294967295` - - **Description**: Maximum size of the log directory in megabytes. -### Key: log_rotation_hour_interval -- **Required**: True -- **Type**: int -- **Default value**: 12 -- **Constraints**: The minimum value is `1`. The maximum value is `4294967295` - - **Description**: Interval in hours for log rotation. If the current log file reaches this value in logging, a new log file starts. -### Key: log_tag_style -- **Required**: True -- **Type**: string -- **Default value**: none -- **Constraints**: The value must be one of the following: `int, uint, null, none, uuid` - - **Description**: Style for log tags. -### Key: extractor_threads -- **Required**: True -- **Type**: int -- **Default value**: 1 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Number of extractor threads. -### Key: read_only -- **Required**: True -- **Type**: boolean -- **Default value**: True -- **Constraints**: None - - **Description**: Indicates if the server should have read-only privileges. -### Key: txn_threshold -- **Required**: True -- **Type**: int -- **Default value**: 0 -- **Constraints**: The minimum value is `0`. The maximum value is `65535` - - **Description**: Transaction threshold value. -### Key: start_sequence -- **Required**: False -- **Type**: int -- **Default value**: None -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Starting ledger index. -### Key: finish_sequence -- **Required**: False -- **Type**: int -- **Default value**: None -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: Ending ledger index. -### Key: ssl_cert_file -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: Path to the SSL certificate file. -### Key: ssl_key_file -- **Required**: False -- **Type**: string -- **Default value**: None -- **Constraints**: None - - **Description**: Path to the SSL key file. -### Key: api_version.default -- **Required**: True -- **Type**: int -- **Default value**: 1 -- **Constraints**: The minimum value is `1`. The maximum value is `3` - - **Description**: Default API version Clio will run on. -### Key: api_version.min -- **Required**: True -- **Type**: int -- **Default value**: 1 -- **Constraints**: The minimum value is `1`. The maximum value is `3` - - **Description**: Minimum API version. -### Key: api_version.max -- **Required**: True -- **Type**: int -- **Default value**: 3 -- **Constraints**: The minimum value is `1`. The maximum value is `3` - - **Description**: Maximum API version. -### Key: migration.full_scan_threads -- **Required**: True -- **Type**: int -- **Default value**: 2 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: The number of threads used to scan the table. -### Key: migration.full_scan_jobs -- **Required**: True -- **Type**: int -- **Default value**: 4 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: The number of coroutines used to scan the table. -### Key: migration.cursors_per_job -- **Required**: True -- **Type**: int -- **Default value**: 100 -- **Constraints**: The minimum value is `0`. The maximum value is `4294967295` - - **Description**: The number of cursors each coroutine will scan. +### database.type +- **Required**: True +- **Type**: string +- **Default value**: `cassandra` +- **Constraints**: The value must be one of the following: `cassandra`. +- **Description**: Specifies the type of database used for storing and retrieving data required by the Clio server. Both ScyllaDB and Cassandra can serve as backends for Clio; however, this value must be set to `cassandra`. + +### database.cassandra.contact_points + +- **Required**: True +- **Type**: string +- **Default value**: `localhost` +- **Constraints**: None +- **Description**: A list of IP addresses or hostnames for the initial cluster nodes (Cassandra or ScyllaDB) that the client connects to when establishing a database connection. If you're running Clio locally, set this value to `localhost` or `127.0.0.1`. + +### database.cassandra.secure_connect_bundle + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: The configuration file that contains the necessary credentials and connection details for securely connecting to a Cassandra database cluster. + +### database.cassandra.port + +- **Required**: False +- **Type**: int +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: The port number used to connect to the Cassandra database. + +### database.cassandra.keyspace + +- **Required**: True +- **Type**: string +- **Default value**: `clio` +- **Constraints**: None +- **Description**: The Cassandra keyspace to use for the database. If you don't provide a value, this is set to `clio` by default. + +### database.cassandra.replication_factor + +- **Required**: True +- **Type**: int +- **Default value**: `3` +- **Constraints**: The minimum value is `0`. The maximum value is `65535`. +- **Description**: Represents the number of replicated nodes for ScyllaDB. For more details see [Fault Tolerance Replication Factor](https://university.scylladb.com/courses/scylla-essentials-overview/lessons/high-availability/topic/fault-tolerance-replication-factor/). + +### database.cassandra.table_prefix + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: An optional field to specify a prefix for the Cassandra database table names. + +### database.cassandra.max_write_requests_outstanding + +- **Required**: True +- **Type**: int +- **Default value**: `10000` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: Represents the maximum number of outstanding write requests. Write requests are API calls that write to the database. + +### database.cassandra.max_read_requests_outstanding + +- **Required**: True +- **Type**: int +- **Default value**: `100000` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: Maximum number of outstanding read requests. Read requests are API calls that read from the database. + +### database.cassandra.threads + +- **Required**: True +- **Type**: int +- **Default value**: The number of available CPU cores. +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: Represents the number of threads that will be used for database operations. + +### database.cassandra.core_connections_per_host + +- **Required**: True +- **Type**: int +- **Default value**: `1` +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: The number of core connections per host for the Cassandra database. + +### database.cassandra.queue_size_io + +- **Required**: False +- **Type**: int +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: Defines the queue size of the input/output (I/O) operations in Cassandra. + +### database.cassandra.write_batch_size + +- **Required**: True +- **Type**: int +- **Default value**: `20` +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: Represents the batch size for write operations in Cassandra. + +### database.cassandra.connect_timeout + +- **Required**: False +- **Type**: int +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The maximum amount of time in seconds that the system waits for a database connection to be established. + +### database.cassandra.request_timeout + +- **Required**: False +- **Type**: int +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The maximum amount of time in seconds that the system waits for a request to be fetched from the database. + +### database.cassandra.username + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: The username used for authenticating with the database. + +### database.cassandra.password + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: The password used for authenticating with the database. + +### database.cassandra.certfile + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: The path to the SSL/TLS certificate file used to establish a secure connection between the client and the Cassandra database. + +### allow_no_etl + +- **Required**: True +- **Type**: boolean +- **Default value**: `True` +- **Constraints**: None +- **Description**: If set to `True`, allows Clio to start without any ETL source. + +### etl_sources.[].ip + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: The value must be a valid IP address. +- **Description**: The IP address of the ETL source. + +### etl_sources.[].ws_port + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: The WebSocket port of the ETL source. + +### etl_sources.[].grpc_port + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: The gRPC port of the ETL source. + +### forwarding.cache_timeout + +- **Required**: True +- **Type**: double +- **Default value**: `0` +- **Constraints**: The value must be a positive double number. +- **Description**: Specifies the timeout duration (in seconds) for the forwarding cache used in `rippled` communication. A value of `0` means disabling this feature. + +### forwarding.request_timeout + +- **Required**: True +- **Type**: double +- **Default value**: `10` +- **Constraints**: The value must be a positive double number. +- **Description**: Specifies the timeout duration (in seconds) for the forwarding request used in `rippled` communication. + +### rpc.cache_timeout + +- **Required**: True +- **Type**: double +- **Default value**: `0` +- **Constraints**: The value must be a positive double number. +- **Description**: Specifies the timeout duration (in seconds) for RPC cache response to timeout. A value of `0` means disabling this feature. + +### num_markers + +- **Required**: False +- **Type**: int +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `256`. +- **Description**: Specifies the number of coroutines used to download the initial ledger. + +### dos_guard.whitelist.[] + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: The list of IP addresses to whitelist for DOS protection. + +### dos_guard.max_fetches + +- **Required**: True +- **Type**: int +- **Default value**: `1000000` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The maximum number of fetch operations allowed by DOS guard. + +### dos_guard.max_connections + +- **Required**: True +- **Type**: int +- **Default value**: `20` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The maximum number of concurrent connections for a specific IP address. + +### dos_guard.max_requests + +- **Required**: True +- **Type**: int +- **Default value**: `20` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The maximum number of requests allowed for a specific IP address. + +### dos_guard.sweep_interval + +- **Required**: True +- **Type**: double +- **Default value**: `1` +- **Constraints**: The value must be a positive double number. +- **Description**: Interval in seconds for DOS guard to sweep(clear) its state. + +### workers + +- **Required**: True +- **Type**: int +- **Default value**: The number of available CPU cores. +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The number of threads used to process RPC requests. + +### server.ip + +- **Required**: True +- **Type**: string +- **Default value**: None +- **Constraints**: The value must be a valid IP address. +- **Description**: The IP address of the Clio HTTP server. + +### server.port + +- **Required**: True +- **Type**: int +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: The port number of the Clio HTTP server. + +### server.max_queue_size + +- **Required**: True +- **Type**: int +- **Default value**: `1` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The maximum size of the server's request queue. If set to `0`, this means there is no queue size limit. + +### server.local_admin + +- **Required**: False +- **Type**: boolean +- **Default value**: None +- **Constraints**: None +- **Description**: Indicates if requests from `localhost` are allowed to call Clio admin-only APIs. Note that this setting cannot be enabled together with [server.admin_password](#serveradmin_password). + +### server.admin_password + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: The password for Clio admin-only APIs. Note that this setting cannot be enabled together with [server.local_admin](#serveradmin_password). + +### server.processing_policy + +- **Required**: True +- **Type**: string +- **Default value**: `parallel` +- **Constraints**: The value must be one of the following: `parallel`, `sequent`. +- **Description**: For the `sequent` policy, requests from a single client connection are processed one by one, with the next request read only after the previous one is processed. For the `parallel` policy, Clio will accept all requests and process them in parallel, sending a reply for each request as soon as it is ready. + +### server.parallel_requests_limit + +- **Required**: False +- **Type**: int +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: This is an optional parameter, used only if the `processing_strategy` is `parallel`. It limits the number of requests processed in parallel for a single client connection. If not specified, no limit is enforced. + +### server.ws_max_sending_queue_size + +- **Required**: True +- **Type**: int +- **Default value**: `1500` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: Maximum queue size for sending subscription data to clients. This queue buffers data when a client is slow to receive it, ensuring delivery once the client is ready. + +### prometheus.enabled + +- **Required**: True +- **Type**: boolean +- **Default value**: `False` +- **Constraints**: None +- **Description**: Enables or disables Prometheus metrics. + +### prometheus.compress_reply + +- **Required**: True +- **Type**: boolean +- **Default value**: `False` +- **Constraints**: None +- **Description**: Enables or disables compression of Prometheus responses. + +### io_threads + +- **Required**: True +- **Type**: int +- **Default value**: `2` +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: The number of input/output (I/O) threads. The value cannot be less than `1`. + +### subscription_workers + +- **Required**: True +- **Type**: int +- **Default value**: `1` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The number of worker threads or processes that are responsible for managing and processing subscription-based tasks from `rippled`. + +### graceful_period + +- **Required**: True +- **Type**: double +- **Default value**: `10` +- **Constraints**: The value must be a positive double number. +- **Description**: The number of milliseconds the server waits to shutdown gracefully. If Clio does not shutdown gracefully after the specified value, it will be killed instead. + +### cache.num_diffs + +- **Required**: True +- **Type**: int +- **Default value**: `32` +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: The number of cursors generated is the number of changed (without counting deleted) objects in the latest `cache.num_diffs` number of ledgers. Cursors are workers that load the ledger cache from the position of markers concurrently. For more information, please read [README.md](../src/etl/README.md). + +### cache.num_markers + +- **Required**: True +- **Type**: int +- **Default value**: `48` +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: Specifies how many markers are placed randomly within the cache. These markers define the positions on the ledger that will be loaded concurrently by the workers. The higher the number, the more places within the cache we potentially cover. + +### cache.num_cursors_from_diff + +- **Required**: True +- **Type**: int +- **Default value**: `0` +- **Constraints**: The minimum value is `0`. The maximum value is `65535`. +- **Description**: `cache.num_cursors_from_diff` number of cursors are generated by looking at the number of changed objects in the most recent ledger. If number of changed objects in current ledger is not enough, it will keep reading previous ledgers until it hit `cache.num_cursors_from_diff`. If set to `0`, the system defaults to generating cursors based on `cache.num_diffs`. + +### cache.num_cursors_from_account + +- **Required**: True +- **Type**: int +- **Default value**: `0` +- **Constraints**: The minimum value is `0`. The maximum value is `65535`. +- **Description**: `cache.num_cursors_from_diff` of cursors are generated by reading accounts in `account_tx` table. If set to `0`, the system defaults to generating cursors based on `cache.num_diffs`. + +### cache.page_fetch_size + +- **Required**: True +- **Type**: int +- **Default value**: `512` +- **Constraints**: The minimum value is `1`. The maximum value is `65535`. +- **Description**: The number of ledger objects to fetch concurrently per marker. + +### cache.load + +- **Required**: True +- **Type**: string +- **Default value**: `async` +- **Constraints**: The value must be one of the following: `sync`, `async`, `none`. +- **Description**: The strategy used for Cache loading. + +### log_channels.[].channel + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: The value must be one of the following: `General`, `WebServer`, `Backend`, `RPC`, `ETL`, `Subscriptions`, `Performance`, `Migration`. +- **Description**: The name of the log channel. + +### log_channels.[].log_level + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: The value must be one of the following: `trace`, `debug`, `info`, `warning`, `error`, `fatal`, `count`. +- **Description**: The log level for the specific log channel. + +### log_level + +- **Required**: True +- **Type**: string +- **Default value**: `info` +- **Constraints**: The value must be one of the following: `trace`, `debug`, `info`, `warning`, `error`, `fatal`, `count`. +- **Description**: The general logging level of Clio. This level is applied to all log channels that do not have an explicitly defined logging level. + +### log_format + +- **Required**: True +- **Type**: string +- **Default value**: `%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%` +- **Constraints**: None +- **Description**: The format string for log messages. The format is described here: . + +### log_to_console + +- **Required**: True +- **Type**: boolean +- **Default value**: `True` +- **Constraints**: None +- **Description**: Enables or disables logging to the console. + +### log_directory + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: The directory path for the log files. + +### log_rotation_size + +- **Required**: True +- **Type**: int +- **Default value**: `2048` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The log rotation size in megabytes. When the log file reaches this particular size, a new log file starts. + +### log_directory_max_size + +- **Required**: True +- **Type**: int +- **Default value**: `51200` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The maximum size of the log directory in megabytes. + +### log_rotation_hour_interval + +- **Required**: True +- **Type**: int +- **Default value**: `12` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: Represents the interval (in hours) for log rotation. If the current log file reaches this value in logging, a new log file starts. + +### log_tag_style + +- **Required**: True +- **Type**: string +- **Default value**: `none` +- **Constraints**: The value must be one of the following: `int`, `uint`, `null`, `none`, `uuid`. +- **Description**: Log tags are unique identifiers for log messages. `uint`/`int` starts logging from 0 and increments, making it faster. In contrast, `uuid` generates a random unique identifier, which adds overhead. + +### extractor_threads + +- **Required**: True +- **Type**: int +- **Default value**: `1` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: Number of threads used to extract data from ETL source. + +### read_only + +- **Required**: True +- **Type**: boolean +- **Default value**: `True` +- **Constraints**: None +- **Description**: Indicates if the server is allowed to write data to the database. + +### start_sequence + +- **Required**: False +- **Type**: int +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: If specified, the ledger index Clio will start writing to the database from. + +### finish_sequence + +- **Required**: False +- **Type**: int +- **Default value**: None +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: If specified, the final ledger that Clio will write to the database. + +### ssl_cert_file + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: The path to the SSL certificate file. + +### ssl_key_file + +- **Required**: False +- **Type**: string +- **Default value**: None +- **Constraints**: None +- **Description**: The path to the SSL key file. + +### api_version.default + +- **Required**: True +- **Type**: int +- **Default value**: `1` +- **Constraints**: The minimum value is `1`. The maximum value is `3`. +- **Description**: The default API version that the Clio server will run on. + +### api_version.min + +- **Required**: True +- **Type**: int +- **Default value**: `1` +- **Constraints**: The minimum value is `1`. The maximum value is `3`. +- **Description**: The minimum API version allowed to use. + +### api_version.max + +- **Required**: True +- **Type**: int +- **Default value**: `3` +- **Constraints**: The minimum value is `1`. The maximum value is `3`. +- **Description**: The maximum API version allowed to use. + +### migration.full_scan_threads + +- **Required**: True +- **Type**: int +- **Default value**: `2` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The number of threads used to scan the table. + +### migration.full_scan_jobs + +- **Required**: True +- **Type**: int +- **Default value**: `4` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The number of coroutines used to scan the table. + +### migration.cursors_per_job + +- **Required**: True +- **Type**: int +- **Default value**: `100` +- **Constraints**: The minimum value is `1`. The maximum value is `4294967295`. +- **Description**: The number of cursors each job will scan. diff --git a/docs/configure-clio.md b/docs/configure-clio.md index ecd596da9..b7705bac5 100644 --- a/docs/configure-clio.md +++ b/docs/configure-clio.md @@ -18,7 +18,8 @@ Clio needs access to a `rippled` server in order to work. The following configur - A port to handle gRPC requests, with the IP(s) of Clio specified in the `secure_gateway` entry -The example configs of [rippled](https://github.com/XRPLF/rippled/blob/develop/cfg/rippled-example.cfg) and [Clio](../docs/examples/config/example-config.json) are set up in a way that minimal changes are required. +The example configs of [rippled](https://github.com/XRPLF/rippled/blob/develop/cfg/rippled-example.cfg) and [Clio](../docs/examples/config/example-config.json) are set up in a way that minimal changes are required. However, if you want to view all configuration keys available in Clio, see [config-description.md](./config-description.md). + When running locally, the only change needed is to uncomment the `port_grpc` section of the `rippled` config. If you're running Clio and `rippled` on separate machines, in addition to uncommenting the `port_grpc` section, a few other steps must be taken: @@ -52,7 +53,7 @@ Here is an example snippet from the config file: ## SSL -The parameters `ssl_cert_file` and `ssl_key_file` can also be added to the top level of precedence of our Clio config. The `ssl_cert_file` field specifies the filepath for your SSL cert, while `ssl_key_file` specifies the filepath for your SSL key. It is up to you how to change ownership of these folders for your designated Clio user. +The parameters `ssl_cert_file` and `ssl_key_file` can also be added to the top level of precedence of our Clio config. The `ssl_cert_file` field specifies the filepath for your SSL cert, while `ssl_key_file` specifies the filepath for your SSL key. It is up to you how to change ownership of these folders for your designated Clio user. Your options include: diff --git a/docs/examples/config/example-config.json b/docs/examples/config/example-config.json index f9d66ce37..8da8fbdd3 100644 --- a/docs/examples/config/example-config.json +++ b/docs/examples/config/example-config.json @@ -2,146 +2,144 @@ * This is an example configuration file. Please do not use without modifying to suit your needs. */ { - "database": { - "type": "cassandra", - "cassandra": { - "contact_points": "127.0.0.1", - "port": 9042, - "keyspace": "clio", - "replication_factor": 1, - "table_prefix": "", - "max_write_requests_outstanding": 25000, - "max_read_requests_outstanding": 30000, - "threads": 8, - // - // Advanced options. USE AT OWN RISK: - // --- - "core_connections_per_host": 1, // Defaults to 1 - "write_batch_size": 20 // Defaults to 20 - // - // Below options will use defaults from cassandra driver if left unspecified. - // See https://docs.datastax.com/en/developer/cpp-driver/2.17/api/struct.CassCluster/ for details. - // - // "queue_size_io": 2 - // - // --- - } - }, - "allow_no_etl": false, // Allow Clio to run without valid ETL source, otherwise Clio will stop if ETL check fails - "etl_sources": [ - { - "ip": "127.0.0.1", - "ws_port": "6005", - "grpc_port": "50051" - } - ], - "forwarding": { - "cache_timeout": 0.250, // in seconds, could be 0, which means no cache - "request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds) - }, - "rpc": { - "cache_timeout": 0.5 // in seconds, could be 0, which means no cache for rpc - }, - "dos_guard": { - // Comma-separated list of IPs to exclude from rate limiting - "whitelist": [ - "127.0.0.1" - ], - // - // The below values are the default values and are only specified here - // for documentation purposes. The rate limiter currently limits - // connections and bandwidth per IP. The rate limiter looks at the raw - // IP of a client connection, and so requests routed through a load - // balancer will all have the same IP and be treated as a single client. - // - "max_fetches": 1000000, // Max bytes per IP per sweep interval - "max_connections": 20, // Max connections per IP - "max_requests": 20, // Max connections per IP per sweep interval - "sweep_interval": 1 // Time in seconds before resetting max_fetches and max_requests - }, - "server": { - "ip": "0.0.0.0", - "port": 51233, - // Max number of requests to queue up before rejecting further requests. - // Defaults to 0, which disables the limit. - "max_queue_size": 500, - // If request contains header with authorization, Clio will check if it matches the prefix 'Password ' + this value's sha256 hash - // If matches, the request will be considered as admin request - "admin_password": "xrp", - // If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests - // It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time - "local_admin": false, - "processing_policy": "parallel", // Could be "sequent" or "parallel". - // For sequent policy request from one client connection will be processed one by one and the next one will not be read before - // the previous one is processed. For parallel policy Clio will take all requests and process them in parallel and - // send a reply for each request whenever it is ready. - "parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified. - // Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection. - "ws_max_sending_queue_size": 1500, - "__ng_web_server": false // Use ng web server. This is a temporary setting which will be deleted after switching to ng web server - }, - // Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet. - "graceful_period": 10.0, - // Overrides log level on a per logging channel. - // Defaults to global "log_level" for each unspecified channel. - "log_channels": [ - { - "channel": "Backend", - "log_level": "fatal" - }, - { - "channel": "WebServer", - "log_level": "info" - }, - { - "channel": "Subscriptions", - "log_level": "info" - }, - { - "channel": "RPC", - "log_level": "error" - }, - { - "channel": "ETL", - "log_level": "debug" - }, - { - "channel": "Performance", - "log_level": "trace" - } - ], - "cache": { - // Configure this to use either "num_diffs", "num_cursors_from_diff", or "num_cursors_from_account". By default, Clio uses "num_diffs". - "num_diffs": 32, // Generate the cursors from the latest ledger diff, then use the cursors to partition the ledger to load concurrently. The cursors number is affected by the busyness of the network. - // "num_cursors_from_diff": 3200, // Read the cursors from the diff table until we have enough cursors to partition the ledger to load concurrently. - // "num_cursors_from_account": 3200, // Read the cursors from the account table until we have enough cursors to partition the ledger to load concurrently. - "num_markers": 48, // The number of markers is the number of coroutines to load the cache concurrently. - "page_fetch_size": 512, // The number of rows to load for each page. - "load": "async" // "sync" to load cache synchronously or "async" to load cache asynchronously or "none"/"no" to turn off the cache. - }, - "prometheus": { - "enabled": true, - "compress_reply": true - }, - "log_level": "info", - // Log format (this is the default format) - "log_format": "%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%", - "log_to_console": true, - // Clio logs to file in the specified directory only if "log_directory" is set - // "log_directory": "./clio_log", - "log_rotation_size": 2048, - "log_directory_max_size": 51200, - "log_rotation_hour_interval": 12, - "log_tag_style": "uint", - "extractor_threads": 8, - "read_only": false, - // "start_sequence": [integer] the ledger index to start from, - // "finish_sequence": [integer] the ledger index to finish at, - // "ssl_cert_file" : "/full/path/to/cert.file", - // "ssl_key_file" : "/full/path/to/key.file" - "api_version": { - "min": 1, // Minimum API version supported (could be 1 or 2) - "max": 2, // Maximum API version supported (could be 1 or 2, but >= min) - "default": 1 // Clio behaves the same as rippled by default + "database": { + "type": "cassandra", + "cassandra": { + "contact_points": "127.0.0.1", + "port": 9042, + "keyspace": "clio", + "replication_factor": 1, + "table_prefix": "", + "max_write_requests_outstanding": 25000, + "max_read_requests_outstanding": 30000, + "threads": 8, + // + // Advanced options. USE AT OWN RISK: + // --- + "core_connections_per_host": 1, // Defaults to 1 + "write_batch_size": 20 // Defaults to 20 + // + // Below options will use defaults from cassandra driver if left unspecified. + // See https://docs.datastax.com/en/developer/cpp-driver/2.17/api/struct.CassCluster/ for details. + // + // "queue_size_io": 2 + // + // --- } + }, + "allow_no_etl": false, // Allow Clio to run without valid ETL source, otherwise Clio will stop if ETL check fails + "etl_sources": [ + { + "ip": "127.0.0.1", + "ws_port": "6005", + "grpc_port": "50051" + } + ], + "forwarding": { + "cache_timeout": 0.25, // in seconds, could be 0, which means no cache + "request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds) + }, + "rpc": { + "cache_timeout": 0.5 // in seconds, could be 0, which means no cache for rpc + }, + "dos_guard": { + // Comma-separated list of IPs to exclude from rate limiting + "whitelist": ["127.0.0.1"], + // + // The below values are the default values and are only specified here + // for documentation purposes. The rate limiter currently limits + // connections and bandwidth per IP. The rate limiter looks at the raw + // IP of a client connection, and so requests routed through a load + // balancer will all have the same IP and be treated as a single client. + // + "max_fetches": 1000000, // Max bytes per IP per sweep interval + "max_connections": 20, // Max connections per IP + "max_requests": 20, // Max connections per IP per sweep interval + "sweep_interval": 1 // Time in seconds before resetting max_fetches and max_requests + }, + "server": { + "ip": "0.0.0.0", + "port": 51233, + // Max number of requests to queue up before rejecting further requests. + // Defaults to 0, which disables the limit. + "max_queue_size": 500, + // If request contains header with authorization, Clio will check if it matches the prefix 'Password ' + this value's sha256 hash + // If matches, the request will be considered as admin request + "admin_password": "xrp", + // If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests + // It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time + "local_admin": false, + "processing_policy": "parallel", // Could be "sequent" or "parallel". + // For sequent policy request from one client connection will be processed one by one and the next one will not be read before + // the previous one is processed. For parallel policy Clio will take all requests and process them in parallel and + // send a reply for each request whenever it is ready. + "parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified. + // Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection. + "ws_max_sending_queue_size": 1500, + "__ng_web_server": false // Use ng web server. This is a temporary setting which will be deleted after switching to ng web server + }, + // Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet. + "graceful_period": 10.0, + // Overrides log level on a per logging channel. + // Defaults to global "log_level" for each unspecified channel. + "log_channels": [ + { + "channel": "Backend", + "log_level": "fatal" + }, + { + "channel": "WebServer", + "log_level": "info" + }, + { + "channel": "Subscriptions", + "log_level": "info" + }, + { + "channel": "RPC", + "log_level": "error" + }, + { + "channel": "ETL", + "log_level": "debug" + }, + { + "channel": "Performance", + "log_level": "trace" + } + ], + "cache": { + // Configure this to use either "num_diffs", "num_cursors_from_diff", or "num_cursors_from_account". By default, Clio uses "num_diffs". + "num_diffs": 32, // Generate the cursors from the latest ledger diff, then use the cursors to partition the ledger to load concurrently. The cursors number is affected by the busyness of the network. + // "num_cursors_from_diff": 3200, // Read the cursors from the diff table until we have enough cursors to partition the ledger to load concurrently. + // "num_cursors_from_account": 3200, // Read the cursors from the account table until we have enough cursors to partition the ledger to load concurrently. + "num_markers": 48, // The number of markers is the number of coroutines to load the cache concurrently. + "page_fetch_size": 512, // The number of rows to load for each page. + "load": "async" // "sync" to load cache synchronously or "async" to load cache asynchronously or "none"/"no" to turn off the cache. + }, + "prometheus": { + "enabled": true, + "compress_reply": true + }, + "log_level": "info", + // Log format (this is the default format) + "log_format": "%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%", + "log_to_console": true, + // Clio logs to file in the specified directory only if "log_directory" is set + // "log_directory": "./clio_log", + "log_rotation_size": 2048, + "log_directory_max_size": 51200, + "log_rotation_hour_interval": 12, + "log_tag_style": "uint", + "extractor_threads": 8, + "read_only": false, + // "start_sequence": [integer] the ledger index to start from, + // "finish_sequence": [integer] the ledger index to finish at, + // "ssl_cert_file" : "/full/path/to/cert.file", + // "ssl_key_file" : "/full/path/to/key.file" + "api_version": { + "min": 1, // Minimum API version supported (could be 1 or 2) + "max": 2, // Maximum API version supported (could be 1 or 2, but >= min) + "default": 1 // Clio behaves the same as rippled by default + } } diff --git a/docs/examples/infrastructure/README.md b/docs/examples/infrastructure/README.md index 7e44d9aab..0fa19cbf0 100644 --- a/docs/examples/infrastructure/README.md +++ b/docs/examples/infrastructure/README.md @@ -4,16 +4,17 @@ > This is only an example of Grafana dashboard for Clio. It was created for demonstration purposes only and may contain errors. > Clio team would not recommend to relate on data from this dashboard or use it for monitoring your Clio instances. -This directory contains an example of docker based infrastructure to collect and visualise metrics from clio. +This directory contains an example of docker based infrastructure to collect and visualize metrics from clio. The structure of the directory: + - `compose.yaml` - Docker-compose file with Prometheus and Grafana set up. + Docker Compose file with Prometheus and Grafana set up. - `prometheus.yaml` Defines metrics collection from Clio and Prometheus itself. - Demonstrates how to setup Clio target and Clio's admin authorisation in Prometheus. + Demonstrates how to setup Clio target and Clio's admin authorization in Prometheus. - `grafana/clio_dashboard.json` - Json file containing preconfigured dashboard in Grafana format. + Json file containing pre-configured dashboard in Grafana format. - `grafana/dashboard_local.yaml` Grafana configuration file defining the directory to search for dashboards json files. - `grafana/datasources.yaml` @@ -21,9 +22,9 @@ The structure of the directory: ## How to try -1. Make sure you have `docker` and `docker-compose` installed. -2. Run `docker-compose up -d` from this directory. It will start docker containers with Prometheus and Grafana. +1. Make sure you have Docker (with `Docker Compose`) installed. +2. Run `docker compose up -d` from this directory. It will start docker containers with Prometheus and Grafana. 3. Open [http://localhost:3000/dashboards](http://localhost:3000/dashboards). Grafana login `admin`, password `grafana`. -There will be preconfigured Clio dashboard. + There will be pre-configured Clio dashboard. If Clio is not running yet launch Clio to see metrics. Some of the metrics may appear only after requests to Clio. diff --git a/docs/examples/infrastructure/compose.yaml b/docs/examples/infrastructure/compose.yaml index 681dadfbc..7e04013dc 100644 --- a/docs/examples/infrastructure/compose.yaml +++ b/docs/examples/infrastructure/compose.yaml @@ -6,7 +6,7 @@ services: volumes: - ./prometheus.yaml:/etc/prometheus/prometheus.yml command: - - '--config.file=/etc/prometheus/prometheus.yml' + - "--config.file=/etc/prometheus/prometheus.yml" grafana: image: grafana/grafana ports: diff --git a/docs/examples/infrastructure/grafana/clio_dashboard.json b/docs/examples/infrastructure/grafana/clio_dashboard.json index 315b1a2d3..ee0cdd163 100644 --- a/docs/examples/infrastructure/grafana/clio_dashboard.json +++ b/docs/examples/infrastructure/grafana/clio_dashboard.json @@ -80,9 +80,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -161,9 +159,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -246,9 +242,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -331,9 +325,7 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": [ - "lastNotNull" - ], + "calcs": ["lastNotNull"], "fields": "", "values": false }, @@ -1406,7 +1398,7 @@ "refId": "B" } ], - "title": "DB Opperations Error Rate", + "title": "DB Operations Error Rate", "type": "timeseries" }, { diff --git a/docs/examples/infrastructure/grafana/dashboard_local.yaml b/docs/examples/infrastructure/grafana/dashboard_local.yaml index c5051ee00..683bf200f 100644 --- a/docs/examples/infrastructure/grafana/dashboard_local.yaml +++ b/docs/examples/infrastructure/grafana/dashboard_local.yaml @@ -1,13 +1,13 @@ apiVersion: 1 providers: - - name: 'Clio dashboard' + - name: "Clio dashboard" # Org id. Default to 1 orgId: 1 # name of the dashboard folder. - folder: '' + folder: "" # folder UID. will be automatically generated if not specified - folderUid: '' + folderUid: "" # provider type. Default to 'file' type: file # disable dashboard deletion diff --git a/docs/examples/infrastructure/grafana/datasources.yaml b/docs/examples/infrastructure/grafana/datasources.yaml index 97659a52c..6c40b98e2 100644 --- a/docs/examples/infrastructure/grafana/datasources.yaml +++ b/docs/examples/infrastructure/grafana/datasources.yaml @@ -1,8 +1,8 @@ apiVersion: 1 datasources: -- name: Prometheus - type: prometheus - url: http://prometheus:9090 - isDefault: true - access: proxy + - name: Prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + access: proxy diff --git a/docs/examples/infrastructure/prometheus.yaml b/docs/examples/infrastructure/prometheus.yaml index dd1f16c1b..a6e3752c8 100644 --- a/docs/examples/infrastructure/prometheus.yaml +++ b/docs/examples/infrastructure/prometheus.yaml @@ -1,19 +1,19 @@ scrape_configs: -- job_name: clio - scrape_interval: 5s - scrape_timeout: 5s - authorization: - type: Password - # sha256sum from password `xrp` - # use echo -n 'your_password' | shasum -a 256 to get hash - credentials: 0e1dcf1ff020cceabf8f4a60a32e814b5b46ee0bb8cd4af5c814e4071bd86a18 - static_configs: - - targets: - - host.docker.internal:51233 -- job_name: prometheus - honor_timestamps: true - scrape_interval: 15s - scrape_timeout: 10s - static_configs: - - targets: - - localhost:9090 + - job_name: clio + scrape_interval: 5s + scrape_timeout: 5s + authorization: + type: Password + # sha256sum from password `xrp` + # use echo -n 'your_password' | shasum -a 256 to get hash + credentials: 0e1dcf1ff020cceabf8f4a60a32e814b5b46ee0bb8cd4af5c814e4071bd86a18 + static_configs: + - targets: + - host.docker.internal:51233 + - job_name: prometheus + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + static_configs: + - targets: + - localhost:9090 diff --git a/docs/logging.md b/docs/logging.md index 9aa5094e7..8e5de6fa3 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -8,12 +8,12 @@ The minimum level of severity at which the log message will be outputted by defa ## `log_format` - The format of log lines produced by Clio. Defaults to `"%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%"`. +The format of log lines produced by Clio. Defaults to `"%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%"`. Each of the variables expands like so: - `TimeStamp`: The full date and time of the log entry -- `SourceLocation`: A partial path to the c++ file and the line number in said file (`source/file/path:linenumber`) +- `SourceLocation`: A partial path to the c++ file and the line number in said file (`source/file/path:linenumber`) - `ThreadID`: The ID of the thread the log entry is written from - `Channel`: The channel that this log entry was sent to - `Severity`: The severity (aka log level) the entry was sent at @@ -30,8 +30,8 @@ Each object is of this format: ```json { - "channel": "Backend", - "log_level": "fatal" + "channel": "Backend", + "log_level": "fatal" } ``` diff --git a/docs/run-clio.md b/docs/run-clio.md index e5309a800..5b6e9ab45 100644 --- a/docs/run-clio.md +++ b/docs/run-clio.md @@ -3,6 +3,7 @@ ## Prerequisites - Access to a Cassandra cluster or ScyllaDB cluster. Can be local or remote. + > [!IMPORTANT] > There are some key considerations when using **ScyllaDB**. By default, Scylla reserves all free RAM on a machine for itself. If you are running `rippled` or other services on the same machine, restrict its memory usage using the `--memory` argument. > @@ -91,4 +92,4 @@ To completely disable Prometheus metrics add `"prometheus": { "enabled": false } It is important to know that Clio responds to Prometheus request only if they are admin requests. If you are using the admin password feature, the same password should be provided in the Authorization header of Prometheus requests. -You can find an example docker-compose file, with Prometheus and Grafana configs, in [examples/infrastructure](../docs/examples/infrastructure/). +You can find an example Docker Compose file, with Prometheus and Grafana configs, in [examples/infrastructure](../docs/examples/infrastructure/). diff --git a/docs/trouble_shooting.md b/docs/trouble_shooting.md index dd5247a43..be524ed86 100644 --- a/docs/trouble_shooting.md +++ b/docs/trouble_shooting.md @@ -1,47 +1,60 @@ # Troubleshooting Guide + This guide will help you troubleshoot common issues of Clio. ## Can't connect to DB + If you see the error log message `Could not connect to Cassandra: No hosts available`, this means that Clio can't connect to the database. Check the following: + - Make sure the database is running at the specified address and port. - Make sure the database is accessible from the machine where Clio is running. -You can use [cqlsh](https://pypi.org/project/cqlsh/) to check the connection to the database. + You can use [cqlsh](https://pypi.org/project/cqlsh/) to check the connection to the database. + If you would like to run a local ScyllaDB, you can call: + ```sh -docker run --rm -p 9042:9042 --name clio-scylla -d scylladb/scylla +docker run --rm -p 9042:9042 --name clio-scylla -d scylladb/scylla ``` ## Check the server status of Clio + To check if Clio is syncing with rippled: + ```sh curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep seq ``` + If Clio is syncing with rippled, the `seq` value will be increasing. ## Clio fails to start + If you see the error log message `Failed to fetch ETL state from...`, this means the configured rippled node is not reachable. Check the following: + - Make sure the rippled node is running at the specified address and port. - Make sure the rippled node is accessible from the machine where Clio is running. -If you would like to run Clio without an avaliable rippled node, you can add below setting to Clio's configuration file: -``` +If you would like to run Clio without an available rippled node, you can add below setting to Clio's configuration file: + +```text "allow_no_etl": true ``` ## Clio is not added to secure_gateway in rippled's config + If you see the warning message `AsyncCallData is_unlimited is false.`, this means that Clio is not added to the `secure_gateway` of `port_grpc` session in the rippled configuration file. It will slow down the sync process. Please add Clio's IP to the `secure_gateway` in the rippled configuration file for both grpc and ws port. ## Clio is slow + To speed up the response time, Clio has a cache inside. However, cache can take time to warm up. If you see slow response time, you can firstly check if cache is still loading. You can check the cache status by calling: + ```sh curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_full curl -v -d '{"method":"server_info", "params":[{}]}' 127.0.0.1:51233|python3 -m json.tool|grep is_enabled ``` -If `is_full` is false, it means the cache is still loading. Normally, the Clio can respond quicker if cache finishs loading. If `is_enabled` is false, it means the cache is disabled in the configuration file or there is data corruption in the database. + +If `is_full` is false, it means the cache is still loading. Normally, the Clio can respond quicker if cache finishes loading. If `is_enabled` is false, it means the cache is disabled in the configuration file or there is data corruption in the database. ## Receive error message `Too many requests` + If client sees the error message `Too many requests`, this means that the client is blocked by Clio's DosGuard protection. You may want to add the client's IP to the whitelist in the configuration file, Or update other your DosGuard settings. - - - diff --git a/.githooks/check-docs b/pre-commit-hooks/check-doxygen-docs.sh similarity index 86% rename from .githooks/check-docs rename to pre-commit-hooks/check-doxygen-docs.sh index 14f9c9ace..546fd259d 100755 --- a/.githooks/check-docs +++ b/pre-commit-hooks/check-doxygen-docs.sh @@ -2,7 +2,7 @@ # Note: This script is intended to be run from the root of the repository. # -# Not really a hook but should be used to check the completness of documentation for added code, otherwise CI will come for you. +# Not really a hook but should be used to check the completeness of documentation for added code, otherwise CI will come for you. # It's good to have /tmp as the output so that consecutive runs are fast but no clutter in the repository. echo "+ Checking documentation..." @@ -15,12 +15,18 @@ DOCDIR=${TMPDIR}/out # Check doxygen is at all installed if [ -z "$DOXYGEN" ]; then + if [[ "${CI}" == "true" ]]; then + # If we are in CI, we should fail the check + echo "doxygen not found in CI, please install it" + exit 1 + fi + # No hard error if doxygen is not installed yet cat < ... + +files="$@" +echo "+ Fixing includes in $files..." + +GNU_SED=$(sed --version 2>&1 | grep -q 'GNU' && echo true || echo false) + +if [[ "$GNU_SED" == "false" ]]; then # macOS sed + main_src_dirs=$(find ./src -maxdepth 1 -type d -exec basename {} \; | tr '\n' '|' | sed 's/|$//' | sed 's/|/\\|/g') +else + main_src_dirs=$(find ./src -maxdepth 1 -type d -exec basename {} \; | paste -sd '|' | sed 's/|/\\|/g') +fi + +fix_includes() { + file_path="$1" + + file_path_all_global="${file_path}.tmp.global" + file_path_fixed="${file_path}.tmp.fixed" + + # Make all includes to be <...> style + sed -E 's|#include "(.*)"|#include <\1>|g' "$file_path" > "$file_path_all_global" + + # Make local includes to be "..." style + sed -E "s|#include <(($main_src_dirs)/.*)>|#include \"\1\"|g" "$file_path_all_global" > "$file_path_fixed" + rm "$file_path_all_global" + + # Check if the temporary file is different from the original file + if ! cmp -s "$file_path" "$file_path_fixed"; then + # Replace the original file with the temporary file + mv "$file_path_fixed" "$file_path" + else + # Remove the temporary file if it's the same as the original + rm "$file_path_fixed" + fi +} + +for file in $files; do + fix_includes "$file" +done diff --git a/pre-commit-hooks/run-go-fmt.sh b/pre-commit-hooks/run-go-fmt.sh new file mode 100755 index 000000000..6f73c8944 --- /dev/null +++ b/pre-commit-hooks/run-go-fmt.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# +# Capture and print stdout, since gofmt doesn't use proper exit codes +# +set -e -o pipefail + +if ! command -v gofmt &> /dev/null ; then + echo "gofmt not installed or available in the PATH" >&2 + exit 1 +fi + +output="$(gofmt -l -w "$@")" +echo "$output" +[[ -z "$output" ]] diff --git a/.githooks/pre-push b/pre-commit-hooks/verify-commits.sh similarity index 54% rename from .githooks/pre-push rename to pre-commit-hooks/verify-commits.sh index 8d4f51a6c..2ebf170f4 100755 --- a/.githooks/pre-push +++ b/pre-commit-hooks/verify-commits.sh @@ -39,20 +39,14 @@ verify_tag_signed() { fi } -while read local_ref local_oid remote_ref remote_oid; do - # Check some things if we're pushing a branch called "release/" - if echo "$remote_ref" | grep ^refs\/heads\/release\/ &> /dev/null ; then - version=$(git tag --points-at HEAD) - echo "Looks like you're trying to push a $version release..." - echo "Making sure you've signed and tagged it." - if verify_commit_signed && verify_tag && verify_tag_signed ; then - : # Ok, I guess you can push - else - exit 1 - fi +# Check some things if we're pushing a branch called "release/" +if echo "$PRE_COMMIT_REMOTE_BRANCH" | grep ^refs\/heads\/release\/ &> /dev/null ; then + version=$(git tag --points-at HEAD) + echo "Looks like you're trying to push a $version release..." + echo "Making sure you've signed and tagged it." + if verify_commit_signed && verify_tag && verify_tag_signed ; then + : # Ok, I guess you can push + else + exit 1 fi -done - -command -v git-lfs >/dev/null 2>&1 || { echo >&2 "\nThis repository is configured for Git LFS but 'git-lfs' was not found on your path. If you no longer wish to use Git LFS, remove this hook by deleting the 'pre-push' file in the hooks directory (set by 'core.hookspath'; usually '.git/hooks').\n"; exit 2; } - -git lfs pre-push "$@" +fi diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 14333113a..a7d581772 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,6 @@ add_subdirectory(util) add_subdirectory(data) +add_subdirectory(cluster) add_subdirectory(etl) add_subdirectory(etlng) add_subdirectory(feed) diff --git a/src/app/CMakeLists.txt b/src/app/CMakeLists.txt index 40cbbbc14..3f9641eea 100644 --- a/src/app/CMakeLists.txt +++ b/src/app/CMakeLists.txt @@ -1,4 +1,13 @@ add_library(clio_app) target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp Stopper.cpp WebHandlers.cpp) -target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc clio_migration) +target_link_libraries( + clio_app + PUBLIC clio_cluster + clio_etl + clio_etlng + clio_feed + clio_web + clio_rpc + clio_migration +) diff --git a/src/app/CliArgs.cpp b/src/app/CliArgs.cpp index ce461c83b..374ee9865 100644 --- a/src/app/CliArgs.cpp +++ b/src/app/CliArgs.cpp @@ -21,7 +21,7 @@ #include "migration/MigrationApplication.hpp" #include "util/build/Build.hpp" -#include "util/newconfig/ConfigDescription.hpp" +#include "util/config/ConfigDescription.hpp" #include #include diff --git a/src/app/ClioApplication.cpp b/src/app/ClioApplication.cpp index e68a2797a..898c38172 100644 --- a/src/app/ClioApplication.cpp +++ b/src/app/ClioApplication.cpp @@ -21,12 +21,15 @@ #include "app/Stopper.hpp" #include "app/WebHandlers.hpp" +#include "cluster/ClusterCommunicationService.hpp" #include "data/AmendmentCenter.hpp" #include "data/BackendFactory.hpp" #include "data/LedgerCache.hpp" #include "etl/ETLService.hpp" #include "etl/LoadBalancer.hpp" #include "etl/NetworkValidatedLedgers.hpp" +#include "etlng/LoadBalancer.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManager.hpp" #include "migration/MigrationInspectorFactory.hpp" #include "rpc/Counters.hpp" @@ -34,14 +37,15 @@ #include "rpc/WorkQueue.hpp" #include "rpc/common/impl/HandlerProvider.hpp" #include "util/build/Build.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include "util/prometheus/Prometheus.hpp" #include "web/AdminVerificationStrategy.hpp" #include "web/RPCServerHandler.hpp" #include "web/Server.hpp" #include "web/dosguard/DOSGuard.hpp" #include "web/dosguard/IntervalSweepHandler.hpp" +#include "web/dosguard/Weights.hpp" #include "web/dosguard/WhitelistHandler.hpp" #include "web/ng/RPCServerHandler.hpp" #include "web/ng/Server.hpp" @@ -101,13 +105,17 @@ ClioApplication::run(bool const useNgWebServer) // Rate limiter, to prevent abuse auto whitelistHandler = web::dosguard::WhitelistHandler{config_}; - auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler}; + auto const dosguardWeights = web::dosguard::Weights::make(config_); + auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler, dosguardWeights}; auto sweepHandler = web::dosguard::IntervalSweepHandler{config_, ioc, dosGuard}; auto cache = data::LedgerCache{}; // Interface to the database auto backend = data::makeBackend(config_, cache); + cluster::ClusterCommunicationService clusterCommunicationService{backend}; + clusterCommunicationService.run(); + auto const amendmentCenter = std::make_shared(backend); { @@ -130,7 +138,12 @@ ClioApplication::run(bool const useNgWebServer) // ETL uses the balancer to extract data. // The server uses the balancer to forward RPCs to a rippled node. // The balancer itself publishes to streams (transactions_proposed and accounts_proposed) - auto balancer = etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers); + auto balancer = [&] -> std::shared_ptr { + if (config_.get("__ng_etl")) + return etlng::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers); + + return etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers); + }(); // ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes auto etl = etl::ETLService::makeETLService(config_, ioc, backend, subscriptions, balancer, ledgers); @@ -142,12 +155,12 @@ ClioApplication::run(bool const useNgWebServer) config_, backend, subscriptions, balancer, etl, amendmentCenter, counters ); - using RPCEngineType = rpc::RPCEngine; + using RPCEngineType = rpc::RPCEngine; auto const rpcEngine = RPCEngineType::makeRPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider); if (useNgWebServer or config_.get("server.__ng_web_server")) { - web::ng::RPCServerHandler handler{config_, backend, rpcEngine, etl}; + web::ng::RPCServerHandler handler{config_, backend, rpcEngine, etl, dosGuard}; auto expectedAdminVerifier = web::makeAdminVerificationStrategy(config_); if (not expectedAdminVerifier.has_value()) { @@ -165,7 +178,7 @@ ClioApplication::run(bool const useNgWebServer) httpServer->onGet("/metrics", MetricsHandler{adminVerifier}); httpServer->onGet("/health", HealthCheckHandler{}); - auto requestHandler = RequestHandler{adminVerifier, handler, dosGuard}; + auto requestHandler = RequestHandler{adminVerifier, handler}; httpServer->onPost("/", requestHandler); httpServer->onWs(std::move(requestHandler)); @@ -188,8 +201,7 @@ ClioApplication::run(bool const useNgWebServer) } // Init the web server - auto handler = - std::make_shared>(config_, backend, rpcEngine, etl); + auto handler = std::make_shared>(config_, backend, rpcEngine, etl, dosGuard); auto const httpServer = web::makeHttpServer(config_, ioc, dosGuard, handler); diff --git a/src/app/ClioApplication.hpp b/src/app/ClioApplication.hpp index 1eac02aae..871fcd6a5 100644 --- a/src/app/ClioApplication.hpp +++ b/src/app/ClioApplication.hpp @@ -21,7 +21,7 @@ #include "app/Stopper.hpp" #include "util/SignalsHandler.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" namespace app { diff --git a/src/app/Stopper.hpp b/src/app/Stopper.hpp index b4faa1377..daba1164e 100644 --- a/src/app/Stopper.hpp +++ b/src/app/Stopper.hpp @@ -20,8 +20,8 @@ #pragma once #include "data/BackendInterface.hpp" -#include "etl/ETLService.hpp" -#include "etl/LoadBalancer.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "util/CoroutineGroup.hpp" #include "util/log/Logger.hpp" @@ -74,15 +74,12 @@ public: * @param ioc The io_context to stop. * @return The callback to be called on application stop. */ - template < - web::ng::SomeServer ServerType, - etl::SomeLoadBalancer LoadBalancerType, - etl::SomeETLService ETLServiceType> + template static std::function makeOnStopCallback( ServerType& server, - LoadBalancerType& balancer, - ETLServiceType& etl, + etlng::LoadBalancerInterface& balancer, + etlng::ETLServiceInterface& etl, feed::SubscriptionManagerInterface& subscriptions, data::BackendInterface& backend, boost::asio::io_context& ioc diff --git a/src/app/VerifyConfig.hpp b/src/app/VerifyConfig.hpp index 9182072ec..298fbafa3 100644 --- a/src/app/VerifyConfig.hpp +++ b/src/app/VerifyConfig.hpp @@ -19,8 +19,8 @@ #pragma once -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ConfigFileJson.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ConfigFileJson.hpp" #include #include diff --git a/src/app/WebHandlers.hpp b/src/app/WebHandlers.hpp index ac7958ac0..5dacc222a 100644 --- a/src/app/WebHandlers.hpp +++ b/src/app/WebHandlers.hpp @@ -147,7 +147,6 @@ class RequestHandler { util::Logger webServerLog_{"WebServer"}; std::shared_ptr adminVerifier_; std::reference_wrapper rpcHandler_; - std::reference_wrapper dosguard_; public: /** @@ -155,14 +154,9 @@ public: * * @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access. * @param rpcHandler The RPC handler to use for handling the request. - * @param dosguard The DOSGuardInterface to use for checking the connection. */ - RequestHandler( - std::shared_ptr adminVerifier, - RpcHandlerType& rpcHandler, - web::dosguard::DOSGuardInterface& dosguard - ) - : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler), dosguard_(dosguard) + RequestHandler(std::shared_ptr adminVerifier, RpcHandlerType& rpcHandler) + : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler) { } @@ -183,21 +177,6 @@ public: boost::asio::yield_context yield ) { - if (not dosguard_.get().request(connectionMetadata.ip())) { - auto error = rpc::makeError(rpc::RippledError::rpcSLOW_DOWN); - - if (not request.isHttp()) { - try { - auto requestJson = boost::json::parse(request.message()); - if (requestJson.is_object() && requestJson.as_object().contains("id")) - error["id"] = requestJson.as_object().at("id"); - error["request"] = request.message(); - } catch (std::exception const&) { - error["request"] = request.message(); - } - } - return web::ng::Response{boost::beast::http::status::service_unavailable, error, request}; - } LOG(webServerLog_.info()) << connectionMetadata.tag() << "Received request from ip = " << connectionMetadata.ip() << " - posting to WorkQueue"; @@ -207,20 +186,7 @@ public: }); try { - auto response = rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield); - - if (not dosguard_.get().add(connectionMetadata.ip(), response.message().size())) { - auto jsonResponse = boost::json::parse(response.message()).as_object(); - jsonResponse["warning"] = "load"; - if (jsonResponse.contains("warnings") && jsonResponse["warnings"].is_array()) { - jsonResponse["warnings"].as_array().push_back(rpc::makeWarning(rpc::WarnRpcRateLimit)); - } else { - jsonResponse["warnings"] = boost::json::array{rpc::makeWarning(rpc::WarnRpcRateLimit)}; - } - response.setMessage(jsonResponse); - } - - return response; + return rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield); } catch (std::exception const&) { return web::ng::Response{ boost::beast::http::status::internal_server_error, diff --git a/src/cluster/CMakeLists.txt b/src/cluster/CMakeLists.txt new file mode 100644 index 000000000..defd5853e --- /dev/null +++ b/src/cluster/CMakeLists.txt @@ -0,0 +1,5 @@ +add_library(clio_cluster) + +target_sources(clio_cluster PRIVATE ClioNode.cpp ClusterCommunicationService.cpp) + +target_link_libraries(clio_cluster PRIVATE clio_util clio_data) diff --git a/src/cluster/ClioNode.cpp b/src/cluster/ClioNode.cpp new file mode 100644 index 000000000..e28585a90 --- /dev/null +++ b/src/cluster/ClioNode.cpp @@ -0,0 +1,64 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "cluster/ClioNode.hpp" + +#include "util/TimeUtils.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace cluster { + +namespace { + +struct Fields { + static constexpr std::string_view const kUPDATE_TIME = "update_time"; +}; + +} // namespace + +void +tag_invoke(boost::json::value_from_tag, boost::json::value& jv, ClioNode const& node) +{ + jv = { + {Fields::kUPDATE_TIME, util::systemTpToUtcStr(node.updateTime, ClioNode::kTIME_FORMAT)}, + }; +} + +ClioNode +tag_invoke(boost::json::value_to_tag, boost::json::value const& jv) +{ + auto const& updateTimeStr = jv.as_object().at(Fields::kUPDATE_TIME).as_string(); + auto const updateTime = util::systemTpFromUtcStr(std::string(updateTimeStr), ClioNode::kTIME_FORMAT); + if (!updateTime.has_value()) { + throw std::runtime_error("Failed to parse update time"); + } + + return ClioNode{.uuid = std::make_shared(), .updateTime = updateTime.value()}; +} + +} // namespace cluster diff --git a/src/cluster/ClioNode.hpp b/src/cluster/ClioNode.hpp new file mode 100644 index 000000000..a350a3715 --- /dev/null +++ b/src/cluster/ClioNode.hpp @@ -0,0 +1,58 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include +#include +#include + +#include +#include + +namespace cluster { + +/** + * @brief Represents a node in the cluster. + */ +struct ClioNode { + /** + * @brief The format of the time to store in the database. + */ + static constexpr char const* kTIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"; + + // enum class WriterRole { + // ReadOnly, + // NotWriter, + // Writer + // }; + + std::shared_ptr uuid; ///< The UUID of the node. + std::chrono::system_clock::time_point updateTime; ///< The time the data about the node was last updated. + + // WriterRole writerRole; +}; + +void +tag_invoke(boost::json::value_from_tag, boost::json::value& jv, ClioNode const& node); + +ClioNode +tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); + +} // namespace cluster diff --git a/src/cluster/ClusterCommunicationService.cpp b/src/cluster/ClusterCommunicationService.cpp new file mode 100644 index 000000000..bd1664493 --- /dev/null +++ b/src/cluster/ClusterCommunicationService.cpp @@ -0,0 +1,185 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "cluster/ClusterCommunicationService.hpp" + +#include "cluster/ClioNode.hpp" +#include "data/BackendInterface.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace cluster { + +ClusterCommunicationService::ClusterCommunicationService( + std::shared_ptr backend, + std::chrono::steady_clock::duration readInterval, + std::chrono::steady_clock::duration writeInterval +) + : backend_(std::move(backend)) + , readInterval_(readInterval) + , writeInterval_(writeInterval) + , selfData_{ClioNode{ + .uuid = std::make_shared(boost::uuids::random_generator{}()), + .updateTime = std::chrono::system_clock::time_point{} + }} +{ + nodesInClusterMetric_.set(1); // The node always sees itself + isHealthy_ = true; +} + +void +ClusterCommunicationService::run() +{ + boost::asio::spawn(strand_, [this](boost::asio::yield_context yield) { + boost::asio::steady_timer timer(yield.get_executor()); + while (true) { + timer.expires_after(readInterval_); + timer.async_wait(yield); + doRead(yield); + } + }); + + boost::asio::spawn(strand_, [this](boost::asio::yield_context yield) { + boost::asio::steady_timer timer(yield.get_executor()); + while (true) { + doWrite(); + timer.expires_after(writeInterval_); + timer.async_wait(yield); + } + }); +} + +ClusterCommunicationService::~ClusterCommunicationService() +{ + stop(); +} + +void +ClusterCommunicationService::stop() +{ + if (stopped_) + return; + + ctx_.stop(); + ctx_.join(); + stopped_ = true; +} + +std::shared_ptr +ClusterCommunicationService::selfUuid() const +{ + // Uuid never changes so it is safe to copy it without using strand_ + return selfData_.uuid; +} + +ClioNode +ClusterCommunicationService::selfData() const +{ + ClioNode result{}; + boost::asio::spawn(strand_, [this, &result](boost::asio::yield_context) { result = selfData_; }); + return result; +} + +std::expected, std::string> +ClusterCommunicationService::clusterData() const +{ + if (not isHealthy_) { + return std::unexpected{"Service is not healthy"}; + } + std::vector result; + boost::asio::spawn(strand_, [this, &result](boost::asio::yield_context) { + result = otherNodesData_; + result.push_back(selfData_); + }); + return result; +} + +void +ClusterCommunicationService::doRead(boost::asio::yield_context yield) +{ + otherNodesData_.clear(); + + BackendInterface::ClioNodesDataFetchResult expectedResult; + try { + expectedResult = backend_->fetchClioNodesData(yield); + } catch (...) { + expectedResult = std::unexpected{"Failed to fecth Clio nodes data"}; + } + + if (!expectedResult.has_value()) { + LOG(log_.error()) << "Failed to fetch nodes data"; + isHealthy_ = false; + return; + } + + // Create a new vector here to not have partially parsed data in otherNodesData_ + std::vector otherNodesData; + for (auto const& [uuid, nodeDataStr] : expectedResult.value()) { + if (uuid == *selfData_.uuid) { + continue; + } + + boost::system::error_code errorCode; + auto const json = boost::json::parse(nodeDataStr, errorCode); + if (errorCode.failed()) { + LOG(log_.error()) << "Error parsing json from DB: " << nodeDataStr; + isHealthy_ = false; + return; + } + + auto expectedNodeData = boost::json::try_value_to(json); + if (expectedNodeData.has_error()) { + LOG(log_.error()) << "Error converting json to ClioNode: " << json; + isHealthy_ = false; + return; + } + *expectedNodeData->uuid = uuid; + otherNodesData.push_back(std::move(expectedNodeData).value()); + } + otherNodesData_ = std::move(otherNodesData); + nodesInClusterMetric_.set(otherNodesData_.size() + 1); + isHealthy_ = true; +} + +void +ClusterCommunicationService::doWrite() +{ + selfData_.updateTime = std::chrono::system_clock::now(); + boost::json::value jsonValue{}; + boost::json::value_from(selfData_, jsonValue); + backend_->writeNodeMessage(*selfData_.uuid, boost::json::serialize(jsonValue.as_object())); +} + +} // namespace cluster diff --git a/src/cluster/ClusterCommunicationService.hpp b/src/cluster/ClusterCommunicationService.hpp new file mode 100644 index 000000000..0b1960885 --- /dev/null +++ b/src/cluster/ClusterCommunicationService.hpp @@ -0,0 +1,142 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "cluster/ClioNode.hpp" +#include "cluster/ClusterCommunicationServiceInterface.hpp" +#include "data/BackendInterface.hpp" +#include "util/log/Logger.hpp" +#include "util/prometheus/Bool.hpp" +#include "util/prometheus/Gauge.hpp" +#include "util/prometheus/Prometheus.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace cluster { + +/** + * @brief Service to post and read messages to/from the cluster. It uses a backend to communicate with the cluster. + */ +class ClusterCommunicationService : public ClusterCommunicationServiceInterface { + util::prometheus::GaugeInt& nodesInClusterMetric_ = PrometheusService::gaugeInt( + "cluster_nodes_total_number", + {}, + "Total number of nodes this node can detect in the cluster." + ); + util::prometheus::Bool isHealthy_ = PrometheusService::boolMetric( + "cluster_communication_is_healthy", + {}, + "Whether cluster communication service is operating healthy (1 - healthy, 0 - we have a problem)" + ); + + // TODO: Use util::async::CoroExecutionContext after https://github.com/XRPLF/clio/issues/1973 is implemented + boost::asio::thread_pool ctx_{1}; + boost::asio::strand strand_ = boost::asio::make_strand(ctx_); + + util::Logger log_{"ClusterCommunication"}; + + std::shared_ptr backend_; + + std::chrono::steady_clock::duration readInterval_; + std::chrono::steady_clock::duration writeInterval_; + + ClioNode selfData_; + std::vector otherNodesData_; + + bool stopped_ = false; + +public: + static constexpr std::chrono::milliseconds kDEFAULT_READ_INTERVAL{2100}; + static constexpr std::chrono::milliseconds kDEFAULT_WRITE_INTERVAL{1200}; + /** + * @brief Construct a new Cluster Communication Service object. + * + * @param backend The backend to use for communication. + * @param readInterval The interval to read messages from the cluster. + * @param writeInterval The interval to write messages to the cluster. + */ + ClusterCommunicationService( + std::shared_ptr backend, + std::chrono::steady_clock::duration readInterval = kDEFAULT_READ_INTERVAL, + std::chrono::steady_clock::duration writeInterval = kDEFAULT_WRITE_INTERVAL + ); + + ~ClusterCommunicationService() override; + + /** + * @brief Start the service. + */ + void + run(); + + /** + * @brief Stop the service. + */ + void + stop(); + + ClusterCommunicationService(ClusterCommunicationService&&) = delete; + ClusterCommunicationService(ClusterCommunicationService const&) = delete; + ClusterCommunicationService& + operator=(ClusterCommunicationService&&) = delete; + ClusterCommunicationService& + operator=(ClusterCommunicationService const&) = delete; + + /** + * @brief Get the UUID of the current node. + * + * @return The UUID of the current node. + */ + std::shared_ptr + selfUuid() const; + + /** + * @brief Get the data of the current node. + * + * @return The data of the current node. + */ + ClioNode + selfData() const override; + + /** + * @brief Get the data of all nodes in the cluster (including self). + * + * @return The data of all nodes in the cluster or error if the service is not healthy. + */ + std::expected, std::string> + clusterData() const override; + +private: + void + doRead(boost::asio::yield_context yield); + + void + doWrite(); +}; + +} // namespace cluster diff --git a/src/cluster/ClusterCommunicationServiceInterface.hpp b/src/cluster/ClusterCommunicationServiceInterface.hpp new file mode 100644 index 000000000..6e79460c5 --- /dev/null +++ b/src/cluster/ClusterCommunicationServiceInterface.hpp @@ -0,0 +1,54 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "cluster/ClioNode.hpp" + +#include +#include +#include + +namespace cluster { + +/** + * @brief Interface for the cluster communication service. + */ +class ClusterCommunicationServiceInterface { +public: + virtual ~ClusterCommunicationServiceInterface() = default; + + /** + * @brief Get the data of the current node. + * + * @return The data of the current node. + */ + [[nodiscard]] virtual ClioNode + selfData() const = 0; + + /** + * @brief Get the data of all nodes in the cluster (including self). + * + * @return The data of all nodes in the cluster or error if the service is not healthy. + */ + [[nodiscard]] virtual std::expected, std::string> + clusterData() const = 0; +}; + +} // namespace cluster diff --git a/src/data/AmendmentCenter.hpp b/src/data/AmendmentCenter.hpp index 1aac59548..127ecd0b0 100644 --- a/src/data/AmendmentCenter.hpp +++ b/src/data/AmendmentCenter.hpp @@ -137,6 +137,8 @@ struct Amendments { REGISTER(fixInvalidTxFlags); REGISTER(fixFrozenLPTokenTransfer); REGISTER(DeepFreeze); + REGISTER(PermissionDelegation); + REGISTER(fixPayChanCancelAfter); // Obsolete but supported by libxrpl REGISTER(CryptoConditionsSuite); diff --git a/src/data/BackendFactory.hpp b/src/data/BackendFactory.hpp index 05237536c..2d247f815 100644 --- a/src/data/BackendFactory.hpp +++ b/src/data/BackendFactory.hpp @@ -23,8 +23,8 @@ #include "data/CassandraBackend.hpp" #include "data/LedgerCacheInterface.hpp" #include "data/cassandra/SettingsProvider.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include #include diff --git a/src/data/BackendInterface.cpp b/src/data/BackendInterface.cpp index e710b4aa3..dc5d19672 100644 --- a/src/data/BackendInterface.cpp +++ b/src/data/BackendInterface.cpp @@ -61,7 +61,7 @@ BackendInterface::finishWrites(std::uint32_t const ledgerSequence) LOG(gLog.debug()) << "Want finish writes for " << ledgerSequence; auto commitRes = doFinishWrites(); if (commitRes) { - LOG(gLog.debug()) << "Successfully commited. Updating range now to " << ledgerSequence; + LOG(gLog.debug()) << "Successfully committed. Updating range now to " << ledgerSequence; updateRange(ledgerSequence); } return commitRes; @@ -246,7 +246,7 @@ BackendInterface::fetchBookOffers( auto end = std::chrono::system_clock::now(); LOG(gLog.debug()) << "Fetching " << std::to_string(keys.size()) << " offers took " << std::to_string(getMillis(mid - begin)) << " milliseconds. Fetching next dir took " - << std::to_string(succMillis) << " milliseonds. Fetched next dir " << std::to_string(numSucc) + << std::to_string(succMillis) << " milliseconds. Fetched next dir " << std::to_string(numSucc) << " times" << " Fetching next page of dir took " << std::to_string(pageMillis) << " milliseconds" << ". num pages = " << std::to_string(numPages) << ". Fetching all objects took " diff --git a/src/data/BackendInterface.hpp b/src/data/BackendInterface.hpp index 5c376d798..b2e31c3e1 100644 --- a/src/data/BackendInterface.hpp +++ b/src/data/BackendInterface.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -68,7 +69,7 @@ public: static constexpr std::size_t kDEFAULT_WAIT_BETWEEN_RETRY = 500; /** - * @brief A helper function that catches DatabaseTimout exceptions and retries indefinitely. + * @brief A helper function that catches DatabaseTimeout exceptions and retries indefinitely. * * @tparam FnType The type of function object to execute * @param func The function object to execute @@ -397,7 +398,7 @@ public: * @brief Fetches a specific ledger object. * * Currently the real fetch happens in doFetchLedgerObject and fetchLedgerObject attempts to fetch from Cache first - * and only calls out to the real DB if a cache miss ocurred. + * and only calls out to the real DB if a cache miss occurred. * * @param key The key of the object * @param sequence The ledger sequence to fetch for @@ -511,7 +512,7 @@ public: * @param key The key to fetch for * @param ledgerSequence The ledger sequence to fetch for * @param yield The coroutine context - * @return The sucessor on success; nullopt otherwise + * @return The successor on success; nullopt otherwise */ std::optional fetchSuccessorObject(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const; @@ -525,7 +526,7 @@ public: * @param key The key to fetch for * @param ledgerSequence The ledger sequence to fetch for * @param yield The coroutine context - * @return The sucessor key on success; nullopt otherwise + * @return The successor key on success; nullopt otherwise */ std::optional fetchSuccessorKey(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const; @@ -536,7 +537,7 @@ public: * @param key The key to fetch for * @param ledgerSequence The ledger sequence to fetch for * @param yield The coroutine context - * @return The sucessor on success; nullopt otherwise + * @return The successor on success; nullopt otherwise */ virtual std::optional doFetchSuccessorKey(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0; @@ -568,6 +569,19 @@ public: virtual std::optional fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const = 0; + /** @brief Return type for fetchClioNodesData() method */ + using ClioNodesDataFetchResult = + std::expected>, std::string>; + + /** + * @brief Fetches the data of all nodes in the cluster. + * + * @param yield The coroutine context + *@return The data of all nodes in the cluster. + */ + [[nodiscard]] virtual ClioNodesDataFetchResult + fetchClioNodesData(boost::asio::yield_context yield) const = 0; + /** * @brief Synchronously fetches the ledger range from DB. * @@ -682,6 +696,15 @@ public: virtual void writeSuccessor(std::string&& key, std::uint32_t seq, std::string&& successor) = 0; + /** + * @brief Write a node message. Used by ClusterCommunicationService + * + * @param uuid The UUID of the node + * @param message The message to write + */ + virtual void + writeNodeMessage(boost::uuids::uuid const& uuid, std::string message) = 0; + /** * @brief Starts a write transaction with the DB. No-op for cassandra. * diff --git a/src/data/CMakeLists.txt b/src/data/CMakeLists.txt index a19533c38..8aa1979a0 100644 --- a/src/data/CMakeLists.txt +++ b/src/data/CMakeLists.txt @@ -5,6 +5,7 @@ target_sources( BackendCounters.cpp BackendInterface.cpp LedgerCache.cpp + LedgerHeaderCache.cpp cassandra/impl/Future.cpp cassandra/impl/Cluster.cpp cassandra/impl/Batch.cpp diff --git a/src/data/CassandraBackend.hpp b/src/data/CassandraBackend.hpp index 7ea02475c..0c4521019 100644 --- a/src/data/CassandraBackend.hpp +++ b/src/data/CassandraBackend.hpp @@ -22,6 +22,7 @@ #include "data/BackendInterface.hpp" #include "data/DBHelpers.hpp" #include "data/LedgerCacheInterface.hpp" +#include "data/LedgerHeaderCache.hpp" #include "data/Types.hpp" #include "data/cassandra/Concepts.hpp" #include "data/cassandra/Handle.hpp" @@ -36,6 +37,8 @@ #include #include +#include +#include #include #include #include @@ -60,6 +63,8 @@ #include #include +class CacheBackendCassandraTest; + namespace data::cassandra { /** @@ -69,21 +74,27 @@ namespace data::cassandra { * * @tparam SettingsProviderType The settings provider type to use * @tparam ExecutionStrategyType The execution strategy type to use + * @tparam FetchLedgerCacheType The ledger header cache type to use */ -template +template < + SomeSettingsProvider SettingsProviderType, + SomeExecutionStrategy ExecutionStrategyType, + typename FetchLedgerCacheType = FetchLedgerCache> class BasicCassandraBackend : public BackendInterface { util::Logger log_{"Backend"}; SettingsProviderType settingsProvider_; Schema schema_; - std::atomic_uint32_t ledgerSequence_ = 0u; + friend class ::CacheBackendCassandraTest; protected: Handle handle_; // have to be mutable because BackendInterface constness :( mutable ExecutionStrategyType executor_; + // TODO: move to interface level + mutable FetchLedgerCacheType ledgerCache_{}; public: /** @@ -127,7 +138,6 @@ public: LOG(log_.error()) << error; throw std::runtime_error(error); } - LOG(log_.info()) << "Created (revamped) CassandraBackend"; } @@ -261,11 +271,16 @@ public: std::optional fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const override { + if (auto const lock = ledgerCache_.get(); lock.has_value() && lock->seq == sequence) + return lock->ledger; + auto const res = executor_.read(yield, schema_->selectLedgerBySeq, sequence); if (res) { if (auto const& result = res.value(); result) { if (auto const maybeValue = result.template get>(); maybeValue) { - return util::deserializeHeader(ripple::makeSlice(*maybeValue)); + auto const header = util::deserializeHeader(ripple::makeSlice(*maybeValue)); + ledgerCache_.put(FetchLedgerCache::CacheEntry{header, sequence}); + return header; } LOG(log_.error()) << "Could not fetch ledger by sequence - no rows"; @@ -778,7 +793,7 @@ public: while (liveAccounts.size() < number) { Statement const statement = lastItem ? schema_->selectAccountFromToken.bind(*lastItem, Limit{pageSize}) - : schema_->selectAccountFromBegining.bind(Limit{pageSize}); + : schema_->selectAccountFromBeginning.bind(Limit{pageSize}); auto const res = executor_.read(yield, statement); if (res) { @@ -878,6 +893,22 @@ public: return {}; } + std::expected>, std::string> + fetchClioNodesData(boost::asio::yield_context yield) const override + { + auto const readResult = executor_.read(yield, schema_->selectClioNodesData); + if (not readResult) + return std::unexpected{readResult.error().message()}; + + std::vector> result; + + for (auto [uuid, message] : extract(*readResult)) { + result.emplace_back(uuid, std::move(message)); + } + + return result; + } + void doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override { @@ -1032,6 +1063,12 @@ public: ); } + void + writeNodeMessage(boost::uuids::uuid const& uuid, std::string message) override + { + executor_.writeSync(schema_->updateClioNodeMessage, data::cassandra::Text{std::move(message)}, uuid); + } + bool isTooBusy() const override { diff --git a/src/data/LedgerCache.cpp b/src/data/LedgerCache.cpp index 7180c4e9d..92b5f87c6 100644 --- a/src/data/LedgerCache.cpp +++ b/src/data/LedgerCache.cpp @@ -63,7 +63,7 @@ LedgerCache::update(std::vector const& objs, uint32_t seq, bool is if (seq > latestSeq_) { ASSERT( seq == latestSeq_ + 1 || latestSeq_ == 0, - "New sequense must be either next or first. seq = {}, latestSeq_ = {}", + "New sequence must be either next or first. seq = {}, latestSeq_ = {}", seq, latestSeq_ ); diff --git a/src/data/LedgerHeaderCache.cpp b/src/data/LedgerHeaderCache.cpp new file mode 100644 index 000000000..0102d4f5c --- /dev/null +++ b/src/data/LedgerHeaderCache.cpp @@ -0,0 +1,46 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "data/LedgerHeaderCache.hpp" + +#include "util/Mutex.hpp" + +#include +#include +#include + +namespace data { + +FetchLedgerCache::FetchLedgerCache() = default; + +void +FetchLedgerCache::put(CacheEntry const& cacheEntry) +{ + auto lock = mutex_.lock(); + *lock = cacheEntry; +} + +std::optional +FetchLedgerCache::get() const +{ + auto const lock = mutex_.lock(); + return lock.get(); +} + +} // namespace data diff --git a/src/data/LedgerHeaderCache.hpp b/src/data/LedgerHeaderCache.hpp new file mode 100644 index 000000000..9bed26304 --- /dev/null +++ b/src/data/LedgerHeaderCache.hpp @@ -0,0 +1,85 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "util/Mutex.hpp" + +#include + +#include +#include +#include + +namespace data { + +/** + * @brief A simple cache holding one `ripple::LedgerHeader` to reduce DB lookups. + * + * Used internally by backend implementations. When a ledger header is + * fetched via `FetchLedgerBySeq` (often triggered by RPC commands), + * the result can be stored here. Subsequent requests for the same ledger + * sequence can proceed to retrieve the header from this cache, avoiding unnecessary + * database reads and improving performance. + */ +class FetchLedgerCache { +public: + FetchLedgerCache(); + + /** + * @brief Struct to store ledger header cache entry and the sequence it belongs to + */ + struct CacheEntry { + ripple::LedgerHeader ledger; + uint32_t seq{}; + + /** + * @brief Comparing CacheEntry. Used in testing for EXPECT_CALL + * + * @param other The other cacheEntry to compare + * @return true if two CacheEntry is the same, false otherwise + */ + bool + operator==(CacheEntry const& other) const + { + return ledger.hash == other.ledger.hash && seq == other.seq; + } + }; + + /** + * @brief Put CacheEntry into thread-safe container + * + * @param cacheEntry The Cache to store into thread-safe container. + */ + void + put(CacheEntry const& cacheEntry); + + /** + * @brief Read CacheEntry from thread-safe container. + * + * @return Optional CacheEntry, depending on if it exists in thread-safe container or not. + */ + std::optional + get() const; + +private: + mutable util::Mutex, std::shared_mutex> mutex_; +}; + +} // namespace data diff --git a/src/data/README.md b/src/data/README.md index 757534748..93b2cb4b8 100644 --- a/src/data/README.md +++ b/src/data/README.md @@ -6,7 +6,7 @@ To support additional database types, you can create new classes that implement ## Data Model -The data model used by Clio to read and write ledger data is different from what `rippled` uses. `rippled` uses a novel data structure named [*SHAMap*](https://github.com/ripple/rippled/blob/master/src/ripple/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record. +The data model used by Clio to read and write ledger data is different from what `rippled` uses. `rippled` uses a novel data structure named [_SHAMap_](https://github.com/ripple/rippled/blob/master/src/ripple/shamap/README.md), which is a combination of a Merkle Tree and a Radix Trie. In a SHAMap, ledger objects are stored in the root vertices of the tree. Thus, looking up a record located at the leaf node of the SHAMap executes a tree search, where the path from the root node to the leaf node is the key of the record. `rippled` nodes can also generate a proof-tree by forming a subtree with all the path nodes and their neighbors, which can then be used to prove the existence of the leaf node data to other `rippled` nodes. In short, the main purpose of the SHAMap data structure is to facilitate the fast validation of data integrity between different decentralized `rippled` nodes. @@ -22,19 +22,19 @@ There are three main types of data in each XRP Ledger version: Due to the structural differences of the different types of databases, Clio may choose to represent these data types using a different schema for each unique database type. -### Keywords +### Keywords **Sequence**: A unique incrementing identification number used to label the different ledger versions. **Hash**: The SHA512-half (calculate SHA512 and take the first 256 bits) hash of various ledger data like the entire ledger or specific ledger objects. -**Ledger Object**: The [binary-encoded](https://xrpl.org/serialization.html) STObject containing specific data (i.e. metadata, transaction data). +**Ledger Object**: The [binary-encoded](https://xrpl.org/serialization.html) STObject containing specific data (i.e. metadata, transaction data). -**Metadata**: The data containing [detailed information](https://xrpl.org/transaction-metadata.html#transaction-metadata) of the outcome of a specific transaction, regardless of whether the transaction was successful. +**Metadata**: The data containing [detailed information](https://xrpl.org/transaction-metadata.html#transaction-metadata) of the outcome of a specific transaction, regardless of whether the transaction was successful. -**Transaction data**: The data containing the [full details](https://xrpl.org/transaction-common-fields.html) of a specific transaction. +**Transaction data**: The data containing the [full details](https://xrpl.org/transaction-common-fields.html) of a specific transaction. -**Object Index**: The pseudo-random unique identifier of a ledger object, created by hashing the data of the object. +**Object Index**: The pseudo-random unique identifier of a ledger object, created by hashing the data of the object. ## Cassandra Implementation @@ -58,11 +58,11 @@ Their schemas and how they work are detailed in the following sections. ### ledger_transactions -``` -CREATE TABLE clio.ledger_transactions ( - ledger_sequence bigint, # The sequence number of the ledger version - hash blob, # Hash of all the transactions on this ledger version - PRIMARY KEY (ledger_sequence, hash) +```sql +CREATE TABLE clio.ledger_transactions ( + ledger_sequence bigint, # The sequence number of the ledger version + hash blob, # Hash of all the transactions on this ledger version + PRIMARY KEY (ledger_sequence, hash) ) WITH CLUSTERING ORDER BY (hash ASC) ... ``` @@ -70,37 +70,37 @@ This table stores the hashes of all transactions in a given ledger sequence and ### transactions -``` -CREATE TABLE clio.transactions ( - hash blob PRIMARY KEY, # The transaction hash - date bigint, # Date of the transaction - ledger_sequence bigint, # The sequence that the transaction was validated - metadata blob, # Metadata of the transaction - transaction blob # Data of the transaction +```sql +CREATE TABLE clio.transactions ( + hash blob PRIMARY KEY, # The transaction hash + date bigint, # Date of the transaction + ledger_sequence bigint, # The sequence that the transaction was validated + metadata blob, # Metadata of the transaction + transaction blob # Data of the transaction ) ... ``` This table stores the full transaction and metadata of each ledger version with the transaction hash as the primary key. -To lookup all the transactions that were validated in a ledger version with sequence `n`, first get the all the transaction hashes in that ledger version by querying `SELECT * FROM ledger_transactions WHERE ledger_sequence = n;`. Then, iterate through the list of hashes and query `SELECT * FROM transactions WHERE hash = one_of_the_hash_from_the_list;` to get the detailed transaction data. +To lookup all the transactions that were validated in a ledger version with sequence `n`, first get the all the transaction hashes in that ledger version by querying `SELECT * FROM ledger_transactions WHERE ledger_sequence = n;`. Then, iterate through the list of hashes and query `SELECT * FROM transactions WHERE hash = one_of_the_hash_from_the_list;` to get the detailed transaction data. ### ledger_hashes -``` +```sql CREATE TABLE clio.ledger_hashes ( - hash blob PRIMARY KEY, # Hash of entire ledger version's data - sequence bigint # The sequence of the ledger version + hash blob PRIMARY KEY, # Hash of entire ledger version's data + sequence bigint # The sequence of the ledger version ) ... ``` -This table stores the hash of all ledger versions by their sequences. +This table stores the hash of all ledger versions by their sequences. ### ledger_range -``` +```sql CREATE TABLE clio.ledger_range ( - is_latest boolean PRIMARY KEY, # Whether this sequence is the stopping range - sequence bigint # The sequence number of the starting/stopping range + is_latest boolean PRIMARY KEY, # Whether this sequence is the stopping range + sequence bigint # The sequence number of the starting/stopping range ) ... ``` @@ -108,12 +108,12 @@ This table marks the range of ledger versions that is stored on this specific Ca ### objects -``` +```sql CREATE TABLE clio.objects ( - key blob, # Object index of the object - sequence bigint, # The sequence this object was last updated - object blob, # Data of the object - PRIMARY KEY (key, sequence) + key blob, # Object index of the object + sequence bigint, # The sequence this object was last updated + object blob, # Data of the object + PRIMARY KEY (key, sequence) ) WITH CLUSTERING ORDER BY (sequence DESC) ... ``` @@ -123,10 +123,10 @@ The table is updated when all data for a given ledger sequence has been written ### ledgers -``` +```sql CREATE TABLE clio.ledgers ( - sequence bigint PRIMARY KEY, # Sequence of the ledger version - header blob # Data of the header + sequence bigint PRIMARY KEY, # Sequence of the ledger version + header blob # Data of the header ) ... ``` @@ -134,11 +134,11 @@ This table stores the ledger header data of specific ledger versions by their se ### diff -``` +```sql CREATE TABLE clio.diff ( - seq bigint, # Sequence of the ledger version - key blob, # Hash of changes in the ledger version - PRIMARY KEY (seq, key) + seq bigint, # Sequence of the ledger version + key blob, # Hash of changes in the ledger version + PRIMARY KEY (seq, key) ) WITH CLUSTERING ORDER BY (key ASC) ... ``` @@ -146,12 +146,12 @@ This table stores the object index of all the changes in each ledger version. ### account_tx -``` +```sql CREATE TABLE clio.account_tx ( - account blob, - seq_idx frozen>, # Tuple of (ledger_index, transaction_index) - hash blob, # Hash of the transaction - PRIMARY KEY (account, seq_idx) + account blob, + seq_idx frozen>, # Tuple of (ledger_index, transaction_index) + hash blob, # Hash of the transaction + PRIMARY KEY (account, seq_idx) ) WITH CLUSTERING ORDER BY (seq_idx DESC) ... ``` @@ -159,18 +159,18 @@ This table stores the list of transactions affecting a given account. This inclu ### successor -``` +```sql CREATE TABLE clio.successor ( - key blob, # Object index - seq bigint, # The sequnce that this ledger object's predecessor and successor was updated - next blob, # Index of the next object that existed in this sequence - PRIMARY KEY (key, seq) + key blob, # Object index + seq bigint, # The sequence that this ledger object's predecessor and successor was updated + next blob, # Index of the next object that existed in this sequence + PRIMARY KEY (key, seq) ) WITH CLUSTERING ORDER BY (seq ASC) ... ``` This table is the important backbone of how histories of ledger objects are stored in Cassandra. The `successor` table stores the object index of all ledger objects that were validated on the XRP network along with the ledger sequence that the object was updated on. -As each key is ordered by the sequence, which is achieved by tracing through the table with a specific sequence number, Clio can recreate a Linked List data structure that represents all the existing ledger objects at that ledger sequence. The special values of `0x00...00` and `0xFF...FF` are used to label the *head* and *tail* of the Linked List in the successor table. +As each key is ordered by the sequence, which is achieved by tracing through the table with a specific sequence number, Clio can recreate a Linked List data structure that represents all the existing ledger objects at that ledger sequence. The special values of `0x00...00` and `0xFF...FF` are used to label the _head_ and _tail_ of the Linked List in the successor table. The diagram below showcases how tracing through the same table, but with different sequence parameter filtering, can result in different Linked List data representing the corresponding past state of the ledger objects. A query like `SELECT * FROM successor WHERE key = ? AND seq <= n ORDER BY seq DESC LIMIT 1;` can effectively trace through the successor table and get the Linked List of a specific sequence `n`. @@ -182,12 +182,12 @@ In each new ledger version with sequence `n`, a ledger object `v` can either be For all three of these operations, the procedure to update the successor table can be broken down into two steps: - 1. Trace through the Linked List of the previous sequence to find the ledger object `e` with the greatest object index smaller or equal than the `v`'s index. Save `e`'s `next` value (the index of the next ledger object) as `w`. +1. Trace through the Linked List of the previous sequence to find the ledger object `e` with the greatest object index smaller or equal than the `v`'s index. Save `e`'s `next` value (the index of the next ledger object) as `w`. - 2. If `v` is... - 1. Being **created**, add two new records of `seq=n` with one being `e` pointing to `v`, and `v` pointing to `w` (Linked List insertion operation). - 2. Being **modified**, do nothing. - 3. Being **deleted**, add a record of `seq=n` with `e` pointing to `v`'s `next` value (Linked List deletion operation). +2. If `v` is... + 1. Being **created**, add two new records of `seq=n` with one being `e` pointing to `v`, and `v` pointing to `w` (Linked List insertion operation). + 2. Being **modified**, do nothing. + 3. Being **deleted**, add a record of `seq=n` with `e` pointing to `v`'s `next` value (Linked List deletion operation). ## NFT data model @@ -195,13 +195,13 @@ In `rippled` NFTs are stored in `NFTokenPage` ledger objects. This object is imp ### nf_tokens -``` +```sql CREATE TABLE clio.nf_tokens ( - token_id blob, # The NFT's ID - sequence bigint, # Sequence of ledger version - owner blob, # The account ID of the owner of this NFT at this ledger - is_burned boolean, # True if token was burned in this ledger - PRIMARY KEY (token_id, sequence) + token_id blob, # The NFT's ID + sequence bigint, # Sequence of ledger version + owner blob, # The account ID of the owner of this NFT at this ledger + is_burned boolean, # True if token was burned in this ledger + PRIMARY KEY (token_id, sequence) ) WITH CLUSTERING ORDER BY (sequence DESC) ... ``` @@ -209,7 +209,7 @@ This table indexes NFT IDs with their owner at a given ledger. The example query below shows how you could search for the owner of token `N` at ledger `Y` and see whether the token was burned. -``` +```sql SELECT * FROM nf_tokens WHERE token_id = N AND seq <= Y ORDER BY seq DESC LIMIT 1; @@ -219,12 +219,12 @@ If the token is burned, the owner field indicates the account that owned the tok ### issuer_nf_tokens_v2 -``` +```sql CREATE TABLE clio.issuer_nf_tokens_v2 ( - issuer blob, # The NFT issuer's account ID - taxon bigint, # The NFT's token taxon - token_id blob, # The NFT's ID - PRIMARY KEY (issuer, taxon, token_id) + issuer blob, # The NFT issuer's account ID + taxon bigint, # The NFT's token taxon + token_id blob, # The NFT's ID + PRIMARY KEY (issuer, taxon, token_id) ) WITH CLUSTERING ORDER BY (taxon ASC, token_id ASC) ... ``` @@ -233,12 +233,12 @@ combination. This is useful for determining all the NFTs a specific account issu ### nf_token_uris -``` +```sql CREATE TABLE clio.nf_token_uris ( - token_id blob, # The NFT's ID - sequence bigint, # Sequence of ledger version - uri blob, # The NFT's URI - PRIMARY KEY (token_id, sequence) + token_id blob, # The NFT's ID + sequence bigint, # Sequence of ledger version + uri blob, # The NFT's URI + PRIMARY KEY (token_id, sequence) ) WITH CLUSTERING ORDER BY (sequence DESC) ... ``` @@ -252,12 +252,12 @@ A given NFT will have only one entry in this table (see caveat below), and will ### nf_token_transactions -``` +```sql CREATE TABLE clio.nf_token_transactions ( - token_id blob, # The NFT's ID - seq_idx tuple, # Tuple of (ledger_index, transaction_index) - hash blob, # Hash of the transaction - PRIMARY KEY (token_id, seq_idx) + token_id blob, # The NFT's ID + seq_idx tuple, # Tuple of (ledger_index, transaction_index) + hash blob, # Hash of the transaction + PRIMARY KEY (token_id, seq_idx) ) WITH CLUSTERING ORDER BY (seq_idx DESC) ... ``` @@ -265,12 +265,12 @@ The `nf_token_transactions` table serves as the NFT counterpart to `account_tx`, ### migrator_status -``` +```sql CREATE TABLE clio.migrator_status ( migrator_name TEXT, # The name of the migrator status TEXT, # The status of the migrator PRIMARY KEY (migrator_name) -) +) ``` The `migrator_status` table stores the status of the migratior in this database. If a migrator's status is `migrated`, it means this database has finished data migration for this migrator. diff --git a/src/data/cassandra/Handle.hpp b/src/data/cassandra/Handle.hpp index 6279a9800..cf1161fed 100644 --- a/src/data/cassandra/Handle.hpp +++ b/src/data/cassandra/Handle.hpp @@ -89,7 +89,7 @@ public: asyncConnect() const; /** - * @brief Synchonous version of the above. + * @brief Synchronous version of the above. * * See @ref asyncConnect() const for how this works. * @@ -108,7 +108,7 @@ public: asyncConnect(std::string_view keyspace) const; /** - * @brief Synchonous version of the above. + * @brief Synchronous version of the above. * * See @ref asyncConnect(std::string_view) const for how this works. * @@ -127,7 +127,7 @@ public: asyncDisconnect() const; /** - * @brief Synchonous version of the above. + * @brief Synchronous version of the above. * * See @ref asyncDisconnect() const for how this works. * @@ -146,7 +146,7 @@ public: asyncReconnect(std::string_view keyspace) const; /** - * @brief Synchonous version of the above. + * @brief Synchronous version of the above. * * See @ref asyncReconnect(std::string_view) const for how this works. * @@ -172,7 +172,7 @@ public: } /** - * @brief Synchonous version of the above. + * @brief Synchronous version of the above. * * See asyncExecute(std::string_view, Args&&...) const for how this works. * @@ -201,7 +201,7 @@ public: asyncExecuteEach(std::vector const& statements) const; /** - * @brief Synchonous version of the above. + * @brief Synchronous version of the above. * * See @ref asyncExecuteEach(std::vector const&) const for how this works. * @@ -227,7 +227,7 @@ public: } /** - * @brief Synchonous version of the above. + * @brief Synchronous version of the above. * * See asyncExecute(std::vector const&, Args&&...) const for how this works. * @@ -262,7 +262,7 @@ public: asyncExecute(StatementType const& statement, std::function&& cb) const; /** - * @brief Synchonous version of the above. + * @brief Synchronous version of the above. * * See @ref asyncExecute(StatementType const&) const for how this works. * @@ -282,7 +282,7 @@ public: asyncExecute(std::vector const& statements) const; /** - * @brief Synchonous version of the above. + * @brief Synchronous version of the above. * * See @ref asyncExecute(std::vector const&) const for how this works. * diff --git a/src/data/cassandra/Schema.hpp b/src/data/cassandra/Schema.hpp index 4ec0356f9..c696a139b 100644 --- a/src/data/cassandra/Schema.hpp +++ b/src/data/cassandra/Schema.hpp @@ -69,11 +69,11 @@ public: std::string createKeyspace = [this]() { return fmt::format( R"( - CREATE KEYSPACE IF NOT EXISTS {} + CREATE KEYSPACE IF NOT EXISTS {} WITH replication = {{ 'class': 'SimpleStrategy', 'replication_factor': '{}' - }} + }} AND durable_writes = True )", settingsProvider_.get().getKeyspace(), @@ -91,13 +91,13 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( - key blob, - sequence bigint, - object blob, - PRIMARY KEY (key, sequence) - ) - WITH CLUSTERING ORDER BY (sequence DESC) + ( + key blob, + sequence bigint, + object blob, + PRIMARY KEY (key, sequence) + ) + WITH CLUSTERING ORDER BY (sequence DESC) )", qualifiedTableName(settingsProvider_.get(), "objects") )); @@ -105,13 +105,13 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( - hash blob PRIMARY KEY, - ledger_sequence bigint, + ( + hash blob PRIMARY KEY, + ledger_sequence bigint, date bigint, - transaction blob, - metadata blob - ) + transaction blob, + metadata blob + ) )", qualifiedTableName(settingsProvider_.get(), "transactions") )); @@ -119,11 +119,11 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( - ledger_sequence bigint, - hash blob, - PRIMARY KEY (ledger_sequence, hash) - ) + ( + ledger_sequence bigint, + hash blob, + PRIMARY KEY (ledger_sequence, hash) + ) )", qualifiedTableName(settingsProvider_.get(), "ledger_transactions") )); @@ -131,12 +131,12 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( + ( key blob, - seq bigint, - next blob, - PRIMARY KEY (key, seq) - ) + seq bigint, + next blob, + PRIMARY KEY (key, seq) + ) )", qualifiedTableName(settingsProvider_.get(), "successor") )); @@ -144,11 +144,11 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( - seq bigint, + ( + seq bigint, key blob, - PRIMARY KEY (seq, key) - ) + PRIMARY KEY (seq, key) + ) )", qualifiedTableName(settingsProvider_.get(), "diff") )); @@ -156,12 +156,12 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( - account blob, - seq_idx tuple, + ( + account blob, + seq_idx tuple, hash blob, - PRIMARY KEY (account, seq_idx) - ) + PRIMARY KEY (account, seq_idx) + ) WITH CLUSTERING ORDER BY (seq_idx DESC) )", qualifiedTableName(settingsProvider_.get(), "account_tx") @@ -170,10 +170,10 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( + ( sequence bigint PRIMARY KEY, header blob - ) + ) )", qualifiedTableName(settingsProvider_.get(), "ledgers") )); @@ -181,10 +181,10 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( + ( hash blob PRIMARY KEY, sequence bigint - ) + ) )", qualifiedTableName(settingsProvider_.get(), "ledger_hashes") )); @@ -192,7 +192,7 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( + ( is_latest boolean PRIMARY KEY, sequence bigint ) @@ -203,13 +203,13 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( - token_id blob, + ( + token_id blob, sequence bigint, owner blob, is_burned boolean, - PRIMARY KEY (token_id, sequence) - ) + PRIMARY KEY (token_id, sequence) + ) WITH CLUSTERING ORDER BY (sequence DESC) )", qualifiedTableName(settingsProvider_.get(), "nf_tokens") @@ -218,12 +218,12 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( + ( issuer blob, taxon bigint, token_id blob, PRIMARY KEY (issuer, taxon, token_id) - ) + ) WITH CLUSTERING ORDER BY (taxon ASC, token_id ASC) )", qualifiedTableName(settingsProvider_.get(), "issuer_nf_tokens_v2") @@ -232,12 +232,12 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( + ( token_id blob, sequence bigint, uri blob, PRIMARY KEY (token_id, sequence) - ) + ) WITH CLUSTERING ORDER BY (sequence DESC) )", qualifiedTableName(settingsProvider_.get(), "nf_token_uris") @@ -246,12 +246,12 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( - token_id blob, + ( + token_id blob, seq_idx tuple, hash blob, - PRIMARY KEY (token_id, seq_idx) - ) + PRIMARY KEY (token_id, seq_idx) + ) WITH CLUSTERING ORDER BY (seq_idx DESC) )", qualifiedTableName(settingsProvider_.get(), "nf_token_transactions") @@ -260,11 +260,11 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( + ( mpt_id blob, holder blob, PRIMARY KEY (mpt_id, holder) - ) + ) WITH CLUSTERING ORDER BY (holder ASC) )", qualifiedTableName(settingsProvider_.get(), "mp_token_holders") @@ -273,15 +273,28 @@ public: statements.emplace_back(fmt::format( R"( CREATE TABLE IF NOT EXISTS {} - ( + ( migrator_name TEXT, status TEXT, PRIMARY KEY (migrator_name) - ) + ) )", qualifiedTableName(settingsProvider_.get(), "migrator_status") )); + statements.emplace_back(fmt::format( + R"( + CREATE TABLE IF NOT EXISTS {} + ( + node_id UUID, + message TEXT, + PRIMARY KEY (node_id) + ) + WITH default_time_to_live = 2 + )", + qualifiedTableName(settingsProvider_.get(), "nodes_chat") + )); + return statements; }(); @@ -311,7 +324,7 @@ public: PreparedStatement insertObject = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (key, sequence, object) VALUES (?, ?, ?) )", @@ -322,7 +335,7 @@ public: PreparedStatement insertTransaction = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (hash, ledger_sequence, date, transaction, metadata) VALUES (?, ?, ?, ?, ?) )", @@ -333,7 +346,7 @@ public: PreparedStatement insertLedgerTransaction = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (ledger_sequence, hash) VALUES (?, ?) )", @@ -344,7 +357,7 @@ public: PreparedStatement insertSuccessor = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (key, seq, next) VALUES (?, ?, ?) )", @@ -355,7 +368,7 @@ public: PreparedStatement insertDiff = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (seq, key) VALUES (?, ?) )", @@ -366,7 +379,7 @@ public: PreparedStatement insertAccountTx = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (account, seq_idx, hash) VALUES (?, ?, ?) )", @@ -377,7 +390,7 @@ public: PreparedStatement insertNFT = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (token_id, sequence, owner, is_burned) VALUES (?, ?, ?, ?) )", @@ -388,7 +401,7 @@ public: PreparedStatement insertIssuerNFT = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (issuer, taxon, token_id) VALUES (?, ?, ?) )", @@ -399,7 +412,7 @@ public: PreparedStatement insertNFTURI = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (token_id, sequence, uri) VALUES (?, ?, ?) )", @@ -410,7 +423,7 @@ public: PreparedStatement insertNFTTx = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (token_id, seq_idx, hash) VALUES (?, ?, ?) )", @@ -421,7 +434,7 @@ public: PreparedStatement insertMPTHolder = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (mpt_id, holder) VALUES (?, ?) )", @@ -432,7 +445,7 @@ public: PreparedStatement insertLedgerHeader = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (sequence, header) VALUES (?, ?) )", @@ -443,7 +456,7 @@ public: PreparedStatement insertLedgerHash = [this]() { return handle_.get().prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (hash, sequence) VALUES (?, ?) )", @@ -458,9 +471,9 @@ public: PreparedStatement updateLedgerRange = [this]() { return handle_.get().prepare(fmt::format( R"( - UPDATE {} + UPDATE {} SET sequence = ? - WHERE is_latest = ? + WHERE is_latest = ? IF sequence IN (?, null) )", qualifiedTableName(settingsProvider_.get(), "ledger_range") @@ -470,7 +483,7 @@ public: PreparedStatement deleteLedgerRange = [this]() { return handle_.get().prepare(fmt::format( R"( - UPDATE {} + UPDATE {} SET sequence = ? WHERE is_latest = False )", @@ -489,6 +502,17 @@ public: )); }(); + PreparedStatement updateClioNodeMessage = [this]() { + return handle_.get().prepare(fmt::format( + R"( + UPDATE {} + SET message = ? + WHERE node_id = ? + )", + qualifiedTableName(settingsProvider_.get(), "nodes_chat") + )); + }(); + // // Select queries // @@ -496,11 +520,11 @@ public: PreparedStatement selectSuccessor = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT next - FROM {} + SELECT next + FROM {} WHERE key = ? AND seq <= ? - ORDER BY seq DESC + ORDER BY seq DESC LIMIT 1 )", qualifiedTableName(settingsProvider_.get(), "successor") @@ -510,7 +534,7 @@ public: PreparedStatement selectDiff = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT key + SELECT key FROM {} WHERE seq = ? )", @@ -521,11 +545,11 @@ public: PreparedStatement selectObject = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT object, sequence - FROM {} + SELECT object, sequence + FROM {} WHERE key = ? AND sequence <= ? - ORDER BY sequence DESC + ORDER BY sequence DESC LIMIT 1 )", qualifiedTableName(settingsProvider_.get(), "objects") @@ -535,7 +559,7 @@ public: PreparedStatement selectTransaction = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT transaction, metadata, ledger_sequence, date + SELECT transaction, metadata, ledger_sequence, date FROM {} WHERE hash = ? )", @@ -546,9 +570,9 @@ public: PreparedStatement selectAllTransactionHashesInLedger = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT hash - FROM {} - WHERE ledger_sequence = ? + SELECT hash + FROM {} + WHERE ledger_sequence = ? )", qualifiedTableName(settingsProvider_.get(), "ledger_transactions") )); @@ -557,11 +581,11 @@ public: PreparedStatement selectLedgerPageKeys = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT key - FROM {} + SELECT key + FROM {} WHERE TOKEN(key) >= ? AND sequence <= ? - PER PARTITION LIMIT 1 + PER PARTITION LIMIT 1 LIMIT ? ALLOW FILTERING )", @@ -587,9 +611,9 @@ public: PreparedStatement getToken = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT TOKEN(key) - FROM {} - WHERE key = ? + SELECT TOKEN(key) + FROM {} + WHERE key = ? LIMIT 1 )", qualifiedTableName(settingsProvider_.get(), "objects") @@ -599,8 +623,8 @@ public: PreparedStatement selectAccountTx = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT hash, seq_idx - FROM {} + SELECT hash, seq_idx + FROM {} WHERE account = ? AND seq_idx < ? LIMIT ? @@ -609,13 +633,13 @@ public: )); }(); - PreparedStatement selectAccountFromBegining = [this]() { + PreparedStatement selectAccountFromBeginning = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT account - FROM {} + SELECT account + FROM {} WHERE token(account) > 0 - PER PARTITION LIMIT 1 + PER PARTITION LIMIT 1 LIMIT ? )", qualifiedTableName(settingsProvider_.get(), "account_tx") @@ -625,10 +649,10 @@ public: PreparedStatement selectAccountFromToken = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT account - FROM {} + SELECT account + FROM {} WHERE token(account) > token(?) - PER PARTITION LIMIT 1 + PER PARTITION LIMIT 1 LIMIT ? )", qualifiedTableName(settingsProvider_.get(), "account_tx") @@ -638,11 +662,11 @@ public: PreparedStatement selectAccountTxForward = [this]() { return handle_.get().prepare(fmt::format( R"( - SELECT hash, seq_idx - FROM {} + SELECT hash, seq_idx + FROM {} WHERE account = ? AND seq_idx > ? - ORDER BY seq_idx ASC + ORDER BY seq_idx ASC LIMIT ? )", qualifiedTableName(settingsProvider_.get(), "account_tx") @@ -653,7 +677,7 @@ public: return handle_.get().prepare(fmt::format( R"( SELECT sequence, owner, is_burned - FROM {} + FROM {} WHERE token_id = ? AND sequence <= ? ORDER BY sequence DESC @@ -667,7 +691,7 @@ public: return handle_.get().prepare(fmt::format( R"( SELECT uri - FROM {} + FROM {} WHERE token_id = ? AND sequence <= ? ORDER BY sequence DESC @@ -681,7 +705,7 @@ public: return handle_.get().prepare(fmt::format( R"( SELECT hash, seq_idx - FROM {} + FROM {} WHERE token_id = ? AND seq_idx < ? ORDER BY seq_idx DESC @@ -695,7 +719,7 @@ public: return handle_.get().prepare(fmt::format( R"( SELECT hash, seq_idx - FROM {} + FROM {} WHERE token_id = ? AND seq_idx >= ? ORDER BY seq_idx ASC @@ -709,7 +733,7 @@ public: return handle_.get().prepare(fmt::format( R"( SELECT token_id - FROM {} + FROM {} WHERE issuer = ? AND (taxon, token_id) > ? ORDER BY taxon ASC, token_id ASC @@ -723,7 +747,7 @@ public: return handle_.get().prepare(fmt::format( R"( SELECT token_id - FROM {} + FROM {} WHERE issuer = ? AND taxon = ? AND token_id > ? @@ -738,7 +762,7 @@ public: return handle_.get().prepare(fmt::format( R"( SELECT holder - FROM {} + FROM {} WHERE mpt_id = ? AND holder > ? ORDER BY holder ASC @@ -753,7 +777,7 @@ public: R"( SELECT sequence FROM {} - WHERE hash = ? + WHERE hash = ? LIMIT 1 )", qualifiedTableName(settingsProvider_.get(), "ledger_hashes") @@ -775,7 +799,7 @@ public: return handle_.get().prepare(fmt::format( R"( SELECT sequence - FROM {} + FROM {} WHERE is_latest = True )", qualifiedTableName(settingsProvider_.get(), "ledger_range") @@ -803,6 +827,16 @@ public: qualifiedTableName(settingsProvider_.get(), "migrator_status") )); }(); + + PreparedStatement selectClioNodesData = [this]() { + return handle_.get().prepare(fmt::format( + R"( + SELECT node_id, message + FROM {} + )", + qualifiedTableName(settingsProvider_.get(), "nodes_chat") + )); + }(); }; /** diff --git a/src/data/cassandra/SettingsProvider.cpp b/src/data/cassandra/SettingsProvider.cpp index 478a23483..0eda7215a 100644 --- a/src/data/cassandra/SettingsProvider.cpp +++ b/src/data/cassandra/SettingsProvider.cpp @@ -22,7 +22,7 @@ #include "data/cassandra/Types.hpp" #include "data/cassandra/impl/Cluster.hpp" #include "util/Constants.hpp" -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ObjectView.hpp" #include #include diff --git a/src/data/cassandra/SettingsProvider.hpp b/src/data/cassandra/SettingsProvider.hpp index 73ceb2fdd..fd3e9d93f 100644 --- a/src/data/cassandra/SettingsProvider.hpp +++ b/src/data/cassandra/SettingsProvider.hpp @@ -21,7 +21,7 @@ #include "data/cassandra/Types.hpp" #include "data/cassandra/impl/Cluster.hpp" -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ObjectView.hpp" #include #include diff --git a/src/data/cassandra/impl/AsyncExecutor.hpp b/src/data/cassandra/impl/AsyncExecutor.hpp index 6ffb35032..0ac1051d2 100644 --- a/src/data/cassandra/impl/AsyncExecutor.hpp +++ b/src/data/cassandra/impl/AsyncExecutor.hpp @@ -38,7 +38,7 @@ namespace data::cassandra::impl { /** - * @brief A query executor with a changable retry policy + * @brief A query executor with a changeable retry policy * * Note: this is a bit of an anti-pattern and should be done differently * eventually. diff --git a/src/data/cassandra/impl/ExecutionStrategy.hpp b/src/data/cassandra/impl/ExecutionStrategy.hpp index 0b95d3f8d..eecdb55a2 100644 --- a/src/data/cassandra/impl/ExecutionStrategy.hpp +++ b/src/data/cassandra/impl/ExecutionStrategy.hpp @@ -267,7 +267,7 @@ public: } /** - * @brief Non-blocking query execution used for writing data. Constrast with write, this method does not execute + * @brief Non-blocking query execution used for writing data. Contrast with write, this method does not execute * the statements in a batch. * * Retries forever with retry policy specified by @ref AsyncExecutor. diff --git a/src/data/cassandra/impl/Result.hpp b/src/data/cassandra/impl/Result.hpp index 6dd004b09..1926deb20 100644 --- a/src/data/cassandra/impl/Result.hpp +++ b/src/data/cassandra/impl/Result.hpp @@ -23,6 +23,8 @@ #include "data/cassandra/impl/Tuple.hpp" #include "util/UnsupportedType.hpp" +#include +#include #include #include #include @@ -47,7 +49,7 @@ inline Type extractColumn(CassRow const* row, std::size_t idx) { using std::to_string; - Type output; + Type output{}; auto throwErrorIfNeeded = [](CassError rc, std::string_view label) { if (rc != CASS_OK) { @@ -99,6 +101,14 @@ extractColumn(CassRow const* row, std::size_t idx) auto const rc = cass_value_get_int64(cass_row_get_column(row, idx), &out); throwErrorIfNeeded(rc, "Extract int64"); output = static_cast(out); + } else if constexpr (std::is_convertible_v) { + CassUuid uuid; + auto const rc = cass_value_get_uuid(cass_row_get_column(row, idx), &uuid); + throwErrorIfNeeded(rc, "Extract uuid"); + std::string uuidStr(CASS_UUID_STRING_LENGTH, '0'); + cass_uuid_string(uuid, uuidStr.data()); + uuidStr.pop_back(); // remove the last \0 character + output = boost::uuids::string_generator{}(uuidStr); } else { // type not supported for extraction static_assert(util::Unsupported); diff --git a/src/data/cassandra/impl/Statement.hpp b/src/data/cassandra/impl/Statement.hpp index c7692aa6d..27b080fc9 100644 --- a/src/data/cassandra/impl/Statement.hpp +++ b/src/data/cassandra/impl/Statement.hpp @@ -25,6 +25,8 @@ #include "data/cassandra/impl/Tuple.hpp" #include "util/UnsupportedType.hpp" +#include +#include #include #include #include @@ -135,9 +137,15 @@ public: } else if constexpr (std::is_same_v) { auto const rc = cass_statement_bind_int32(*this, idx, value.limit); throwErrorIfNeeded(rc, "Bind limit (int32)"); - } - // clio only uses bigint (int64_t) so we convert any incoming type - else if constexpr (std::is_convertible_v) { + } else if constexpr (std::is_convertible_v) { + auto const uuidStr = boost::uuids::to_string(value); + CassUuid cassUuid; + auto rc = cass_uuid_from_string(uuidStr.c_str(), &cassUuid); + throwErrorIfNeeded(rc, "CassUuid from string"); + rc = cass_statement_bind_uuid(*this, idx, cassUuid); + throwErrorIfNeeded(rc, "Bind boost::uuid"); + // clio only uses bigint (int64_t) so we convert any incoming type + } else if constexpr (std::is_convertible_v) { auto const rc = cass_statement_bind_int64(*this, idx, value); throwErrorIfNeeded(rc, "Bind int64"); } else { diff --git a/src/etl/CacheLoaderSettings.cpp b/src/etl/CacheLoaderSettings.cpp index 9f6a16232..608ff53c5 100644 --- a/src/etl/CacheLoaderSettings.cpp +++ b/src/etl/CacheLoaderSettings.cpp @@ -19,7 +19,7 @@ #include "etl/CacheLoaderSettings.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include diff --git a/src/etl/CacheLoaderSettings.hpp b/src/etl/CacheLoaderSettings.hpp index d14390581..f9d762299 100644 --- a/src/etl/CacheLoaderSettings.hpp +++ b/src/etl/CacheLoaderSettings.hpp @@ -19,7 +19,7 @@ #pragma once -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include diff --git a/src/etl/ETLHelpers.hpp b/src/etl/ETLHelpers.hpp index 8054ecb18..5152292b9 100644 --- a/src/etl/ETLHelpers.hpp +++ b/src/etl/ETLHelpers.hpp @@ -143,7 +143,7 @@ public: }; /** - * @brief Parititions the uint256 keyspace into numMarkers partitions, each of equal size. + * @brief Partitions the uint256 keyspace into numMarkers partitions, each of equal size. * * @param numMarkers Total markers to partition for * @return The markers diff --git a/src/etl/ETLService.cpp b/src/etl/ETLService.cpp index 6eca76339..6211e5ed1 100644 --- a/src/etl/ETLService.cpp +++ b/src/etl/ETLService.cpp @@ -22,11 +22,12 @@ #include "data/BackendInterface.hpp" #include "etl/CorruptionDetector.hpp" #include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "util/Assert.hpp" #include "util/Constants.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -43,6 +44,7 @@ #include namespace etl { + // Database must be populated when this starts std::optional ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors) @@ -98,7 +100,7 @@ ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors) } // Main loop of ETL. -// The software begins monitoring the ledgers that are validated by the nework. +// The software begins monitoring the ledgers that are validated by the network. // The member networkValidatedLedgers_ keeps track of the sequences of ledgers validated by the network. // Whenever a ledger is validated by the network, the software looks for that ledger in the database. Once the ledger is // found in the database, the software publishes that ledger to the ledgers stream. If a network validated ledger is not @@ -183,7 +185,7 @@ ETLService::publishNextSequence(uint32_t nextSequence) if (!success) { LOG(log_.warn()) << "Failed to publish ledger with sequence = " << nextSequence << " . Beginning ETL"; - // returns the most recent sequence published empty optional if no sequence was published + // returns the most recent sequence published. empty optional if no sequence was published std::optional lastPublished = runETLPipeline(nextSequence, extractorThreads_); LOG(log_.info()) << "Aborting ETL. Falling back to publishing"; @@ -265,7 +267,7 @@ ETLService::ETLService( boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr balancer, + std::shared_ptr balancer, std::shared_ptr ledgers ) : backend_(backend) @@ -281,7 +283,6 @@ ETLService::ETLService( finishSequence_ = config.maybeValue("finish_sequence"); state_.isReadOnly = config.get("read_only"); extractorThreads_ = config.get("extractor_threads"); - txnThreshold_ = config.get("txn_threshold"); // This should probably be done in the backend factory but we don't have state available until here backend_->setCorruptionDetector(CorruptionDetector{state_, backend->cache()}); diff --git a/src/etl/ETLService.hpp b/src/etl/ETLService.hpp index d1d865475..964258dbf 100644 --- a/src/etl/ETLService.hpp +++ b/src/etl/ETLService.hpp @@ -32,7 +32,12 @@ #include "etl/impl/LedgerLoader.hpp" #include "etl/impl/LedgerPublisher.hpp" #include "etl/impl/Transformer.hpp" +#include "etlng/ETLService.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancer.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" +#include "util/Assert.hpp" #include "util/log/Logger.hpp" #include @@ -41,7 +46,6 @@ #include #include -#include #include #include #include @@ -81,14 +85,13 @@ concept SomeETLService = std::derived_from; * the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring * to writing and from writing to monitoring, based on the activity of other processes running on different machines. */ -class ETLService : public ETLServiceTag { +class ETLService : public etlng::ETLServiceInterface, ETLServiceTag { // TODO: make these template parameters in ETLService - using LoadBalancerType = LoadBalancer; using DataPipeType = etl::impl::ExtractionDataPipe; using CacheLoaderType = etl::CacheLoader<>; - using LedgerFetcherType = etl::impl::LedgerFetcher; + using LedgerFetcherType = etl::impl::LedgerFetcher; using ExtractorType = etl::impl::Extractor; - using LedgerLoaderType = etl::impl::LedgerLoader; + using LedgerLoaderType = etl::impl::LedgerLoader; using LedgerPublisherType = etl::impl::LedgerPublisher; using AmendmentBlockHandlerType = etl::impl::AmendmentBlockHandler; using TransformerType = @@ -97,7 +100,7 @@ class ETLService : public ETLServiceTag { util::Logger log_{"ETL"}; std::shared_ptr backend_; - std::shared_ptr loadBalancer_; + std::shared_ptr loadBalancer_; std::shared_ptr networkValidatedLedgers_; std::uint32_t extractorThreads_ = 1; @@ -114,7 +117,6 @@ class ETLService : public ETLServiceTag { size_t numMarkers_ = 2; std::optional startSequence_; std::optional finishSequence_; - size_t txnThreshold_ = 0; public: /** @@ -132,7 +134,7 @@ public: boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr balancer, + std::shared_ptr balancer, std::shared_ptr ledgers ); @@ -154,20 +156,37 @@ public: * @param ledgers The network validated ledgers datastructure * @return A shared pointer to a new instance of ETLService */ - static std::shared_ptr + static std::shared_ptr makeETLService( util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr balancer, + std::shared_ptr balancer, std::shared_ptr ledgers ) { - auto etl = std::make_shared(config, ioc, backend, subscriptions, balancer, ledgers); - etl->run(); + std::shared_ptr ret; - return etl; + if (config.get("__ng_etl")) { + ASSERT( + std::dynamic_pointer_cast(balancer), + "LoadBalancer type must be etlng::LoadBalancer" + ); + ret = std::make_shared(config, backend, subscriptions, balancer, ledgers); + } else { + ASSERT( + std::dynamic_pointer_cast(balancer), "LoadBalancer type must be etl::LoadBalancer" + ); + ret = std::make_shared(config, ioc, backend, subscriptions, balancer, ledgers); + } + + // inject networkID into subscriptions, as transaction feed require it to inject CTID in response + if (auto const state = ret->getETLState(); state) + subscriptions->setNetworkID(state->networkID); + + ret->run(); + return ret; } /** @@ -184,7 +203,7 @@ public: * @note This method blocks until the ETL service has stopped. */ void - stop() + stop() override { LOG(log_.info()) << "Stop called"; @@ -203,7 +222,7 @@ public: * @return Time passed since last ledger close */ std::uint32_t - lastCloseAgeSeconds() const + lastCloseAgeSeconds() const override { return ledgerPublisher_.lastCloseAgeSeconds(); } @@ -214,7 +233,7 @@ public: * @return true if currently amendment blocked; false otherwise */ bool - isAmendmentBlocked() const + isAmendmentBlocked() const override { return state_.isAmendmentBlocked; } @@ -225,7 +244,7 @@ public: * @return true if corruption of DB was detected and cache was stopped. */ bool - isCorruptionDetected() const + isCorruptionDetected() const override { return state_.isCorruptionDetected; } @@ -236,7 +255,7 @@ public: * @return The state of ETL as a JSON object */ boost::json::object - getInfo() const + getInfo() const override { boost::json::object result; @@ -254,11 +273,17 @@ public: * @return The etl nodes' state, nullopt if etl nodes are not connected */ std::optional - getETLState() const noexcept + getETLState() const noexcept override { return loadBalancer_->getETLState(); } + /** + * @brief Start all components to run ETL service. + */ + void + run() override; + private: /** * @brief Run the ETL pipeline. @@ -315,7 +340,7 @@ private: /** * @brief Get the number of markers to use during the initial ledger download. * - * This is equivelent to the degree of parallelism during the initial ledger download. + * This is equivalent to the degree of parallelism during the initial ledger download. * * @return The number of markers */ @@ -325,12 +350,6 @@ private: return numMarkers_; } - /** - * @brief Start all components to run ETL service. - */ - void - run(); - /** * @brief Spawn the worker thread and start monitoring. */ diff --git a/src/etl/ETLState.cpp b/src/etl/ETLState.cpp index cd0bbe0f8..5b7f4ba6e 100644 --- a/src/etl/ETLState.cpp +++ b/src/etl/ETLState.cpp @@ -27,7 +27,6 @@ #include #include -#include namespace etl { @@ -40,7 +39,7 @@ tag_invoke(boost::json::value_to_tag, boost::json::value const& jv) if (jsonObject.contains(JS(result)) && jsonObject.at(JS(result)).as_object().contains(JS(info))) { auto const rippledInfo = jsonObject.at(JS(result)).as_object().at(JS(info)).as_object(); if (rippledInfo.contains(JS(network_id))) - state.networkID.emplace(boost::json::value_to(rippledInfo.at(JS(network_id)))); + state.networkID = boost::json::value_to(rippledInfo.at(JS(network_id))); } return state; diff --git a/src/etl/ETLState.hpp b/src/etl/ETLState.hpp index 0302aab5f..3b447d985 100644 --- a/src/etl/ETLState.hpp +++ b/src/etl/ETLState.hpp @@ -38,7 +38,12 @@ namespace etl { * @brief This class is responsible for fetching and storing the state of the ETL information, such as the network id */ struct ETLState { - std::optional networkID; + /* + * NOTE: Rippled NetworkID: Mainnet = 0; Testnet = 1; Devnet = 2 + * However, if rippled is running on neither of these (ie. standalone mode) rippled will default to 0, but + * is not included in the stateOpt response. Must manually add it here. + */ + uint32_t networkID{0}; /** * @brief Fetch the ETL state from the rippled server diff --git a/src/etl/LedgerFetcherInterface.hpp b/src/etl/LedgerFetcherInterface.hpp index 2ce1d39ae..bdab1efd0 100644 --- a/src/etl/LedgerFetcherInterface.hpp +++ b/src/etl/LedgerFetcherInterface.hpp @@ -40,7 +40,7 @@ struct LedgerFetcherInterface { /** * @brief Extract data for a particular ledger from an ETL source * - * This function continously tries to extract the specified ledger (using all available ETL sources) until the + * This function continuously tries to extract the specified ledger (using all available ETL sources) until the * extraction succeeds, or the server shuts down. * * @param seq sequence of the ledger to extract @@ -52,7 +52,7 @@ struct LedgerFetcherInterface { /** * @brief Extract diff data for a particular ledger from an ETL source. * - * This function continously tries to extract the specified ledger (using all available ETL sources) until the + * This function continuously tries to extract the specified ledger (using all available ETL sources) until the * extraction succeeds, or the server shuts down. * * @param seq sequence of the ledger to extract diff --git a/src/etl/LoadBalancer.cpp b/src/etl/LoadBalancer.cpp index 7dc0d1925..12b8877ff 100644 --- a/src/etl/LoadBalancer.cpp +++ b/src/etl/LoadBalancer.cpp @@ -23,16 +23,20 @@ #include "etl/ETLState.hpp" #include "etl/NetworkValidatedLedgersInterface.hpp" #include "etl/Source.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" #include "util/Assert.hpp" #include "util/CoroutineGroup.hpp" +#include "util/Profiler.hpp" #include "util/Random.hpp" #include "util/ResponseExpirationCache.hpp" +#include "util/config/ArrayView.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ObjectView.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ArrayView.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ObjectView.hpp" +#include "util/prometheus/Label.hpp" +#include "util/prometheus/Prometheus.hpp" #include #include @@ -53,13 +57,15 @@ #include #include #include +#include #include using namespace util::config; +using util::prometheus::Labels; namespace etl { -std::shared_ptr +std::shared_ptr LoadBalancer::makeLoadBalancer( ClioConfigDefinition const& config, boost::asio::io_context& ioc, @@ -82,6 +88,33 @@ LoadBalancer::LoadBalancer( std::shared_ptr validatedLedgers, SourceFactory sourceFactory ) + : forwardingCounters_{ + .successDuration = PrometheusService::counterInt( + "forwarding_duration_milliseconds_counter", + Labels({util::prometheus::Label{"status", "success"}}), + "The duration of processing successful forwarded requests" + ), + .failDuration = PrometheusService::counterInt( + "forwarding_duration_milliseconds_counter", + Labels({util::prometheus::Label{"status", "fail"}}), + "The duration of processing failed forwarded requests" + ), + .retries = PrometheusService::counterInt( + "forwarding_retries_counter", + Labels(), + "The number of retries before a forwarded request was successful. Initial attempt excluded" + ), + .cacheHit = PrometheusService::counterInt( + "forwarding_cache_hit_counter", + Labels(), + "The number of requests that we served from the cache" + ), + .cacheMiss = PrometheusService::counterInt( + "forwarding_cache_miss_counter", + Labels(), + "The number of requests that were not served from the cache" + ) + } { auto const forwardingCacheTimeout = config.get("forwarding.cache_timeout"); if (forwardingCacheTimeout > 0.f) { @@ -141,12 +174,11 @@ LoadBalancer::LoadBalancer( if (!stateOpt) { LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString() << " Please check the configuration and network"; - } else if (etlState_ && etlState_->networkID && stateOpt->networkID && - etlState_->networkID != stateOpt->networkID) { + } else if (etlState_ && etlState_->networkID != stateOpt->networkID) { checkOnETLFailure(fmt::format( "ETL sources must be on the same network. Source network id = {} does not match others network id = {}", - *(stateOpt->networkID), - *(etlState_->networkID) + stateOpt->networkID, + etlState_->networkID )); } else { etlState_ = stateOpt; @@ -169,18 +201,13 @@ LoadBalancer::LoadBalancer( } } -LoadBalancer::~LoadBalancer() -{ - sources_.clear(); -} - std::vector -LoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly, std::chrono::steady_clock::duration retryAfter) +LoadBalancer::loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter) { std::vector response; execute( - [this, &response, &sequence, cacheOnly](auto& source) { - auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, cacheOnly); + [this, &response, &sequence](auto& source) { + auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_); if (!res) { LOG(log_.error()) << "Failed to download initial ledger." @@ -227,7 +254,7 @@ LoadBalancer::fetchLedger( return response; } -std::expected +std::expected LoadBalancer::forwardToRippled( boost::json::object const& request, std::optional const& clientIp, @@ -239,40 +266,42 @@ LoadBalancer::forwardToRippled( return std::unexpected{rpc::ClioError::RpcCommandIsMissing}; auto const cmd = boost::json::value_to(request.at("command")); - if (forwardingCache_) { - if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) { - return std::move(cachedResponse).value(); + + if (forwardingCache_ and forwardingCache_->shouldCache(cmd)) { + bool servedFromCache = true; + auto updater = + [this, &request, &clientIp, &servedFromCache, isAdmin](boost::asio::yield_context yield + ) -> std::expected { + servedFromCache = false; + auto result = forwardToRippledImpl(request, clientIp, isAdmin, yield); + if (result.has_value()) { + return util::ResponseExpirationCache::EntryData{ + .lastUpdated = std::chrono::steady_clock::now(), .response = std::move(result).value() + }; + } + return std::unexpected{ + util::ResponseExpirationCache::Error{.status = rpc::Status{result.error()}, .warnings = {}} + }; + }; + + auto result = forwardingCache_->getOrUpdate( + yield, + cmd, + std::move(updater), + [](util::ResponseExpirationCache::EntryData const& entry) { return not entry.response.contains("error"); } + ); + if (servedFromCache) { + ++forwardingCounters_.cacheHit.get(); } - } - - ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests."); - std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1); - - auto numAttempts = 0u; - - auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE; - - std::optional response; - rpc::ClioError error = rpc::ClioError::EtlConnectionError; - while (numAttempts < sources_.size()) { - auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); - if (res) { - response = std::move(res).value(); - break; + if (result.has_value()) { + return std::move(result).value(); } - error = std::max(error, res.error()); // Choose the best result between all sources - - sourceIdx = (sourceIdx + 1) % sources_.size(); - ++numAttempts; + auto const combinedError = result.error().status.code; + ASSERT(std::holds_alternative(combinedError), "There could be only ClioError here"); + return std::unexpected{std::get(combinedError)}; } - if (response) { - if (forwardingCache_ and not response->contains("error")) - forwardingCache_->put(cmd, *response); - return std::move(response).value(); - } - - return std::unexpected{error}; + return forwardToRippledImpl(request, clientIp, isAdmin, yield); } boost::json::value @@ -363,4 +392,47 @@ LoadBalancer::chooseForwardingSource() } } +std::expected +LoadBalancer::forwardToRippledImpl( + boost::json::object const& request, + std::optional const& clientIp, + bool const isAdmin, + boost::asio::yield_context yield +) +{ + ++forwardingCounters_.cacheMiss.get(); + + ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests."); + std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1); + + auto numAttempts = 0u; + + auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE; + + std::optional response; + rpc::ClioError error = rpc::ClioError::EtlConnectionError; + while (numAttempts < sources_.size()) { + auto [res, duration] = + util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); }); + + if (res) { + forwardingCounters_.successDuration.get() += duration; + response = std::move(res).value(); + break; + } + forwardingCounters_.failDuration.get() += duration; + ++forwardingCounters_.retries.get(); + error = std::max(error, res.error()); // Choose the best result between all sources + + sourceIdx = (sourceIdx + 1) % sources_.size(); + ++numAttempts; + } + + if (response.has_value()) { + return std::move(response).value(); + } + + return std::unexpected{error}; +} + } // namespace etl diff --git a/src/etl/LoadBalancer.hpp b/src/etl/LoadBalancer.hpp index cfe01d019..8edb74788 100644 --- a/src/etl/LoadBalancer.hpp +++ b/src/etl/LoadBalancer.hpp @@ -23,12 +23,16 @@ #include "etl/ETLState.hpp" #include "etl/NetworkValidatedLedgersInterface.hpp" #include "etl/Source.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" +#include "util/Assert.hpp" #include "util/Mutex.hpp" #include "util/ResponseExpirationCache.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/prometheus/Counter.hpp" #include #include @@ -48,6 +52,7 @@ #include #include #include +#include #include namespace etl { @@ -69,7 +74,7 @@ concept SomeLoadBalancer = std::derived_from; * which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also * allows requests for ledger data to be load balanced across all possible ETL sources. */ -class LoadBalancer : public LoadBalancerTag { +class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag { public: using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject; using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse; @@ -88,7 +93,15 @@ private: std::uint32_t downloadRanges_ = kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */ - // Using mutext instead of atomic_bool because choosing a new source to + struct ForwardingCounters { + std::reference_wrapper successDuration; + std::reference_wrapper failDuration; + std::reference_wrapper retries; + std::reference_wrapper cacheHit; + std::reference_wrapper cacheMiss; + } forwardingCounters_; + + // Using mutex instead of atomic_bool because choosing a new source to // forward messages should be done with a mutual exclusion otherwise there will be a race condition util::Mutex hasForwardingSource_{false}; @@ -129,11 +142,11 @@ public: * @param ioc The io_context to run on * @param backend BackendInterface implementation * @param subscriptions Subscription manager - * @param validatedLedgers The network validated ledgers datastructure + * @param validatedLedgers The network validated ledgers data structure * @param sourceFactory A factory function to create a source * @return A shared pointer to a new instance of LoadBalancer */ - static std::shared_ptr + static std::shared_ptr makeLoadBalancer( util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, @@ -143,23 +156,37 @@ public: SourceFactory sourceFactory = makeSource ); - ~LoadBalancer() override; + /** + * @brief Load the initial ledger, writing data to the queue. + * @note This function will retry indefinitely until the ledger is downloaded. + * + * @param sequence Sequence of ledger to download + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return A std::vector The ledger data + */ + std::vector + loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}) + override; /** * @brief Load the initial ledger, writing data to the queue. * @note This function will retry indefinitely until the ledger is downloaded. * * @param sequence Sequence of ledger to download - * @param cacheOnly Whether to only write to cache and not to the DB; defaults to false + * @param observer The observer to notify of progress * @param retryAfter Time to wait between retries (2 seconds by default) * @return A std::vector The ledger data */ std::vector loadInitialLedger( - uint32_t sequence, - bool cacheOnly = false, - std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2} - ); + [[maybe_unused]] uint32_t sequence, + [[maybe_unused]] etlng::InitialLoadObserverInterface& observer, + [[maybe_unused]] std::chrono::steady_clock::duration retryAfter + ) override + { + ASSERT(false, "Not available for old ETL"); + std::unreachable(); + } /** * @brief Fetch data for a specific ledger. @@ -180,7 +207,7 @@ public: bool getObjects, bool getObjectNeighbors, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2} - ); + ) override; /** * @brief Represent the state of this load balancer as a JSON object @@ -188,7 +215,7 @@ public: * @return JSON representation of the state of this load balancer. */ boost::json::value - toJson() const; + toJson() const override; /** * @brief Forward a JSON RPC request to a randomly selected rippled node. @@ -199,20 +226,20 @@ public: * @param yield The coroutine context * @return Response received from rippled node as JSON object on success or error on failure */ - std::expected + std::expected forwardToRippled( boost::json::object const& request, std::optional const& clientIp, bool isAdmin, boost::asio::yield_context yield - ); + ) override; /** * @brief Return state of ETL nodes. * @return ETL state, nullopt if etl nodes not available */ std::optional - getETLState() noexcept; + getETLState() noexcept override; /** * @brief Stop the load balancer. This will stop all subscription sources. @@ -221,7 +248,7 @@ public: * @param yield The coroutine context */ void - stop(boost::asio::yield_context yield); + stop(boost::asio::yield_context yield) override; private: /** @@ -245,6 +272,14 @@ private: */ void chooseForwardingSource(); + + std::expected + forwardToRippledImpl( + boost::json::object const& request, + std::optional const& clientIp, + bool isAdmin, + boost::asio::yield_context yield + ); }; } // namespace etl diff --git a/src/etl/NetworkValidatedLedgersInterface.hpp b/src/etl/NetworkValidatedLedgersInterface.hpp index d2ab9d12b..067407c8c 100644 --- a/src/etl/NetworkValidatedLedgersInterface.hpp +++ b/src/etl/NetworkValidatedLedgersInterface.hpp @@ -26,6 +26,7 @@ #include #include + namespace etl { /** diff --git a/src/etl/README.md b/src/etl/README.md index e422cfe82..39e5575d2 100644 --- a/src/etl/README.md +++ b/src/etl/README.md @@ -23,17 +23,17 @@ For example, if segment **0x08581464C55B0B2C8C4FA27FA8DE0ED695D3BE019E7BE0969C92 Because of the nature of the Linked List, the cursors are crucial to balancing the workload of each coroutine. There are 3 types of cursor generation that can be used: -- **cache.num_diffs**: Cursors will be generated by the changed objects in the latest `cache.num_diffs` number of ledgers. The default value is 32. In *mainnet*, this type works well because the network is fairly busy and the number of changed objects in each ledger is relatively stable. Thus, we are able to get enough cursors after removing the deleted objects on *mainnet*. -For other networks, like the *devnet*, the number of changed objects in each ledger is not stable. When the network is silent, one coroutine may load a large number of objects while the other coroutines are idle. Below is a comparison of the number of cursors and loading time on *devnet*: +- **cache.num_diffs**: Cursors will be generated by the changed objects in the latest `cache.num_diffs` number of ledgers. The default value is 32. In _mainnet_, this type works well because the network is fairly busy and the number of changed objects in each ledger is relatively stable. Thus, we are able to get enough cursors after removing the deleted objects on _mainnet_. + For other networks, like the _devnet_, the number of changed objects in each ledger is not stable. When the network is silent, one coroutine may load a large number of objects while the other coroutines are idle. Below is a comparison of the number of cursors and loading time on _devnet_: - | Cursors | Loading time /seconds | - | ------- | --------------------- | - | 11 | 2072 | - | 33 | 983 | - | 120 | 953 | - | 200 | 843 | - | 250 | 816 | - | 500 | 792 | + | Cursors | Loading time /seconds | + | ------- | --------------------- | + | 11 | 2072 | + | 33 | 983 | + | 120 | 953 | + | 200 | 843 | + | 250 | 816 | + | 500 | 792 | - **cache.num_cursors_from_diff**: Cursors will be generated by the changed objects in the recent ledgers. The generator will keep reading the previous ledger until we have `cache.num_cursors_from_diff` cursors. This type is the evolved version of `cache.num_diffs`. It removes the network busyness factor and only considers the number of cursors. The cache loading can be well tuned by this configuration. diff --git a/src/etl/Source.cpp b/src/etl/Source.cpp index 5de0586c4..27b09d2fc 100644 --- a/src/etl/Source.cpp +++ b/src/etl/Source.cpp @@ -26,7 +26,7 @@ #include "etl/impl/SourceImpl.hpp" #include "etl/impl/SubscriptionSource.hpp" #include "feed/SubscriptionManagerInterface.hpp" -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ObjectView.hpp" #include diff --git a/src/etl/Source.hpp b/src/etl/Source.hpp index 91d9bf817..be33dcb66 100644 --- a/src/etl/Source.hpp +++ b/src/etl/Source.hpp @@ -23,9 +23,7 @@ #include "etl/NetworkValidatedLedgersInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" -#include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ObjectView.hpp" #include #include @@ -130,11 +128,10 @@ public: * * @param sequence Sequence of the ledger to download * @param numMarkers Number of markers to generate for async calls - * @param cacheOnly Only insert into cache, not the DB; defaults to false * @return A std::pair of the data and a bool indicating whether the download was successful */ virtual std::pair, bool> - loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) = 0; + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) = 0; /** * @brief Forward a request to rippled. diff --git a/src/etl/impl/ExtractionDataPipe.hpp b/src/etl/impl/ExtractionDataPipe.hpp index 1a5044483..4206a705c 100644 --- a/src/etl/impl/ExtractionDataPipe.hpp +++ b/src/etl/impl/ExtractionDataPipe.hpp @@ -66,7 +66,7 @@ public: /** * @brief Push new data package for the specified sequence. * - * Note: Potentially blocks until the underlying queue can accomodate another entry. + * Note: Potentially blocks until the underlying queue can accommodate another entry. * * @param sequence The sequence for which to enqueue the data package * @param data The data to store diff --git a/src/etl/impl/GrpcSource.cpp b/src/etl/impl/GrpcSource.cpp index 29be5b224..539cdf481 100644 --- a/src/etl/impl/GrpcSource.cpp +++ b/src/etl/impl/GrpcSource.cpp @@ -98,7 +98,7 @@ GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighb } std::pair, bool> -GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers, bool const cacheOnly) +GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers) { if (!stub_) return {{}, false}; @@ -130,7 +130,7 @@ GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers LOG(log_.trace()) << "Marker prefix = " << ptr->getMarkerPrefix(); - auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly); + auto result = ptr->process(stub_, cq, *backend_, abort); if (result != etl::impl::AsyncCallData::CallStatus::MORE) { ++numFinished; LOG(log_.debug()) << "Finished a marker. Current number of finished = " << numFinished; diff --git a/src/etl/impl/GrpcSource.hpp b/src/etl/impl/GrpcSource.hpp index 5d41f193d..248f4191d 100644 --- a/src/etl/impl/GrpcSource.hpp +++ b/src/etl/impl/GrpcSource.hpp @@ -60,11 +60,10 @@ public: * * @param sequence Sequence of the ledger to download * @param numMarkers Number of markers to generate for async calls - * @param cacheOnly Only insert into cache, not the DB; defaults to false * @return A std::pair of the data and a bool indicating whether the download was successful */ std::pair, bool> - loadInitialLedger(uint32_t sequence, uint32_t numMarkers, bool cacheOnly = false); + loadInitialLedger(uint32_t sequence, uint32_t numMarkers); }; } // namespace etl::impl diff --git a/src/etl/impl/LedgerFetcher.hpp b/src/etl/impl/LedgerFetcher.hpp index 9d8a0df88..162629246 100644 --- a/src/etl/impl/LedgerFetcher.hpp +++ b/src/etl/impl/LedgerFetcher.hpp @@ -20,6 +20,8 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etl/LedgerFetcherInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "util/log/Logger.hpp" #include @@ -34,22 +36,18 @@ namespace etl::impl { /** * @brief GRPC Ledger data fetcher */ -template -class LedgerFetcher { -public: - using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType; - +class LedgerFetcher : public LedgerFetcherInterface { private: util::Logger log_{"ETL"}; std::shared_ptr backend_; - std::shared_ptr loadBalancer_; + std::shared_ptr loadBalancer_; public: /** * @brief Create an instance of the fetcher */ - LedgerFetcher(std::shared_ptr backend, std::shared_ptr balancer) + LedgerFetcher(std::shared_ptr backend, std::shared_ptr balancer) : backend_(std::move(backend)), loadBalancer_(std::move(balancer)) { } @@ -57,14 +55,14 @@ public: /** * @brief Extract data for a particular ledger from an ETL source * - * This function continously tries to extract the specified ledger (using all available ETL sources) until the + * This function continuously tries to extract the specified ledger (using all available ETL sources) until the * extraction succeeds, or the server shuts down. * * @param sequence sequence of the ledger to extract * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down */ [[nodiscard]] OptionalGetLedgerResponseType - fetchData(uint32_t sequence) + fetchData(uint32_t sequence) override { LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence; @@ -77,14 +75,14 @@ public: /** * @brief Extract diff data for a particular ledger from an ETL source. * - * This function continously tries to extract the specified ledger (using all available ETL sources) until the + * This function continuously tries to extract the specified ledger (using all available ETL sources) until the * extraction succeeds, or the server shuts down. * * @param sequence sequence of the ledger to extract * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down */ [[nodiscard]] OptionalGetLedgerResponseType - fetchDataAndDiff(uint32_t sequence) + fetchDataAndDiff(uint32_t sequence) override { LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence; diff --git a/src/etl/impl/LedgerLoader.hpp b/src/etl/impl/LedgerLoader.hpp index 716fa8718..8eeb7553d 100644 --- a/src/etl/impl/LedgerLoader.hpp +++ b/src/etl/impl/LedgerLoader.hpp @@ -26,6 +26,7 @@ #include "etl/NFTHelpers.hpp" #include "etl/SystemState.hpp" #include "etl/impl/LedgerFetcher.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "util/Assert.hpp" #include "util/LedgerUtils.hpp" #include "util/Profiler.hpp" @@ -65,18 +66,18 @@ namespace etl::impl { /** * @brief Loads ledger data into the DB */ -template +template class LedgerLoader { public: - using GetLedgerResponseType = typename LoadBalancerType::GetLedgerResponseType; - using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType; - using RawLedgerObjectType = typename LoadBalancerType::RawLedgerObjectType; + using GetLedgerResponseType = etlng::LoadBalancerInterface::GetLedgerResponseType; + using OptionalGetLedgerResponseType = etlng::LoadBalancerInterface::OptionalGetLedgerResponseType; + using RawLedgerObjectType = etlng::LoadBalancerInterface::RawLedgerObjectType; private: util::Logger log_{"ETL"}; std::shared_ptr backend_; - std::shared_ptr loadBalancer_; + std::shared_ptr loadBalancer_; std::reference_wrapper fetcher_; std::reference_wrapper state_; // shared state for ETL @@ -86,7 +87,7 @@ public: */ LedgerLoader( std::shared_ptr backend, - std::shared_ptr balancer, + std::shared_ptr balancer, LedgerFetcherType& fetcher, SystemState const& state ) @@ -101,11 +102,11 @@ public: * @brief Insert extracted transaction into the ledger * * Insert all of the extracted transactions into the ledger, returning transactions related to accounts, - * transactions related to NFTs, and NFTs themselves for later processsing. + * transactions related to NFTs, and NFTs themselves for later processing. * * @param ledger ledger to insert transactions into * @param data data extracted from an ETL source - * @return The neccessary info to write the account_transactions/account_tx and nft_token_transactions tables + * @return The necessary info to write the account_transactions/account_tx and nft_token_transactions tables */ FormattedTransactionsData insertTransactions(ripple::LedgerHeader const& ledger, GetLedgerResponseType& data) @@ -219,7 +220,7 @@ public: ripple::uint256 prev = data::kFIRST_KEY; while (auto cur = backend_->cache().getSuccessor(prev, sequence)) { - ASSERT(cur.has_value(), "Succesor for key {} must exist", ripple::strHex(prev)); + ASSERT(cur.has_value(), "Successor for key {} must exist", ripple::strHex(prev)); if (prev == data::kFIRST_KEY) backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key)); diff --git a/src/etl/impl/LedgerPublisher.hpp b/src/etl/impl/LedgerPublisher.hpp index 9e881890e..4c21b4c21 100644 --- a/src/etl/impl/LedgerPublisher.hpp +++ b/src/etl/impl/LedgerPublisher.hpp @@ -267,7 +267,7 @@ public: } /** - * @brief Get the sequence of the last schueduled ledger to publish, Be aware that the ledger may not have been + * @brief Get the sequence of the last scheduled ledger to publish, Be aware that the ledger may not have been * published to network */ std::optional diff --git a/src/etl/impl/SourceImpl.hpp b/src/etl/impl/SourceImpl.hpp index 3f2f25a33..a2b4175e0 100644 --- a/src/etl/impl/SourceImpl.hpp +++ b/src/etl/impl/SourceImpl.hpp @@ -202,9 +202,9 @@ public: * @return A std::pair of the data and a bool indicating whether the download was successful */ std::pair, bool> - loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) final + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) final { - return grpcSource_.loadInitialLedger(sequence, numMarkers, cacheOnly); + return grpcSource_.loadInitialLedger(sequence, numMarkers); } /** diff --git a/src/etl/impl/SubscriptionSource.cpp b/src/etl/impl/SubscriptionSource.cpp index 48d498b8c..43d66de32 100644 --- a/src/etl/impl/SubscriptionSource.cpp +++ b/src/etl/impl/SubscriptionSource.cpp @@ -235,7 +235,7 @@ SubscriptionSource::handleMessage(std::string const& message) } else { if (isForwarding_) { - // Clio as rippled's proposed_transactions subscirber, will receive two jsons for each transaction + // Clio as rippled's proposed_transactions subscriber, will receive two jsons for each transaction // 1 - Proposed transaction // 2 - Validated transaction // Only forward proposed transaction, validated transactions are sent by Clio itself diff --git a/src/etl/impl/Transformer.hpp b/src/etl/impl/Transformer.hpp index b1d33b054..47a11c05a 100644 --- a/src/etl/impl/Transformer.hpp +++ b/src/etl/impl/Transformer.hpp @@ -141,7 +141,7 @@ private: auto fetchResponse = pipe_.get().popNext(currentSequence); ++currentSequence; - // if fetchResponse is an empty optional, the extracter thread has stopped and the transformer should + // if fetchResponse is an empty optional, the extractor thread has stopped and the transformer should // stop as well if (!fetchResponse) break; diff --git a/src/etlng/CMakeLists.txt b/src/etlng/CMakeLists.txt index d0f2854e9..49e443a34 100644 --- a/src/etlng/CMakeLists.txt +++ b/src/etlng/CMakeLists.txt @@ -2,10 +2,13 @@ add_library(clio_etlng) target_sources( clio_etlng - PRIVATE impl/AmendmentBlockHandler.cpp + PRIVATE LoadBalancer.cpp + Source.cpp + impl/AmendmentBlockHandler.cpp impl/AsyncGrpcCall.cpp impl/Extraction.cpp impl/GrpcSource.cpp + impl/ForwardingSource.cpp impl/Loading.cpp impl/Monitor.cpp impl/TaskManager.cpp diff --git a/src/etlng/ETLService.hpp b/src/etlng/ETLService.hpp new file mode 100644 index 000000000..5a54f14e6 --- /dev/null +++ b/src/etlng/ETLService.hpp @@ -0,0 +1,282 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "data/LedgerCache.hpp" +#include "data/Types.hpp" +#include "etl/CacheLoader.hpp" +#include "etl/ETLState.hpp" +#include "etl/LedgerFetcherInterface.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etl/SystemState.hpp" +#include "etl/impl/AmendmentBlockHandler.hpp" +#include "etl/impl/LedgerFetcher.hpp" +#include "etl/impl/LedgerPublisher.hpp" +#include "etlng/AmendmentBlockHandlerInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/ExtractorInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" +#include "etlng/impl/AmendmentBlockHandler.hpp" +#include "etlng/impl/Extraction.hpp" +#include "etlng/impl/Loading.hpp" +#include "etlng/impl/Monitor.hpp" +#include "etlng/impl/Registry.hpp" +#include "etlng/impl/Scheduling.hpp" +#include "etlng/impl/TaskManager.hpp" +#include "etlng/impl/ext/Cache.hpp" +#include "etlng/impl/ext/Core.hpp" +#include "etlng/impl/ext/NFT.hpp" +#include "etlng/impl/ext/Successor.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "util/Assert.hpp" +#include "util/Profiler.hpp" +#include "util/async/context/BasicExecutionContext.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng { + +/** + * @brief This class is responsible for continuously extracting data from a p2p node, and writing that data to the + * databases. + * + * Usually, multiple different processes share access to the same network accessible databases, in which case only one + * such process is performing ETL and writing to the database. The other processes simply monitor the database for new + * ledgers, and publish those ledgers to the various subscription streams. If a monitoring process determines that the + * ETL writer has failed (no new ledgers written for some time), the process will attempt to become the ETL writer. + * + * If there are multiple monitoring processes that try to become the ETL writer at the same time, one will win out, and + * the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring + * to writing and from writing to monitoring, based on the activity of other processes running on different machines. + */ +class ETLService : public ETLServiceInterface { + util::Logger log_{"ETL"}; + + std::shared_ptr backend_; + std::shared_ptr subscriptions_; + std::shared_ptr balancer_; + std::shared_ptr ledgers_; + std::shared_ptr> cacheLoader_; + + std::shared_ptr fetcher_; + std::shared_ptr extractor_; + + etl::SystemState state_; + util::async::CoroExecutionContext ctx_{8}; + + std::shared_ptr amendmentBlockHandler_; + std::shared_ptr loader_; + + std::optional> mainLoop_; + +public: + /** + * @brief Create an instance of ETLService. + * + * @param config The configuration to use + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param balancer Load balancer to use + * @param ledgers The network validated ledgers datastructure + */ + ETLService( + util::config::ClioConfigDefinition const& config, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr balancer, + std::shared_ptr ledgers + ) + : backend_(std::move(backend)) + , subscriptions_(std::move(subscriptions)) + , balancer_(std::move(balancer)) + , ledgers_(std::move(ledgers)) + , cacheLoader_(std::make_shared>(config, backend_, backend_->cache())) + , fetcher_(std::make_shared(backend_, balancer_)) + , extractor_(std::make_shared(fetcher_)) + , amendmentBlockHandler_(std::make_shared(ctx_, state_)) + , loader_(std::make_shared( + backend_, + fetcher_, + impl::makeRegistry( + impl::CacheExt{backend_->cache()}, + impl::CoreExt{backend_}, + impl::SuccessorExt{backend_, backend_->cache()}, + impl::NFTExt{backend_} + ), + amendmentBlockHandler_ + )) + { + LOG(log_.info()) << "Creating ETLng..."; + } + + ~ETLService() override + { + LOG(log_.debug()) << "Stopping ETLng"; + } + + void + run() override + { + LOG(log_.info()) << "run() in ETLng..."; + + mainLoop_.emplace(ctx_.execute([this] { + auto const rng = loadInitialLedgerIfNeeded(); + + LOG(log_.info()) << "Waiting for next ledger to be validated by network..."; + std::optional const mostRecentValidated = ledgers_->getMostRecent(); + + if (not mostRecentValidated) { + LOG(log_.info()) << "The wait for the next validated ledger has been aborted. " + "Exiting monitor loop"; + return; + } + + ASSERT(rng.has_value(), "Ledger range can't be null"); + auto const nextSequence = rng->maxSequence + 1; + + LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence; + + auto scheduler = impl::makeScheduler(impl::ForwardScheduler{*ledgers_, nextSequence} + // impl::BackfillScheduler{nextSequence - 1, nextSequence - 1000}, + // TODO lift limit and start with rng.minSeq + ); + + auto man = impl::TaskManager(ctx_, *scheduler, *extractor_, *loader_); + + // TODO: figure out this: std::make_shared(backend_, ledgers_, nextSequence) + man.run({}); // TODO: needs to be interruptible and fill out settings + })); + } + + void + stop() override + { + LOG(log_.info()) << "Stop called"; + // TODO: stop the service correctly + } + + boost::json::object + getInfo() const override + { + // TODO + return {{"ok", true}}; + } + + bool + isAmendmentBlocked() const override + { + // TODO + return false; + } + + bool + isCorruptionDetected() const override + { + // TODO + return false; + } + + std::optional + getETLState() const override + { + // TODO + return std::nullopt; + } + + std::uint32_t + lastCloseAgeSeconds() const override + { + // TODO + return 0; + } + +private: + // TODO: this better be std::expected + std::optional + loadInitialLedgerIfNeeded() + { + if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); not rng.has_value()) { + LOG(log_.info()) << "Database is empty. Will download a ledger from the network."; + + try { + LOG(log_.info()) << "Waiting for next ledger to be validated by network..."; + if (auto const mostRecentValidated = ledgers_->getMostRecent(); mostRecentValidated.has_value()) { + auto const seq = *mostRecentValidated; + LOG(log_.info()) << "Ledger " << seq << " has been validated. Downloading... "; + + auto [ledger, timeDiff] = ::util::timed>([this, seq]() { + return extractor_->extractLedgerOnly(seq).and_then([this, seq](auto&& data) { + // TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar + data.edgeKeys = balancer_->loadInitialLedger(seq, *loader_); + + // TODO: this should be interruptible for graceful shutdown + return loader_->loadInitialLedger(data); + }); + }); + + LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff; + LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size(); + + if (ledger.has_value()) + return backend_->hardFetchLedgerRangeNoThrow(); + + LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop"; + } else { + LOG(log_.info()) << "The wait for the next validated ledger has been aborted. " + "Exiting monitor loop"; + } + } catch (std::runtime_error const& e) { + LOG(log_.fatal()) << "Failed to load initial ledger: " << e.what(); + amendmentBlockHandler_->notifyAmendmentBlocked(); + } + } else { + LOG(log_.info()) << "Database already populated. Picking up from the tip of history"; + cacheLoader_->load(rng->maxSequence); + + return rng; + } + + return std::nullopt; + } +}; +} // namespace etlng diff --git a/src/etlng/ETLServiceInterface.hpp b/src/etlng/ETLServiceInterface.hpp new file mode 100644 index 000000000..7f0b2fc2b --- /dev/null +++ b/src/etlng/ETLServiceInterface.hpp @@ -0,0 +1,92 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "etl/ETLState.hpp" + +#include + +#include +#include + +namespace etlng { + +/** + * @brief This is a base class for any ETL service implementations. + * @note A ETL service is responsible for continuously extracting data from a p2p node, and writing that data to the + * databases. + */ +struct ETLServiceInterface { + virtual ~ETLServiceInterface() = default; + + /** + * @brief Start all components to run ETL service. + */ + virtual void + run() = 0; + + /** + * @brief Stop the ETL service. + * @note This method blocks until the ETL service has stopped. + */ + virtual void + stop() = 0; + + /** + * @brief Get state of ETL as a JSON object + * + * @return The state of ETL as a JSON object + */ + [[nodiscard]] virtual boost::json::object + getInfo() const = 0; + + /** + * @brief Check for the amendment blocked state. + * + * @return true if currently amendment blocked; false otherwise + */ + [[nodiscard]] virtual bool + isAmendmentBlocked() const = 0; + + /** + * @brief Check whether Clio detected DB corruptions. + * + * @return true if corruption of DB was detected and cache was stopped. + */ + [[nodiscard]] virtual bool + isCorruptionDetected() const = 0; + + /** + * @brief Get the etl nodes' state + * @return The etl nodes' state, nullopt if etl nodes are not connected + */ + [[nodiscard]] virtual std::optional + getETLState() const = 0; + + /** + * @brief Get time passed since last ledger close, in seconds. + * + * @return Time passed since last ledger close + */ + [[nodiscard]] virtual std::uint32_t + lastCloseAgeSeconds() const = 0; +}; + +} // namespace etlng diff --git a/src/etlng/LedgerPublisherInterface.hpp b/src/etlng/LedgerPublisherInterface.hpp index e2d414214..fa6096251 100644 --- a/src/etlng/LedgerPublisherInterface.hpp +++ b/src/etlng/LedgerPublisherInterface.hpp @@ -26,7 +26,7 @@ namespace etlng { /** - * @brief The interface of a scheduler for the extraction proccess + * @brief The interface of a scheduler for the extraction process */ struct LedgerPublisherInterface { virtual ~LedgerPublisherInterface() = default; diff --git a/src/etlng/LoadBalancer.cpp b/src/etlng/LoadBalancer.cpp new file mode 100644 index 000000000..1dc528e61 --- /dev/null +++ b/src/etlng/LoadBalancer.cpp @@ -0,0 +1,442 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2023, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/LoadBalancer.hpp" + +#include "data/BackendInterface.hpp" +#include "etl/ETLState.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" +#include "etlng/Source.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/Assert.hpp" +#include "util/CoroutineGroup.hpp" +#include "util/Profiler.hpp" +#include "util/Random.hpp" +#include "util/ResponseExpirationCache.hpp" +#include "util/config/ArrayView.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ObjectView.hpp" +#include "util/log/Logger.hpp" +#include "util/prometheus/Label.hpp" +#include "util/prometheus/Prometheus.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace util::config; +using util::prometheus::Labels; + +namespace etlng { + +std::shared_ptr +LoadBalancer::makeLoadBalancer( + ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory +) +{ + return std::make_shared( + config, ioc, std::move(backend), std::move(subscriptions), std::move(validatedLedgers), std::move(sourceFactory) + ); +} + +LoadBalancer::LoadBalancer( + ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory +) + : forwardingCounters_{ + .successDuration = PrometheusService::counterInt( + "forwarding_duration_milliseconds_counter", + Labels({util::prometheus::Label{"status", "success"}}), + "The duration of processing successful forwarded requests" + ), + .failDuration = PrometheusService::counterInt( + "forwarding_duration_milliseconds_counter", + Labels({util::prometheus::Label{"status", "fail"}}), + "The duration of processing failed forwarded requests" + ), + .retries = PrometheusService::counterInt( + "forwarding_retries_counter", + Labels(), + "The number of retries before a forwarded request was successful. Initial attempt excluded" + ), + .cacheHit = PrometheusService::counterInt( + "forwarding_cache_hit_counter", + Labels(), + "The number of requests that we served from the cache" + ), + .cacheMiss = PrometheusService::counterInt( + "forwarding_cache_miss_counter", + Labels(), + "The number of requests that were not served from the cache" + ) + } +{ + auto const forwardingCacheTimeout = config.get("forwarding.cache_timeout"); + if (forwardingCacheTimeout > 0.f) { + forwardingCache_ = util::ResponseExpirationCache{ + util::config::ClioConfigDefinition::toMilliseconds(forwardingCacheTimeout), + {"server_info", "server_state", "server_definitions", "fee", "ledger_closed"} + }; + } + + auto const numMarkers = config.getValueView("num_markers"); + if (numMarkers.hasValue()) { + auto const value = numMarkers.asIntType(); + downloadRanges_ = value; + } else if (backend->fetchLedgerRange()) { + downloadRanges_ = 4; + } + + auto const allowNoEtl = config.get("allow_no_etl"); + + auto const checkOnETLFailure = [this, allowNoEtl](std::string const& log) { + LOG(log_.warn()) << log; + + if (!allowNoEtl) { + LOG(log_.error()) << "Set allow_no_etl as true in config to allow clio run without valid ETL sources."; + throw std::logic_error("ETL configuration error."); + } + }; + + auto const forwardingTimeout = + ClioConfigDefinition::toMilliseconds(config.get("forwarding.request_timeout")); + auto const etlArray = config.getArray("etl_sources"); + for (auto it = etlArray.begin(); it != etlArray.end(); ++it) { + auto source = sourceFactory( + *it, + ioc, + subscriptions, + validatedLedgers, + forwardingTimeout, + [this]() { + if (not hasForwardingSource_.lock().get()) + chooseForwardingSource(); + }, + [this](bool wasForwarding) { + if (wasForwarding) + chooseForwardingSource(); + }, + [this]() { + if (forwardingCache_.has_value()) + forwardingCache_->invalidate(); + } + ); + + // checking etl node validity + auto const stateOpt = etl::ETLState::fetchETLStateFromSource(*source); + + if (!stateOpt) { + LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString() + << " Please check the configuration and network"; + } else if (etlState_ && etlState_->networkID != stateOpt->networkID) { + checkOnETLFailure(fmt::format( + "ETL sources must be on the same network. Source network id = {} does not match others network id = {}", + stateOpt->networkID, + etlState_->networkID + )); + } else { + etlState_ = stateOpt; + } + + sources_.push_back(std::move(source)); + LOG(log_.info()) << "Added etl source - " << sources_.back()->toString(); + } + + if (!etlState_) + checkOnETLFailure("Failed to fetch ETL state from any source. Please check the configuration and network"); + + if (sources_.empty()) + checkOnETLFailure("No ETL sources configured. Please check the configuration"); + + // This is made separate from source creation to prevent UB in case one of the sources will call + // chooseForwardingSource while we are still filling the sources_ vector + for (auto const& source : sources_) { + source->run(); + } +} + +std::vector +LoadBalancer::loadInitialLedger( + uint32_t sequence, + etlng::InitialLoadObserverInterface& loadObserver, + std::chrono::steady_clock::duration retryAfter +) +{ + std::vector response; + execute( + [this, &response, &sequence, &loadObserver](auto& source) { + auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, loadObserver); + + if (!res) { + LOG(log_.error()) << "Failed to download initial ledger." + << " Sequence = " << sequence << " source = " << source->toString(); + } else { + response = std::move(data); + } + + return res; + }, + sequence, + retryAfter + ); + return response; +} + +LoadBalancer::OptionalGetLedgerResponseType +LoadBalancer::fetchLedger( + uint32_t ledgerSequence, + bool getObjects, + bool getObjectNeighbors, + std::chrono::steady_clock::duration retryAfter +) +{ + GetLedgerResponseType response; + execute( + [&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](auto& source) { + auto [status, data] = source->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors); + response = std::move(data); + if (status.ok() && response.validated()) { + LOG(log.info()) << "Successfully fetched ledger = " << ledgerSequence + << " from source = " << source->toString(); + return true; + } + + LOG(log.warn()) << "Could not fetch ledger " << ledgerSequence << ", Reply: " << response.DebugString() + << ", error_code: " << status.error_code() << ", error_msg: " << status.error_message() + << ", source = " << source->toString(); + return false; + }, + ledgerSequence, + retryAfter + ); + return response; +} + +std::expected +LoadBalancer::forwardToRippled( + boost::json::object const& request, + std::optional const& clientIp, + bool isAdmin, + boost::asio::yield_context yield +) +{ + if (not request.contains("command")) + return std::unexpected{rpc::ClioError::RpcCommandIsMissing}; + + auto const cmd = boost::json::value_to(request.at("command")); + + if (forwardingCache_ and forwardingCache_->shouldCache(cmd)) { + bool servedFromCache = true; + auto updater = + [this, &request, &clientIp, &servedFromCache, isAdmin](boost::asio::yield_context yield + ) -> std::expected { + servedFromCache = false; + auto result = forwardToRippledImpl(request, clientIp, isAdmin, yield); + if (result.has_value()) { + return util::ResponseExpirationCache::EntryData{ + .lastUpdated = std::chrono::steady_clock::now(), .response = std::move(result).value() + }; + } + return std::unexpected{ + util::ResponseExpirationCache::Error{.status = rpc::Status{result.error()}, .warnings = {}} + }; + }; + + auto result = forwardingCache_->getOrUpdate( + yield, + cmd, + std::move(updater), + [](util::ResponseExpirationCache::EntryData const& entry) { return not entry.response.contains("error"); } + ); + if (servedFromCache) { + ++forwardingCounters_.cacheHit.get(); + } + if (result.has_value()) { + return std::move(result).value(); + } + auto const combinedError = result.error().status.code; + ASSERT(std::holds_alternative(combinedError), "There could be only ClioError here"); + return std::unexpected{std::get(combinedError)}; + } + + return forwardToRippledImpl(request, clientIp, isAdmin, yield); +} + +boost::json::value +LoadBalancer::toJson() const +{ + boost::json::array ret; + for (auto& src : sources_) + ret.push_back(src->toJson()); + + return ret; +} + +template +void +LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter) +{ + ASSERT(not sources_.empty(), "ETL sources must be configured to execute functions."); + size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1); + + size_t numAttempts = 0; + + while (true) { + auto& source = sources_[sourceIdx]; + + LOG(log_.debug()) << "Attempting to execute func. ledger sequence = " << ledgerSequence + << " - source = " << source->toString(); + // Originally, it was (source->hasLedger(ledgerSequence) || true) + /* Sometimes rippled has ledger but doesn't actually know. However, + but this does NOT happen in the normal case and is safe to remove + This || true is only needed when loading full history standalone */ + if (source->hasLedger(ledgerSequence)) { + bool const res = f(source); + if (res) { + LOG(log_.debug()) << "Successfully executed func at source = " << source->toString() + << " - ledger sequence = " << ledgerSequence; + break; + } + + LOG(log_.warn()) << "Failed to execute func at source = " << source->toString() + << " - ledger sequence = " << ledgerSequence; + } else { + LOG(log_.warn()) << "Ledger not present at source = " << source->toString() + << " - ledger sequence = " << ledgerSequence; + } + sourceIdx = (sourceIdx + 1) % sources_.size(); + numAttempts++; + if (numAttempts % sources_.size() == 0) { + LOG(log_.info()) << "Ledger sequence " << ledgerSequence + << " is not yet available from any configured sources. Sleeping and trying again"; + std::this_thread::sleep_for(retryAfter); + } + } +} + +std::optional +LoadBalancer::getETLState() noexcept +{ + if (!etlState_) { + // retry ETLState fetch + etlState_ = etl::ETLState::fetchETLStateFromSource(*this); + } + return etlState_; +} + +void +LoadBalancer::stop(boost::asio::yield_context yield) +{ + util::CoroutineGroup group{yield}; + std::ranges::for_each(sources_, [&group, yield](auto& source) { + group.spawn(yield, [&source](boost::asio::yield_context innerYield) { source->stop(innerYield); }); + }); + group.asyncWait(yield); +} + +void +LoadBalancer::chooseForwardingSource() +{ + LOG(log_.info()) << "Choosing a new source to forward subscriptions"; + auto hasForwardingSourceLock = hasForwardingSource_.lock(); + hasForwardingSourceLock.get() = false; + for (auto& source : sources_) { + if (not hasForwardingSourceLock.get() and source->isConnected()) { + source->setForwarding(true); + hasForwardingSourceLock.get() = true; + } else { + source->setForwarding(false); + } + } +} + +std::expected +LoadBalancer::forwardToRippledImpl( + boost::json::object const& request, + std::optional const& clientIp, + bool isAdmin, + boost::asio::yield_context yield +) +{ + ++forwardingCounters_.cacheMiss.get(); + + ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests."); + std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1); + + auto numAttempts = 0u; + + auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE; + + std::optional response; + rpc::ClioError error = rpc::ClioError::EtlConnectionError; + while (numAttempts < sources_.size()) { + auto [res, duration] = + util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); }); + + if (res) { + forwardingCounters_.successDuration.get() += duration; + response = std::move(res).value(); + break; + } + forwardingCounters_.failDuration.get() += duration; + ++forwardingCounters_.retries.get(); + error = std::max(error, res.error()); // Choose the best result between all sources + + sourceIdx = (sourceIdx + 1) % sources_.size(); + ++numAttempts; + } + + if (response.has_value()) { + return std::move(response).value(); + } + + return std::unexpected{error}; +} + +} // namespace etlng diff --git a/src/etlng/LoadBalancer.hpp b/src/etlng/LoadBalancer.hpp new file mode 100644 index 000000000..3c6fd6032 --- /dev/null +++ b/src/etlng/LoadBalancer.hpp @@ -0,0 +1,287 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2022, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etl/ETLState.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" +#include "etlng/Source.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/Assert.hpp" +#include "util/Mutex.hpp" +#include "util/ResponseExpirationCache.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/log/Logger.hpp" +#include "util/prometheus/Counter.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng { + +/** + * @brief A tag class to help identify LoadBalancer in templated code. + */ +struct LoadBalancerTag { + virtual ~LoadBalancerTag() = default; +}; + +template +concept SomeLoadBalancer = std::derived_from; + +/** + * @brief This class is used to manage connections to transaction processing processes. + * + * This class spawns a listener for each etl source, which listens to messages on the ledgers stream (to keep track of + * which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also + * allows requests for ledger data to be load balanced across all possible ETL sources. + */ +class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag { +public: + using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject; + using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse; + using OptionalGetLedgerResponseType = std::optional; + +private: + static constexpr std::uint32_t kDEFAULT_DOWNLOAD_RANGES = 16; + + util::Logger log_{"ETL"}; + // Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache + std::optional forwardingCache_; + std::optional forwardingXUserValue_; + + std::vector sources_; + std::optional etlState_; + std::uint32_t downloadRanges_ = + kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */ + + struct ForwardingCounters { + std::reference_wrapper successDuration; + std::reference_wrapper failDuration; + std::reference_wrapper retries; + std::reference_wrapper cacheHit; + std::reference_wrapper cacheMiss; + } forwardingCounters_; + + // Using mutext instead of atomic_bool because choosing a new source to + // forward messages should be done with a mutual exclusion otherwise there will be a race condition + util::Mutex hasForwardingSource_{false}; + +public: + /** + * @brief Value for the X-User header when forwarding admin requests + */ + static constexpr std::string_view kADMIN_FORWARDING_X_USER_VALUE = "clio_admin"; + + /** + * @brief Value for the X-User header when forwarding user requests + */ + static constexpr std::string_view kUSER_FORWARDING_X_USER_VALUE = "clio_user"; + + /** + * @brief Create an instance of the load balancer. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param sourceFactory A factory function to create a source + */ + LoadBalancer( + util::config::ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory = makeSource + ); + + /** + * @brief A factory function for the load balancer. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param sourceFactory A factory function to create a source + * @return A shared pointer to a new instance of LoadBalancer + */ + static std::shared_ptr + makeLoadBalancer( + util::config::ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory = makeSource + ); + + /** + * @brief Load the initial ledger, writing data to the queue. + * @note This function will retry indefinitely until the ledger is downloaded. + * + * @param sequence Sequence of ledger to download + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return A std::vector The ledger data + */ + std::vector + loadInitialLedger( + [[maybe_unused]] uint32_t sequence, + [[maybe_unused]] std::chrono::steady_clock::duration retryAfter + ) override + { + ASSERT(false, "Not available for new ETL"); + std::unreachable(); + }; + + /** + * @brief Load the initial ledger, writing data to the queue. + * @note This function will retry indefinitely until the ledger is downloaded. + * + * @param sequence Sequence of ledger to download + * @param observer The observer to notify of progress + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return A std::vector The ledger data + */ + std::vector + loadInitialLedger( + uint32_t sequence, + etlng::InitialLoadObserverInterface& observer, + std::chrono::steady_clock::duration retryAfter + ) override; + + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param ledgerSequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one + * @param getObjectNeighbors Whether to request object neighbors + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return The extracted data, if extraction was successful. If the ledger was found + * in the database or the server is shutting down, the optional will be empty + */ + OptionalGetLedgerResponseType + fetchLedger( + uint32_t ledgerSequence, + bool getObjects, + bool getObjectNeighbors, + std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2} + ) override; + + /** + * @brief Represent the state of this load balancer as a JSON object + * + * @return JSON representation of the state of this load balancer. + */ + boost::json::value + toJson() const override; + + /** + * @brief Forward a JSON RPC request to a randomly selected rippled node. + * + * @param request JSON-RPC request to forward + * @param clientIp The IP address of the peer, if known + * @param isAdmin Whether the request is from an admin + * @param yield The coroutine context + * @return Response received from rippled node as JSON object on success or error on failure + */ + std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& clientIp, + bool isAdmin, + boost::asio::yield_context yield + ) override; + + /** + * @brief Return state of ETL nodes. + * @return ETL state, nullopt if etl nodes not available + */ + std::optional + getETLState() noexcept override; + + /** + * @brief Stop the load balancer. This will stop all subscription sources. + * @note This function will asynchronously wait for all sources to stop. + * + * @param yield The coroutine context + */ + void + stop(boost::asio::yield_context yield) override; + +private: + /** + * @brief Execute a function on a randomly selected source. + * + * @note f is a function that takes an Source as an argument and returns a bool. + * Attempt to execute f for one randomly chosen Source that has the specified ledger. If f returns false, another + * randomly chosen Source is used. The process repeats until f returns true. + * + * @param f Function to execute. This function takes the ETL source as an argument, and returns a bool + * @param ledgerSequence f is executed for each Source that has this ledger + * @param retryAfter Time to wait between retries (2 seconds by default) + * server is shutting down + */ + template + void + execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}); + + /** + * @brief Choose a new source to forward requests + */ + void + chooseForwardingSource(); + + std::expected + forwardToRippledImpl( + boost::json::object const& request, + std::optional const& clientIp, + bool isAdmin, + boost::asio::yield_context yield + ); +}; + +} // namespace etlng diff --git a/src/etlng/LoadBalancerInterface.hpp b/src/etlng/LoadBalancerInterface.hpp index 3466a6e79..200bbb3f3 100644 --- a/src/etlng/LoadBalancerInterface.hpp +++ b/src/etlng/LoadBalancerInterface.hpp @@ -115,7 +115,7 @@ public: * @param yield The coroutine context * @return Response received from rippled node as JSON object on success or error on failure */ - virtual std::expected + virtual std::expected forwardToRippled( boost::json::object const& request, std::optional const& clientIp, @@ -129,6 +129,15 @@ public: */ virtual std::optional getETLState() noexcept = 0; + + /** + * @brief Stop the load balancer. This will stop all subscription sources. + * @note This function will asynchronously wait for all sources to stop. + * + * @param yield The coroutine context + */ + virtual void + stop(boost::asio::yield_context yield) = 0; }; } // namespace etlng diff --git a/src/etlng/LoaderInterface.hpp b/src/etlng/LoaderInterface.hpp index 929d71679..72cad3192 100644 --- a/src/etlng/LoaderInterface.hpp +++ b/src/etlng/LoaderInterface.hpp @@ -45,7 +45,7 @@ struct LoaderInterface { * @param data The data to load * @return Optional ledger header */ - virtual std::optional + [[nodiscard]] virtual std::optional loadInitialLedger(model::LedgerData const& data) = 0; }; diff --git a/src/etlng/SchedulerInterface.hpp b/src/etlng/SchedulerInterface.hpp index 606757b60..deeddcea9 100644 --- a/src/etlng/SchedulerInterface.hpp +++ b/src/etlng/SchedulerInterface.hpp @@ -26,7 +26,7 @@ namespace etlng { /** - * @brief The interface of a scheduler for the extraction proccess + * @brief The interface of a scheduler for the extraction process */ struct SchedulerInterface { virtual ~SchedulerInterface() = default; diff --git a/src/etlng/Source.cpp b/src/etlng/Source.cpp new file mode 100644 index 000000000..fb6fa6df9 --- /dev/null +++ b/src/etlng/Source.cpp @@ -0,0 +1,73 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/Source.hpp" + +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etl/impl/ForwardingSource.hpp" +#include "etl/impl/SubscriptionSource.hpp" +#include "etlng/impl/GrpcSource.hpp" +#include "etlng/impl/SourceImpl.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "util/config/ObjectView.hpp" + +#include + +#include +#include +#include +#include + +namespace etlng { + +SourcePtr +makeSource( + util::config::ObjectView const& config, + boost::asio::io_context& ioc, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + std::chrono::steady_clock::duration forwardingTimeout, + SourceBase::OnConnectHook onConnect, + SourceBase::OnDisconnectHook onDisconnect, + SourceBase::OnLedgerClosedHook onLedgerClosed +) +{ + auto const ip = config.get("ip"); + auto const wsPort = config.get("ws_port"); + auto const grpcPort = config.get("grpc_port"); + + etl::impl::ForwardingSource forwardingSource{ip, wsPort, forwardingTimeout}; + impl::GrpcSource grpcSource{ip, grpcPort}; + auto subscriptionSource = std::make_unique( + ioc, + ip, + wsPort, + std::move(validatedLedgers), + std::move(subscriptions), + std::move(onConnect), + std::move(onDisconnect), + std::move(onLedgerClosed) + ); + + return std::make_unique>( + ip, wsPort, grpcPort, std::move(grpcSource), std::move(subscriptionSource), std::move(forwardingSource) + ); +} + +} // namespace etlng diff --git a/src/etlng/Source.hpp b/src/etlng/Source.hpp new file mode 100644 index 000000000..3e84ce568 --- /dev/null +++ b/src/etlng/Source.hpp @@ -0,0 +1,194 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/config/ObjectView.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng { + +/** + * @brief Provides an implementation of a ETL source + */ +class SourceBase { +public: + using OnConnectHook = std::function; + using OnDisconnectHook = std::function; + using OnLedgerClosedHook = std::function; + + virtual ~SourceBase() = default; + + /** + * @brief Run subscriptions loop of the source + */ + virtual void + run() = 0; + + /** + * @brief Stop Source. + * @note This method will asynchronously wait for source to be stopped. + * + * @param yield The coroutine context. + */ + virtual void + stop(boost::asio::yield_context yield) = 0; + + /** + * @brief Check if source is connected + * + * @return true if source is connected; false otherwise + */ + [[nodiscard]] virtual bool + isConnected() const = 0; + + /** + * @brief Set the forwarding state of the source. + * + * @param isForwarding Whether to forward or not + */ + virtual void + setForwarding(bool isForwarding) = 0; + + /** + * @brief Represent the source as a JSON object + * + * @return JSON representation of the source + */ + [[nodiscard]] virtual boost::json::object + toJson() const = 0; + + /** @return String representation of the source (for debug) */ + [[nodiscard]] virtual std::string + toString() const = 0; + + /** + * @brief Check if ledger is known by this source. + * + * @param sequence The ledger sequence to check + * @return true if ledger is in the range of this source; false otherwise + */ + [[nodiscard]] virtual bool + hasLedger(uint32_t sequence) const = 0; + + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param sequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true + * @param getObjectNeighbors Whether to request object neighbors; defaults to false + * @return A std::pair of the response status and the response itself + */ + [[nodiscard]] virtual std::pair + fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) = 0; + + /** + * @brief Download a ledger in full. + * + * @param sequence Sequence of the ledger to download + * @param numMarkers Number of markers to generate for async calls + * @param loader InitialLoadObserverInterface implementation + * @return A std::pair of the data and a bool indicating whether the download was successful + */ + virtual std::pair, bool> + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) = 0; + + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param forwardToRippledClientIp IP of the client forwarding this request if known + * @param xUserValue Value of the X-User header + * @param yield The coroutine context + * @return Response on success or error on failure + */ + [[nodiscard]] virtual std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield + ) const = 0; +}; + +using SourcePtr = std::unique_ptr; + +using SourceFactory = std::function subscriptions, + std::shared_ptr validatedLedgers, + std::chrono::steady_clock::duration forwardingTimeout, + SourceBase::OnConnectHook onConnect, + SourceBase::OnDisconnectHook onDisconnect, + SourceBase::OnLedgerClosedHook onLedgerClosed +)>; + +/** + * @brief Create a source + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers data structure + * @param forwardingTimeout The timeout for forwarding to rippled + * @param onConnect The hook to call on connect + * @param onDisconnect The hook to call on disconnect + * @param onLedgerClosed The hook to call on ledger closed. This is called when a ledger is closed and the source is set + * as forwarding. + * @return The created source + */ +[[nodiscard]] SourcePtr +makeSource( + util::config::ObjectView const& config, + boost::asio::io_context& ioc, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + std::chrono::steady_clock::duration forwardingTimeout, + SourceBase::OnConnectHook onConnect, + SourceBase::OnDisconnectHook onDisconnect, + SourceBase::OnLedgerClosedHook onLedgerClosed +); + +} // namespace etlng diff --git a/src/etlng/impl/ForwardingSource.cpp b/src/etlng/impl/ForwardingSource.cpp new file mode 100644 index 000000000..b4fb024bf --- /dev/null +++ b/src/etlng/impl/ForwardingSource.cpp @@ -0,0 +1,116 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ForwardingSource.hpp" + +#include "rpc/Errors.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +ForwardingSource::ForwardingSource( + std::string ip, + std::string wsPort, + std::chrono::steady_clock::duration forwardingTimeout, + std::chrono::steady_clock::duration connTimeout +) + : log_(fmt::format("ForwardingSource[{}:{}]", ip, wsPort)) + , connectionBuilder_(std::move(ip), std::move(wsPort)) + , forwardingTimeout_{forwardingTimeout} +{ + connectionBuilder_.setConnectionTimeout(connTimeout) + .addHeader( + {boost::beast::http::field::user_agent, fmt::format("{} websocket-client-coro", BOOST_BEAST_VERSION_STRING)} + ); +} + +std::expected +ForwardingSource::forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield +) const +{ + auto connectionBuilder = connectionBuilder_; + if (forwardToRippledClientIp) { + connectionBuilder.addHeader( + {boost::beast::http::field::forwarded, fmt::format("for={}", *forwardToRippledClientIp)} + ); + } + + connectionBuilder.addHeader({"X-User", std::string{xUserValue}}); + + auto expectedConnection = connectionBuilder.connect(yield); + if (not expectedConnection) { + LOG(log_.debug()) << "Couldn't connect to rippled to forward request."; + return std::unexpected{rpc::ClioError::EtlConnectionError}; + } + auto& connection = expectedConnection.value(); + + auto writeError = connection->write(boost::json::serialize(request), yield, forwardingTimeout_); + if (writeError) { + LOG(log_.debug()) << "Error sending request to rippled to forward request."; + return std::unexpected{rpc::ClioError::EtlRequestError}; + } + + auto response = connection->read(yield, forwardingTimeout_); + if (not response) { + if (auto errorCode = response.error().errorCode(); + errorCode.has_value() and errorCode->value() == boost::system::errc::timed_out) { + LOG(log_.debug()) << "Request to rippled timed out"; + return std::unexpected{rpc::ClioError::EtlRequestTimeout}; + } + LOG(log_.debug()) << "Error sending request to rippled to forward request."; + return std::unexpected{rpc::ClioError::EtlRequestError}; + } + + boost::json::value parsedResponse; + try { + parsedResponse = boost::json::parse(*response); + if (not parsedResponse.is_object()) + throw std::runtime_error("response is not an object"); + } catch (std::exception const& e) { + LOG(log_.debug()) << "Error parsing response from rippled: " << e.what() << ". Response: " << *response; + return std::unexpected{rpc::ClioError::EtlInvalidResponse}; + } + + auto responseObject = parsedResponse.as_object(); + responseObject["forwarded"] = true; + + return responseObject; +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ForwardingSource.hpp b/src/etlng/impl/ForwardingSource.hpp new file mode 100644 index 000000000..8624e4e50 --- /dev/null +++ b/src/etlng/impl/ForwardingSource.hpp @@ -0,0 +1,70 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "rpc/Errors.hpp" +#include "util/log/Logger.hpp" +#include "util/requests/WsConnection.hpp" + +#include +#include + +#include +#include +#include +#include +#include + +namespace etlng::impl { + +class ForwardingSource { + util::Logger log_; + util::requests::WsConnectionBuilder connectionBuilder_; + std::chrono::steady_clock::duration forwardingTimeout_; + + static constexpr std::chrono::seconds kCONNECTION_TIMEOUT{3}; + +public: + ForwardingSource( + std::string ip, + std::string wsPort, + std::chrono::steady_clock::duration forwardingTimeout, + std::chrono::steady_clock::duration connTimeout = ForwardingSource::kCONNECTION_TIMEOUT + ); + + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param forwardToRippledClientIp IP of the client forwarding this request if known + * @param xUserValue Optional value for X-User header + * @param yield The coroutine context + * @return Response on success or error on failure + */ + std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield + ) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/Loading.hpp b/src/etlng/impl/Loading.hpp index 435dcb4e4..caa677bce 100644 --- a/src/etlng/impl/Loading.hpp +++ b/src/etlng/impl/Loading.hpp @@ -39,7 +39,6 @@ #include #include -#include #include #include #include diff --git a/src/etlng/impl/Registry.hpp b/src/etlng/impl/Registry.hpp index d5d262388..921b70811 100644 --- a/src/etlng/impl/Registry.hpp +++ b/src/etlng/impl/Registry.hpp @@ -24,7 +24,6 @@ #include -#include #include #include #include @@ -81,7 +80,7 @@ concept ContainsValidHook = HasLedgerDataHook or HasInitialDataHook or template concept NoTwoOfKind = not(HasLedgerDataHook and HasTransactionHook) and - not(HasInitialDataHook and HasInitialTransactionHook) and not(HasInitialDataHook and HasObjectHook) and + not(HasInitialDataHook and HasInitialTransactionHook) and not(HasInitialObjectsHook and HasInitialObjectHook); template @@ -216,4 +215,10 @@ public: } }; +static auto +makeRegistry(auto&&... exts) +{ + return std::make_unique...>>(std::forward(exts)...); +} + } // namespace etlng::impl diff --git a/src/etlng/impl/SourceImpl.hpp b/src/etlng/impl/SourceImpl.hpp new file mode 100644 index 000000000..1c99d973b --- /dev/null +++ b/src/etlng/impl/SourceImpl.hpp @@ -0,0 +1,232 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "etl/impl/ForwardingSource.hpp" +#include "etl/impl/SubscriptionSource.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/Source.hpp" +#include "etlng/impl/GrpcSource.hpp" +#include "rpc/Errors.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +/** + * @brief Provides an implementation of a ETL source + * + * @tparam GrpcSourceType The type of the gRPC source + * @tparam SubscriptionSourceTypePtr The type of the subscription source + * @tparam ForwardingSourceType The type of the forwarding source + */ +template < + typename GrpcSourceType = GrpcSource, + typename SubscriptionSourceTypePtr = std::unique_ptr, + typename ForwardingSourceType = etl::impl::ForwardingSource> +class SourceImpl : public SourceBase { + std::string ip_; + std::string wsPort_; + std::string grpcPort_; + + GrpcSourceType grpcSource_; + SubscriptionSourceTypePtr subscriptionSource_; + ForwardingSourceType forwardingSource_; + +public: + /** + * @brief Construct a new SourceImpl object + * + * @param ip The IP of the source + * @param wsPort The web socket port of the source + * @param grpcPort The gRPC port of the source + * @param grpcSource The gRPC source + * @param subscriptionSource The subscription source + * @param forwardingSource The forwarding source + */ + template + requires std::is_same_v and + std::is_same_v + SourceImpl( + std::string ip, + std::string wsPort, + std::string grpcPort, + SomeGrpcSourceType&& grpcSource, + SubscriptionSourceTypePtr subscriptionSource, + SomeForwardingSourceType&& forwardingSource + ) + : ip_(std::move(ip)) + , wsPort_(std::move(wsPort)) + , grpcPort_(std::move(grpcPort)) + , grpcSource_(std::forward(grpcSource)) + , subscriptionSource_(std::move(subscriptionSource)) + , forwardingSource_(std::forward(forwardingSource)) + { + } + + /** + * @brief Run subscriptions loop of the source + */ + void + run() final + { + subscriptionSource_->run(); + } + + void + stop(boost::asio::yield_context yield) final + { + subscriptionSource_->stop(yield); + } + + /** + * @brief Check if source is connected + * + * @return true if source is connected; false otherwise + */ + bool + isConnected() const final + { + return subscriptionSource_->isConnected(); + } + + /** + * @brief Set the forwarding state of the source. + * + * @param isForwarding Whether to forward or not + */ + void + setForwarding(bool isForwarding) final + { + subscriptionSource_->setForwarding(isForwarding); + } + + /** + * @brief Represent the source as a JSON object + * + * @return JSON representation of the source + */ + boost::json::object + toJson() const final + { + boost::json::object res; + + res["validated_range"] = subscriptionSource_->validatedRange(); + res["is_connected"] = std::to_string(static_cast(subscriptionSource_->isConnected())); + res["ip"] = ip_; + res["ws_port"] = wsPort_; + res["grpc_port"] = grpcPort_; + + auto last = subscriptionSource_->lastMessageTime(); + if (last.time_since_epoch().count() != 0) { + res["last_msg_age_seconds"] = std::to_string( + std::chrono::duration_cast(std::chrono::steady_clock::now() - last).count() + ); + } + + return res; + } + + /** @return String representation of the source (for debug) */ + std::string + toString() const final + { + return "{validated range: " + subscriptionSource_->validatedRange() + ", ip: " + ip_ + + ", web socket port: " + wsPort_ + ", grpc port: " + grpcPort_ + "}"; + } + + /** + * @brief Check if ledger is known by this source. + * + * @param sequence The ledger sequence to check + * @return true if ledger is in the range of this source; false otherwise + */ + bool + hasLedger(uint32_t sequence) const final + { + return subscriptionSource_->hasLedger(sequence); + } + + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param sequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true + * @param getObjectNeighbors Whether to request object neighbors; defaults to false + * @return A std::pair of the response status and the response itself + */ + std::pair + fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) final + { + return grpcSource_.fetchLedger(sequence, getObjects, getObjectNeighbors); + } + + /** + * @brief Download a ledger in full. + * + * @param sequence Sequence of the ledger to download + * @param numMarkers Number of markers to generate for async calls + * @param loader InitialLoadObserverInterface implementation + * @return A std::pair of the data and a bool indicating whether the download was successful + */ + std::pair, bool> + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) final + { + return grpcSource_.loadInitialLedger(sequence, numMarkers, loader); + } + + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param forwardToRippledClientIp IP of the client forwarding this request if known + * @param xUserValue Optional value of the X-User header + * @param yield The coroutine context + * @return Response or ClioError + */ + std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield + ) const final + { + return forwardingSource_.forwardToRippled(request, forwardToRippledClientIp, xUserValue, yield); + } +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/NFT.hpp b/src/etlng/impl/ext/NFT.hpp index 65e45233c..6d6a146cf 100644 --- a/src/etlng/impl/ext/NFT.hpp +++ b/src/etlng/impl/ext/NFT.hpp @@ -20,15 +20,11 @@ #pragma once #include "data/BackendInterface.hpp" -#include "data/DBHelpers.hpp" -#include "etl/NFTHelpers.hpp" #include "etlng/Models.hpp" #include "util/log/Logger.hpp" #include #include -#include -#include namespace etlng::impl { diff --git a/src/feed/SubscriptionManager.cpp b/src/feed/SubscriptionManager.cpp index 7db63718a..af7bd574f 100644 --- a/src/feed/SubscriptionManager.cpp +++ b/src/feed/SubscriptionManager.cpp @@ -191,7 +191,7 @@ SubscriptionManager::unsubBook(ripple::Book const& book, SubscriberSharedPtr con void SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& txMeta, ripple::LedgerHeader const& lgrInfo) { - transactionFeed_.pub(txMeta, lgrInfo, backend_, amendmentCenter_); + transactionFeed_.pub(txMeta, lgrInfo, backend_, amendmentCenter_, networkID_); } boost::json::object @@ -210,4 +210,16 @@ SubscriptionManager::report() const }; } +void +SubscriptionManager::setNetworkID(uint32_t const networkID) +{ + networkID_ = networkID; +} + +uint32_t +SubscriptionManager::getNetworkID() const +{ + return networkID_; +} + } // namespace feed diff --git a/src/feed/SubscriptionManager.hpp b/src/feed/SubscriptionManager.hpp index 09c004a5b..b227625df 100644 --- a/src/feed/SubscriptionManager.hpp +++ b/src/feed/SubscriptionManager.hpp @@ -31,8 +31,8 @@ #include "feed/impl/TransactionFeed.hpp" #include "util/async/AnyExecutionContext.hpp" #include "util/async/context/BasicExecutionContext.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -69,6 +69,7 @@ class SubscriptionManager : public SubscriptionManagerInterface { impl::BookChangesFeed bookChangesFeed_; impl::TransactionFeed transactionFeed_; impl::ProposedTransactionFeed proposedTransactionFeed_; + uint32_t networkID_{0}; public: /** @@ -332,6 +333,21 @@ public: */ boost::json::object report() const final; + + /** + * @brief Set the networkID. + * @param networkID The network id to set. + */ + void + setNetworkID(uint32_t networkID) final; + + /** + * @brief Get the networkID. + * + * @return The network id. + */ + uint32_t + getNetworkID() const final; }; } // namespace feed diff --git a/src/feed/SubscriptionManagerInterface.hpp b/src/feed/SubscriptionManagerInterface.hpp index 4339e5951..8303fb693 100644 --- a/src/feed/SubscriptionManagerInterface.hpp +++ b/src/feed/SubscriptionManagerInterface.hpp @@ -244,10 +244,25 @@ public: /** * @brief Get the number of subscribers. * - * @return The report of the number of subscribers + * @return The report of the number of subscribers. */ virtual boost::json::object report() const = 0; + + /** + * @brief Set the networkID. + * @param networkID The network id to set. + */ + virtual void + setNetworkID(uint32_t networkID) = 0; + + /** + * @brief Get the networkID. + * + * @return The network id. + */ + virtual uint32_t + getNetworkID() const = 0; }; } // namespace feed diff --git a/src/feed/impl/SingleFeedBase.hpp b/src/feed/impl/SingleFeedBase.hpp index d31eeb0c1..dd81e965f 100644 --- a/src/feed/impl/SingleFeedBase.hpp +++ b/src/feed/impl/SingleFeedBase.hpp @@ -50,7 +50,7 @@ public: /** * @brief Construct a new Single Feed Base object * @param executionCtx The actual publish will be called in the strand of this. - * @param name The promethues counter name of the feed. + * @param name The prometheus counter name of the feed. */ SingleFeedBase(util::async::AnyExecutionContext& executionCtx, std::string const& name); diff --git a/src/feed/impl/TrackableSignal.hpp b/src/feed/impl/TrackableSignal.hpp index f190be3c1..c0629ec3a 100644 --- a/src/feed/impl/TrackableSignal.hpp +++ b/src/feed/impl/TrackableSignal.hpp @@ -72,7 +72,7 @@ public: } // This class can't hold the trackable's shared_ptr, because disconnect should be able to be called in the - // the trackable's destructor. However, the trackable can not be destroied when the slot is being called + // the trackable's destructor. However, the trackable can not be destroyed when the slot is being called // either. track_foreign will hold a weak_ptr to the connection, which makes sure the connection is valid when // the slot is called. connections->emplace( diff --git a/src/feed/impl/TransactionFeed.cpp b/src/feed/impl/TransactionFeed.cpp index ac65cca6f..b2b85db4b 100644 --- a/src/feed/impl/TransactionFeed.cpp +++ b/src/feed/impl/TransactionFeed.cpp @@ -25,6 +25,7 @@ #include "feed/Types.hpp" #include "rpc/JS.hpp" #include "rpc/RPCHelpers.hpp" +#include "util/Assert.hpp" #include "util/log/Logger.hpp" #include @@ -176,7 +177,8 @@ TransactionFeed::pub( data::TransactionAndMetadata const& txMeta, ripple::LedgerHeader const& lgrInfo, std::shared_ptr const& backend, - std::shared_ptr const& amendmentCenter + std::shared_ptr const& amendmentCenter, + uint32_t const networkID ) { auto [tx, meta] = rpc::deserializeTxPlusMeta(txMeta, lgrInfo.seq); @@ -205,6 +207,15 @@ TransactionFeed::pub( rpc::insertDeliverMaxAlias(pubObj[txKey].as_object(), version); rpc::insertMPTIssuanceID(pubObj[JS(meta)].as_object(), tx, meta); + auto const& metaObj = pubObj[JS(meta)]; + ASSERT(metaObj.is_object(), "meta must be an obj in rippled and clio"); + if (metaObj.as_object().contains("TransactionIndex") && metaObj.as_object().at("TransactionIndex").is_int64()) { + if (auto const& ctid = + rpc::encodeCTID(lgrInfo.seq, metaObj.as_object().at("TransactionIndex").as_int64(), networkID); + ctid) + pubObj[JS(ctid)] = ctid.value(); + } + pubObj[JS(type)] = "transaction"; pubObj[JS(validated)] = true; pubObj[JS(status)] = "closed"; diff --git a/src/feed/impl/TransactionFeed.hpp b/src/feed/impl/TransactionFeed.hpp index 4500575d4..ac16ca79a 100644 --- a/src/feed/impl/TransactionFeed.hpp +++ b/src/feed/impl/TransactionFeed.hpp @@ -182,12 +182,14 @@ public: * @param txMeta The transaction and metadata. * @param lgrInfo The ledger header. * @param backend The backend. + * @param networkID The network ID. */ void pub(data::TransactionAndMetadata const& txMeta, ripple::LedgerHeader const& lgrInfo, std::shared_ptr const& backend, - std::shared_ptr const& amendmentCenter); + std::shared_ptr const& amendmentCenter, + uint32_t networkID); /** * @brief Get the number of subscribers of the transaction feed. diff --git a/src/main/Main.cpp b/src/main/Main.cpp index 6d1ff8e11..4aebd3c8f 100644 --- a/src/main/Main.cpp +++ b/src/main/Main.cpp @@ -23,8 +23,8 @@ #include "migration/MigrationApplication.hpp" #include "rpc/common/impl/HandlerProvider.hpp" #include "util/TerminationHandler.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include #include diff --git a/src/migration/MigrationApplication.cpp b/src/migration/MigrationApplication.cpp index e6d63569b..f0da3c286 100644 --- a/src/migration/MigrationApplication.cpp +++ b/src/migration/MigrationApplication.cpp @@ -22,8 +22,8 @@ #include "migration/MigratiorStatus.hpp" #include "migration/impl/MigrationManagerFactory.hpp" #include "util/OverloadSet.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include "util/prometheus/Prometheus.hpp" #include diff --git a/src/migration/MigrationApplication.hpp b/src/migration/MigrationApplication.hpp index f40a99530..b0f33c020 100644 --- a/src/migration/MigrationApplication.hpp +++ b/src/migration/MigrationApplication.hpp @@ -21,7 +21,7 @@ #include "data/LedgerCache.hpp" #include "migration/MigrationManagerInterface.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include #include diff --git a/src/migration/MigrationInspectorFactory.hpp b/src/migration/MigrationInspectorFactory.hpp index 2baa29e8b..e42de7289 100644 --- a/src/migration/MigrationInspectorFactory.hpp +++ b/src/migration/MigrationInspectorFactory.hpp @@ -24,8 +24,8 @@ #include "migration/MigratiorStatus.hpp" #include "migration/cassandra/CassandraMigrationManager.hpp" #include "util/Assert.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include #include diff --git a/src/migration/MigrationManagerInterface.hpp b/src/migration/MigrationManagerInterface.hpp index e76bfeb54..0a39283f0 100644 --- a/src/migration/MigrationManagerInterface.hpp +++ b/src/migration/MigrationManagerInterface.hpp @@ -28,7 +28,7 @@ namespace migration { /** * @brief The interface for the migration manager. The migration application layer will use this interface to run the * migrations. Unlike the MigrationInspectorInterface which only provides the status of migration, this interface - * contains the acutal migration running method. + * contains the actual migration running method. */ struct MigrationManagerInterface : virtual public MigrationInspectorInterface { /** diff --git a/src/migration/README.md b/src/migration/README.md index 9c297a894..ae01b880e 100644 --- a/src/migration/README.md +++ b/src/migration/README.md @@ -1,35 +1,27 @@ - -# Clio Migration +# Clio Migration Clio maintains the off-chain data of XRPL and multiple indexes tables to powering complex queries. To simplify the creation of index tables, this migration framework handles the process of database change and facilitates the migration of historical data seamlessly. - ## Command Line Usage -Clio provides a migration command-line tool to migrate data in database. +Clio provides a migration command-line tool to migrate data in database. +> [!NOTE] +> We need a **configuration file** to run the migration tool. This configuration file has the same format as the configuration file of the Clio server, ensuring consistency and ease of use. It reads the database configuration from the same session as the server's configuration, eliminating the need for separate setup or additional configuration files. Be aware that migration-specific configuration is under `.migration` session. -> Note: We need a **configuration file** to run the migration tool. This configuration file has the same format as the configuration file of the Clio server, ensuring consistency and ease of use. It reads the database configuration from the same session as the server's configuration, eliminating the need for separate setup or additional configuration files. Be aware that migration-specific configuration is under `.migration` session. +### Query migration status - -### To query migration status: - - ./clio_server --migrate status ~/config/migrator.json - -This command returns the current migration status of each migrator. The example output: - +This command returns the current migration status of each migrator. The example output: + Current Migration Status: Migrator: ExampleMigrator - Feature v1, Clio v3 - not migrated - -### To start a migration: - - +### Start a migration + ./clio_server --migrate ExampleMigrator ~/config/migrator.json - - + Migration will run if the migrator has not been migrated. The migrator will be marked as migrated after the migration is completed. ## How to write a migrator @@ -52,44 +44,48 @@ It contains: > **Note** Each migrator is designed to work with a specific database. -- Register your migrator in MigrationManager. Currently we only support Cassandra/ScyllaDB. Migrator needs to be registered in `CassandraSupportedMigrators`. - +- Register your migrator in MigrationManager. Currently we only support Cassandra/ScyllaDB. Migrator needs to be registered in `CassandraSupportedMigrators`. ## How to use full table scanner (Only for Cassandra/ScyllaDB) -Sometimes migrator isn't able to query the historical data by table's partition key. For example, migrator of transactions needs the historical transaction data without knowing each transaction hash. Full table scanner can help to get all the rows in parallel. + +Sometimes migrator isn't able to query the historical data by table's partition key. For example, migrator of transactions needs the historical transaction data without knowing each transaction hash. Full table scanner can help to get all the rows in parallel. Most indexes are based on either ledger states or transactions. We provide the `objects` and `transactions` scanner. Developers only need to implement the callback function to receive the historical data. Please find the examples in `tests/integration/migration/cassandra/ExampleTransactionsMigrator.cpp` and `tests/integration/migration/cassandra/ExampleObjectsMigrator.cpp`. -> **Note** The full table scanner splits the table into multiple ranges by token(https://opensource.docs.scylladb.com/stable/cql/functions.html#token). A few of rows maybe read 2 times if its token happens to be at the edge of ranges. **Deduplication is needed** in the callback function. +> [!NOTE] +> The full table scanner splits the table into multiple ranges by token(). A few of rows maybe read 2 times if its token happens to be at the edge of ranges. **Deduplication is needed** in the callback function. ## How to write a full table scan adapter (Only for Cassandra/ScyllaDB) If you need to do full scan against other table, you can follow below steps: + - Describe the table which needs full scan in a struct. It has to satisfy the `TableSpec`(cassandra/Spec.hpp) concept, containing static member: - - Tuple type `Row`, it's the type of each field in a row. The order of types should match what database will return in a row. Key types should come first, followed by other field types sorted in alphabetical order. - - `kPARTITION_KEY`, it's the name of the partition key of the table. - - `kTABLE_NAME` + + - Tuple type `Row`, it's the type of each field in a row. The order of types should match what database will return in a row. Key types should come first, followed by other field types sorted in alphabetical order. + - `kPARTITION_KEY`, it's the name of the partition key of the table. + - `kTABLE_NAME` - Inherent from `FullTableScannerAdapterBase`. - Implement `onRowRead`, its parameter is the `Row` we defined. It's the callback function when a row is read. - Please take ObjectsAdapter/TransactionsAdapter as example. -## Examples: +## Examples We have some example migrators under `tests/integration/migration/cassandra` folder. - ExampleDropTableMigrator - This migrator drops `diff` table. + This migrator drops `diff` table. + - ExampleLedgerMigrator - This migrator shows how to migrate data when we don't need to do full table scan. This migrator creates an index table `ledger_example` which maintains the map of ledger sequence and its account hash. + This migrator shows how to migrate data when we don't need to do full table scan. This migrator creates an index table `ledger_example` which maintains the map of ledger sequence and its account hash. + - ExampleObjectsMigrator - This migrator shows how to migrate ledger states related data. It uses `ObjectsScanner` to proceed the full scan in parallel. It counts the number of ACCOUNT_ROOT. + This migrator shows how to migrate ledger states related data. It uses `ObjectsScanner` to proceed the full scan in parallel. It counts the number of ACCOUNT_ROOT. + - ExampleTransactionsMigrator - This migrator shows how to migrate transactions related data. It uses `TransactionsScanner` to proceed the `transactions` table full scan in parallel. It creates an index table `tx_index_example` which tracks the transaction hash and its according transaction type. - + This migrator shows how to migrate transactions related data. It uses `TransactionsScanner` to proceed the `transactions` table full scan in parallel. It creates an index table `tx_index_example` which tracks the transaction hash and its according transaction type. diff --git a/src/migration/cassandra/CassandraMigrationManager.hpp b/src/migration/cassandra/CassandraMigrationManager.hpp index a658a8201..63a4c1d17 100644 --- a/src/migration/cassandra/CassandraMigrationManager.hpp +++ b/src/migration/cassandra/CassandraMigrationManager.hpp @@ -33,7 +33,7 @@ template using CassandraSupportedMigrators = migration::impl::MigratorsRegister; // Instantiates with the backend which supports actual migration running -using MigrationProcesser = CassandraSupportedMigrators; +using MigrationProcessor = CassandraSupportedMigrators; // Instantiates with backend interface, it doesn't support actual migration. But it can be used to inspect the migrators // status @@ -45,6 +45,6 @@ namespace migration::cassandra { using CassandraMigrationInspector = migration::impl::MigrationInspectorBase; -using CassandraMigrationManager = migration::impl::MigrationManagerBase; +using CassandraMigrationManager = migration::impl::MigrationManagerBase; } // namespace migration::cassandra diff --git a/src/migration/cassandra/impl/CassandraMigrationSchema.hpp b/src/migration/cassandra/impl/CassandraMigrationSchema.hpp index 56fde4985..d59864db4 100644 --- a/src/migration/cassandra/impl/CassandraMigrationSchema.hpp +++ b/src/migration/cassandra/impl/CassandraMigrationSchema.hpp @@ -65,8 +65,8 @@ public: { return handler.prepare(fmt::format( R"( - SELECT * - FROM {} + SELECT * + FROM {} WHERE TOKEN({}) >= ? AND TOKEN({}) <= ? )", data::cassandra::qualifiedTableName(settingsProvider_.get(), tableName), @@ -86,7 +86,7 @@ public: { static auto kPREPARED = handler.prepare(fmt::format( R"( - INSERT INTO {} + INSERT INTO {} (migrator_name, status) VALUES (?, ?) )", diff --git a/src/migration/cassandra/impl/Spec.hpp b/src/migration/cassandra/impl/Spec.hpp index a7ed3748b..c6c6089dc 100644 --- a/src/migration/cassandra/impl/Spec.hpp +++ b/src/migration/cassandra/impl/Spec.hpp @@ -30,7 +30,7 @@ namespace migration::cassandra::impl { template concept TableSpec = requires { // Check that 'row' exists and is a tuple - // keys types are at the begining and the other fields types sort in alphabetical order + // keys types are at the beginning and the other fields types sort in alphabetical order typename T::Row; requires std::tuple_size::value >= 0; // Ensures 'row' is a tuple diff --git a/src/migration/impl/MigrationInspectorBase.hpp b/src/migration/impl/MigrationInspectorBase.hpp index fe712ec6b..106ba9556 100644 --- a/src/migration/impl/MigrationInspectorBase.hpp +++ b/src/migration/impl/MigrationInspectorBase.hpp @@ -34,7 +34,7 @@ namespace migration::impl { * @brief The migration inspector implementation for Cassandra. It will report the migration status for Cassandra * database. * - * @tparam SupportedMigrators The migrators resgister that contains all the migrators + * @tparam SupportedMigrators The migrators register that contains all the migrators */ template class MigrationInspectorBase : virtual public MigrationInspectorInterface { @@ -101,7 +101,7 @@ public: } /** - * @brief Return if there is uncomplete migrator blocking the server + * @brief Return if there is incomplete migrator blocking the server * * @return True if server is blocked, false otherwise */ diff --git a/src/migration/impl/MigrationManagerBase.hpp b/src/migration/impl/MigrationManagerBase.hpp index 76c21231a..afe891755 100644 --- a/src/migration/impl/MigrationManagerBase.hpp +++ b/src/migration/impl/MigrationManagerBase.hpp @@ -21,7 +21,7 @@ #include "migration/MigrationManagerInterface.hpp" #include "migration/impl/MigrationInspectorBase.hpp" -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ObjectView.hpp" #include #include @@ -33,7 +33,7 @@ namespace migration::impl { * @brief The migration manager implementation for Cassandra. It will run the migration for the Cassandra * database. * - * @tparam SupportedMigrators The migrators resgister that contains all the migrators + * @tparam SupportedMigrators The migrators register that contains all the migrators */ template class MigrationManagerBase : public MigrationManagerInterface, public MigrationInspectorBase { diff --git a/src/migration/impl/MigrationManagerFactory.cpp b/src/migration/impl/MigrationManagerFactory.cpp index e83c1e476..216171f02 100644 --- a/src/migration/impl/MigrationManagerFactory.cpp +++ b/src/migration/impl/MigrationManagerFactory.cpp @@ -24,8 +24,8 @@ #include "migration/MigrationManagerInterface.hpp" #include "migration/cassandra/CassandraMigrationBackend.hpp" #include "migration/cassandra/CassandraMigrationManager.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include diff --git a/src/migration/impl/MigrationManagerFactory.hpp b/src/migration/impl/MigrationManagerFactory.hpp index 33bc14073..6de3dbe89 100644 --- a/src/migration/impl/MigrationManagerFactory.hpp +++ b/src/migration/impl/MigrationManagerFactory.hpp @@ -21,7 +21,7 @@ #include "data/LedgerCacheInterface.hpp" #include "migration/MigrationManagerInterface.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include #include @@ -30,7 +30,7 @@ namespace migration::impl { /** - * @brief The factory to create a MigrationManagerInferface + * @brief The factory to create a MigrationManagerInterface * * @param config The configuration of the migration application, it contains the database connection configuration and * other migration specific configurations diff --git a/src/migration/impl/MigratorsRegister.hpp b/src/migration/impl/MigratorsRegister.hpp index 95bbc783e..ef571eb10 100644 --- a/src/migration/impl/MigratorsRegister.hpp +++ b/src/migration/impl/MigratorsRegister.hpp @@ -24,8 +24,8 @@ #include "migration/impl/Spec.hpp" #include "util/Assert.hpp" #include "util/Concepts.hpp" +#include "util/config/ObjectView.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ObjectView.hpp" #include #include diff --git a/src/migration/impl/Spec.hpp b/src/migration/impl/Spec.hpp index d9d379441..59decd1fd 100644 --- a/src/migration/impl/Spec.hpp +++ b/src/migration/impl/Spec.hpp @@ -20,7 +20,7 @@ #pragma once -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ObjectView.hpp" #include diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 557e843d6..bce9f8475 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,3 +1,8 @@ +# Have to use RPCCenter as a separate library since it is used in util +add_library(clio_rpc_center) +target_sources(clio_rpc_center PRIVATE RPCCenter.cpp) +target_include_directories(clio_rpc_center PUBLIC "${CMAKE_SOURCE_DIR}/src") + add_library(clio_rpc) target_sources( diff --git a/src/rpc/Errors.cpp b/src/rpc/Errors.cpp index 2609fc881..b57f85bf7 100644 --- a/src/rpc/Errors.cpp +++ b/src/rpc/Errors.cpp @@ -97,8 +97,8 @@ getErrorInfo(ClioError code) .message = "Method is not specified or is not a string."}, {.code = ClioError::RpcCommandNotString, .error = "commandNotString", .message = "Method is not a string."}, {.code = ClioError::RpcCommandIsEmpty, .error = "emptyCommand", .message = "Method is an empty string."}, - {.code = ClioError::RpcParamsUnparseable, - .error = "paramsUnparseable", + {.code = ClioError::RpcParamsUnparsable, + .error = "paramsUnparsable", .message = "Params must be an array holding exactly one object."}, // etl related errors {.code = ClioError::EtlConnectionError, .error = "connectionError", .message = "Couldn't connect to rippled."}, diff --git a/src/rpc/Errors.hpp b/src/rpc/Errors.hpp index 4ea7a4ebc..e3735a86c 100644 --- a/src/rpc/Errors.hpp +++ b/src/rpc/Errors.hpp @@ -49,7 +49,7 @@ enum class ClioError { RpcCommandIsMissing = 6001, RpcCommandNotString = 6002, RpcCommandIsEmpty = 6003, - RpcParamsUnparseable = 6004, + RpcParamsUnparsable = 6004, // TODO: Since it is not only rpc errors here now, we should move it to util // etl related errors start with 7000 diff --git a/src/rpc/Factories.cpp b/src/rpc/Factories.cpp index 0747a343e..a4b2bf2cb 100644 --- a/src/rpc/Factories.cpp +++ b/src/rpc/Factories.cpp @@ -99,12 +99,12 @@ makeHttpContext( return Error{{RippledError::rpcBAD_SYNTAX, "Subscribe and unsubscribe are only allowed for websocket."}}; if (!request.at("params").is_array()) - return Error{{ClioError::RpcParamsUnparseable, "Missing params array."}}; + return Error{{ClioError::RpcParamsUnparsable, "Missing params array."}}; boost::json::array const& array = request.at("params").as_array(); if (array.size() != 1 || !array.at(0).is_object()) - return Error{{ClioError::RpcParamsUnparseable}}; + return Error{{ClioError::RpcParamsUnparsable}}; auto const apiVersion = apiVersionParser.get().parse(request.at("params").as_array().at(0).as_object()); if (!apiVersion) diff --git a/src/rpc/README.md b/src/rpc/README.md index bc193a9c5..7de7fbe80 100644 --- a/src/rpc/README.md +++ b/src/rpc/README.md @@ -17,7 +17,7 @@ See [tests/unit/rpc](https://github.com/XRPLF/clio/tree/develop/tests/unit/rpc) Handlers need to fulfil the requirements specified by the `SomeHandler` concept (see `rpc/common/Concepts.hpp`): - Expose types: - + - `Input` - The POD struct which acts as input for the handler - `Output` - The POD struct which acts as output of a valid handler invocation diff --git a/src/rpc/RPCCenter.cpp b/src/rpc/RPCCenter.cpp new file mode 100644 index 000000000..ce2e9bb11 --- /dev/null +++ b/src/rpc/RPCCenter.cpp @@ -0,0 +1,112 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "rpc/RPCCenter.hpp" + +#include +#include + +namespace rpc { + +namespace { + +std::unordered_set const& +handledRpcs() +{ + static std::unordered_set const kHANDLED_RPCS = { + "account_channels", + "account_currencies", + "account_info", + "account_lines", + "account_nfts", + "account_objects", + "account_offers", + "account_tx", + "amm_info", + "book_changes", + "book_offers", + "deposit_authorized", + "feature", + "gateway_balances", + "get_aggregate_price", + "ledger", + "ledger_data", + "ledger_entry", + "ledger_index", + "ledger_range", + "mpt_holders", + "nfts_by_issuer", + "nft_history", + "nft_buy_offers", + "nft_info", + "nft_sell_offers", + "noripple_check", + "ping", + "random", + "server_info", + "transaction_entry", + "tx", + "subscribe", + "unsubscribe", + "version", + }; + return kHANDLED_RPCS; +} + +std::unordered_set const& +forwardedRpcs() +{ + static std::unordered_set const kFORWARDED_RPCS = { + "server_definitions", + "server_state", + "submit", + "submit_multisigned", + "fee", + "ledger_closed", + "ledger_current", + "ripple_path_find", + "manifest", + "channel_authorize", + "channel_verify", + "simulate", + }; + return kFORWARDED_RPCS; +} + +} // namespace + +bool +RPCCenter::isRpcName(std::string_view s) +{ + return isHandled(s) || isForwarded(s); +} + +bool +RPCCenter::isHandled(std::string_view s) +{ + return handledRpcs().contains(s); +} + +bool +RPCCenter::isForwarded(std::string_view s) +{ + return forwardedRpcs().contains(s); +} + +} // namespace rpc diff --git a/src/rpc/RPCCenter.hpp b/src/rpc/RPCCenter.hpp new file mode 100644 index 000000000..c772c5a25 --- /dev/null +++ b/src/rpc/RPCCenter.hpp @@ -0,0 +1,61 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include + +namespace rpc { + +/** + * @brief Registry of RPC commands supported by Clio + * + * The RPCCenter maintains lists of RPC commands that can be handled locally + * and those that need to be forwarded to rippled. + */ +struct RPCCenter { + /** + * @brief Checks if a string is a valid RPC command name + * + * @param s The string to check + * @return true if the string is a recognized RPC name, false otherwise + */ + static bool + isRpcName(std::string_view s); + + /** + * @brief Checks if a string is a RPC command handled by Clio without forwarding to rippled + * + * @param s The string to check + * @return true if the string is a handled RPC command, false otherwise + */ + static bool + isHandled(std::string_view s); + + /** + * @brief Checks if a string is a RPC command that will be forwarded to rippled + * + * @param s The string to check + * @return true if the string is a forwarded RPC command, false otherwise + */ + static bool + isForwarded(std::string_view s); +}; + +} // namespace rpc diff --git a/src/rpc/RPCEngine.hpp b/src/rpc/RPCEngine.hpp index 13216c20d..119665b59 100644 --- a/src/rpc/RPCEngine.hpp +++ b/src/rpc/RPCEngine.hpp @@ -20,12 +20,14 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/RPCHelpers.hpp" #include "rpc/WorkQueue.hpp" #include "rpc/common/HandlerProvider.hpp" #include "rpc/common/Types.hpp" #include "rpc/common/impl/ForwardingProxy.hpp" +#include "util/OverloadSet.hpp" #include "util/ResponseExpirationCache.hpp" #include "util/log/Logger.hpp" #include "web/Context.hpp" @@ -34,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -55,7 +58,7 @@ namespace rpc { /** * @brief The RPC engine that ties all RPC-related functionality together. */ -template +template class RPCEngine { util::Logger perfLog_{"Performance"}; util::Logger log_{"RPC"}; @@ -67,7 +70,7 @@ class RPCEngine { std::shared_ptr handlerProvider_; - impl::ForwardingProxy forwardingProxy_; + impl::ForwardingProxy forwardingProxy_; std::optional responseCache_; @@ -86,7 +89,7 @@ public: RPCEngine( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, - std::shared_ptr const& balancer, + std::shared_ptr const& balancer, web::dosguard::DOSGuardInterface const& dosGuard, WorkQueue& workQueue, CountersType& counters, @@ -128,7 +131,7 @@ public: makeRPCEngine( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, - std::shared_ptr const& balancer, + std::shared_ptr const& balancer, web::dosguard::DOSGuardInterface const& dosGuard, WorkQueue& workQueue, CountersType& counters, @@ -155,55 +158,51 @@ public: return forwardingProxy_.forward(ctx); } - if (not ctx.isAdmin and responseCache_) { - if (auto res = responseCache_->get(ctx.method); res.has_value()) - return Result{std::move(res).value()}; - } - - if (backend_->isTooBusy()) { - LOG(log_.error()) << "Database is too busy. Rejecting request"; - notifyTooBusy(); // TODO: should we add ctx.method if we have it? - return Result{Status{RippledError::rpcTOO_BUSY}}; - } - - auto const method = handlerProvider_->getHandler(ctx.method); - if (!method) { - notifyUnknownCommand(); - return Result{Status{RippledError::rpcUNKNOWN_COMMAND}}; - } - - try { - LOG(perfLog_.debug()) << ctx.tag() << " start executing rpc `" << ctx.method << '`'; - - auto const context = Context{ - .yield = ctx.yield, - .session = ctx.session, - .isAdmin = ctx.isAdmin, - .clientIp = ctx.clientIp, - .apiVersion = ctx.apiVersion + if (not ctx.isAdmin and responseCache_ and responseCache_->shouldCache(ctx.method)) { + auto updater = + [this, &ctx](boost::asio::yield_context + ) -> std::expected { + auto result = buildResponseImpl(ctx); + auto extracted = std::visit( + util::OverloadSet{ + [&result](Status status + ) -> std::expected { + return std::unexpected{util::ResponseExpirationCache::Error{ + .status = std::move(status), .warnings = std::move(result.warnings) + }}; + }, + [](boost::json::object obj + ) -> std::expected { return obj; } + }, + std::move(result.response) + ); + if (extracted.has_value()) { + return util::ResponseExpirationCache::EntryData{ + .lastUpdated = std::chrono::steady_clock::now(), .response = std::move(extracted).value() + }; + } + return std::unexpected{std::move(extracted).error()}; }; - auto v = (*method).process(ctx.params, context); - LOG(perfLog_.debug()) << ctx.tag() << " finish executing rpc `" << ctx.method << '`'; - - if (not v) { - notifyErrored(ctx.method); - } else if (not ctx.isAdmin and responseCache_) { - responseCache_->put(ctx.method, v.result->as_object()); + auto result = responseCache_->getOrUpdate( + ctx.yield, + ctx.method, + std::move(updater), + [&ctx](util::ResponseExpirationCache::EntryData const& entry) { + return not ctx.isAdmin and not entry.response.contains("error"); + } + ); + if (result.has_value()) { + return Result{std::move(result).value()}; } - return Result{std::move(v)}; - } catch (data::DatabaseTimeout const& t) { - LOG(log_.error()) << "Database timeout"; - notifyTooBusy(); - - return Result{Status{RippledError::rpcTOO_BUSY}}; - } catch (std::exception const& ex) { - LOG(log_.error()) << ctx.tag() << "Caught exception: " << ex.what(); - notifyInternalError(); - - return Result{Status{RippledError::rpcINTERNAL}}; + auto error = std::move(result).error(); + Result errorResult{std::move(error.status)}; + errorResult.warnings = std::move(error.warnings); + return errorResult; } + + return buildResponseImpl(ctx); } /** @@ -252,7 +251,7 @@ public: /** * @brief Notify the system that specified method failed due to some unrecoverable error. * - * Used for erors such as database timeout, internal errors, etc. + * Used for errors such as database timeout, internal errors, etc. * * @param method */ @@ -316,6 +315,53 @@ private: { return handlerProvider_->contains(method) || forwardingProxy_.isProxied(method); } + + Result + buildResponseImpl(web::Context const& ctx) + { + if (backend_->isTooBusy()) { + LOG(log_.error()) << "Database is too busy. Rejecting request"; + notifyTooBusy(); // TODO: should we add ctx.method if we have it? + return Result{Status{RippledError::rpcTOO_BUSY}}; + } + + auto const method = handlerProvider_->getHandler(ctx.method); + if (!method) { + notifyUnknownCommand(); + return Result{Status{RippledError::rpcUNKNOWN_COMMAND}}; + } + + try { + LOG(perfLog_.debug()) << ctx.tag() << " start executing rpc `" << ctx.method << '`'; + + auto const context = Context{ + .yield = ctx.yield, + .session = ctx.session, + .isAdmin = ctx.isAdmin, + .clientIp = ctx.clientIp, + .apiVersion = ctx.apiVersion + }; + auto v = (*method).process(ctx.params, context); + + LOG(perfLog_.debug()) << ctx.tag() << " finish executing rpc `" << ctx.method << '`'; + + if (not v) { + notifyErrored(ctx.method); + } + + return Result{std::move(v)}; + } catch (data::DatabaseTimeout const& t) { + LOG(log_.error()) << "Database timeout"; + notifyTooBusy(); + + return Result{Status{RippledError::rpcTOO_BUSY}}; + } catch (std::exception const& ex) { + LOG(log_.error()) << ctx.tag() << "Caught exception: " << ex.what(); + notifyInternalError(); + + return Result{Status{RippledError::rpcINTERNAL}}; + } + } }; } // namespace rpc diff --git a/src/rpc/RPCHelpers.cpp b/src/rpc/RPCHelpers.cpp index 651b84120..c11d58d41 100644 --- a/src/rpc/RPCHelpers.cpp +++ b/src/rpc/RPCHelpers.cpp @@ -290,7 +290,10 @@ std::optional encodeCTID(uint32_t ledgerSeq, uint16_t txnIndex, uint16_t networkId) noexcept { static constexpr uint32_t kMAX_LEDGER_SEQ = 0x0FFF'FFFF; - if (ledgerSeq > kMAX_LEDGER_SEQ) + static constexpr uint32_t kMAX_TXN_INDEX = 0xFFFF; + static constexpr uint32_t kMAX_NETWORK_ID = 0xFFFF; + + if (ledgerSeq > kMAX_LEDGER_SEQ || txnIndex > kMAX_TXN_INDEX || networkId > kMAX_NETWORK_ID) return {}; static constexpr uint64_t kCTID_PREFIX = 0xC000'0000; @@ -1264,7 +1267,7 @@ postProcessOrderBook( ripple::STAmount const dirRate = ripple::amountFromQuality(getQuality(bookDir)); if (rate != ripple::parityRate - // Have a tranfer fee. + // Have a transfer fee. && takerID != book.out.account // Not taking offers of own IOUs. && book.out.account != uOfferOwnerID) diff --git a/src/rpc/RPCHelpers.hpp b/src/rpc/RPCHelpers.hpp index 040a211a0..a2c830517 100644 --- a/src/rpc/RPCHelpers.hpp +++ b/src/rpc/RPCHelpers.hpp @@ -30,6 +30,7 @@ #include "rpc/Errors.hpp" #include "rpc/common/Types.hpp" #include "util/JsonUtils.hpp" +#include "util/Taggable.hpp" #include "util/log/Logger.hpp" #include "web/Context.hpp" @@ -659,7 +660,7 @@ ripple::Issue parseIssue(boost::json::object const& issue); /** - * @brief Check whethe the request specifies the `current` or `closed` ledger + * @brief Check whether the request specifies the `current` or `closed` ledger * @param request The request to check * @return true if the request specifies the `current` or `closed` ledger */ @@ -744,12 +745,13 @@ decodeCTID(T const ctid) noexcept * @brief Log the duration of the request processing * * @tparam T The type of the duration - * @param ctx The context of the request + * @param request The request to log + * @param tag The tag of the context of the request * @param dur The duration to log */ -template +template void -logDuration(web::Context const& ctx, T const& dur) +logDuration(boost::json::object const& request, util::BaseTagDecorator const& tag, DurationType const& dur) { using boost::json::serialize; @@ -759,15 +761,15 @@ logDuration(web::Context const& ctx, T const& dur) auto const millis = std::chrono::duration_cast(dur).count(); auto const seconds = std::chrono::duration_cast(dur).count(); auto const msg = fmt::format( - "Request processing duration = {} milliseconds. request = {}", millis, serialize(util::removeSecret(ctx.params)) + "Request processing duration = {} milliseconds. request = {}", millis, serialize(util::removeSecret(request)) ); if (seconds > kDURATION_ERROR_THRESHOLD_SECONDS) { - LOG(log.error()) << ctx.tag() << msg; + LOG(log.error()) << tag << msg; } else if (seconds > 1) { - LOG(log.warn()) << ctx.tag() << msg; + LOG(log.warn()) << tag << msg; } else - LOG(log.info()) << ctx.tag() << msg; + LOG(log.info()) << tag << msg; } /** diff --git a/src/rpc/WorkQueue.cpp b/src/rpc/WorkQueue.cpp index 6f946f966..b19ccc7fe 100644 --- a/src/rpc/WorkQueue.cpp +++ b/src/rpc/WorkQueue.cpp @@ -58,7 +58,7 @@ WorkQueue::WorkQueue(std::uint32_t numWorkers, uint32_t maxSize) "The total number of tasks queued for processing" )} , durationUs_{PrometheusService::counterInt( - "work_queue_cumulitive_tasks_duration_us", + "work_queue_cumulative_tasks_duration_us", util::prometheus::Labels(), "The total number of microseconds tasks were waiting to be executed" )} diff --git a/src/rpc/WorkQueue.hpp b/src/rpc/WorkQueue.hpp index 2472135e3..f74f72f35 100644 --- a/src/rpc/WorkQueue.hpp +++ b/src/rpc/WorkQueue.hpp @@ -20,8 +20,8 @@ #pragma once #include "util/Mutex.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include "util/prometheus/Counter.hpp" #include "util/prometheus/Gauge.hpp" @@ -167,7 +167,7 @@ public: /** * @brief Get the size of the queue. * - * @return The numver of jobs in the queue. + * @return The number of jobs in the queue. */ size_t size() const; diff --git a/src/rpc/common/impl/APIVersionParser.cpp b/src/rpc/common/impl/APIVersionParser.cpp index 445979bec..2d5e4c2d6 100644 --- a/src/rpc/common/impl/APIVersionParser.cpp +++ b/src/rpc/common/impl/APIVersionParser.cpp @@ -19,8 +19,8 @@ #include "rpc/common/impl/APIVersionParser.hpp" +#include "util/config/ObjectView.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ObjectView.hpp" #include #include diff --git a/src/rpc/common/impl/APIVersionParser.hpp b/src/rpc/common/impl/APIVersionParser.hpp index 3fa84d21c..2852446ea 100644 --- a/src/rpc/common/impl/APIVersionParser.hpp +++ b/src/rpc/common/impl/APIVersionParser.hpp @@ -20,8 +20,8 @@ #pragma once #include "rpc/common/APIVersion.hpp" +#include "util/config/ObjectView.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ObjectView.hpp" #include diff --git a/src/rpc/common/impl/ForwardingProxy.hpp b/src/rpc/common/impl/ForwardingProxy.hpp index c60773df1..8bf96ceb0 100644 --- a/src/rpc/common/impl/ForwardingProxy.hpp +++ b/src/rpc/common/impl/ForwardingProxy.hpp @@ -19,7 +19,9 @@ #pragma once +#include "etlng/LoadBalancerInterface.hpp" #include "rpc/Errors.hpp" +#include "rpc/RPCCenter.hpp" #include "rpc/RPCHelpers.hpp" #include "rpc/common/Types.hpp" #include "util/log/Logger.hpp" @@ -31,20 +33,21 @@ #include #include #include +#include namespace rpc::impl { -template +template class ForwardingProxy { util::Logger log_{"RPC"}; - std::shared_ptr balancer_; + std::shared_ptr balancer_; std::reference_wrapper counters_; std::shared_ptr handlerProvider_; public: ForwardingProxy( - std::shared_ptr const& balancer, + std::shared_ptr const& balancer, CountersType& counters, std::shared_ptr const& handlerProvider ) @@ -104,22 +107,7 @@ public: bool isProxied(std::string const& method) const { - static std::unordered_set const kPROXIED_COMMANDS{ - "server_definitions", - "server_state", - "submit", - "submit_multisigned", - "fee", - "ledger_closed", - "ledger_current", - "ripple_path_find", - "manifest", - "channel_authorize", - "channel_verify", - "simulate", - }; - - return kPROXIED_COMMANDS.contains(method); + return RPCCenter::isForwarded(method); } private: diff --git a/src/rpc/common/impl/HandlerProvider.cpp b/src/rpc/common/impl/HandlerProvider.cpp index cf4485cc2..83f93e99a 100644 --- a/src/rpc/common/impl/HandlerProvider.cpp +++ b/src/rpc/common/impl/HandlerProvider.cpp @@ -21,7 +21,8 @@ #include "data/AmendmentCenterInterface.hpp" #include "data/BackendInterface.hpp" -#include "etl/ETLService.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Counters.hpp" #include "rpc/common/AnyHandler.hpp" @@ -60,11 +61,12 @@ #include "rpc/handlers/Tx.hpp" #include "rpc/handlers/Unsubscribe.hpp" #include "rpc/handlers/VersionHandler.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include #include #include +#include namespace rpc::impl { @@ -72,8 +74,8 @@ ProductionHandlerProvider::ProductionHandlerProvider( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& subscriptionManager, - std::shared_ptr const& balancer, - std::shared_ptr const& etl, + std::shared_ptr const& balancer, + std::shared_ptr const& etl, std::shared_ptr const& amendmentCenter, Counters const& counters ) @@ -85,7 +87,7 @@ ProductionHandlerProvider::ProductionHandlerProvider( {"account_nfts", {.handler = AccountNFTsHandler{backend}}}, {"account_objects", {.handler = AccountObjectsHandler{backend}}}, {"account_offers", {.handler = AccountOffersHandler{backend}}}, - {"account_tx", {.handler = AccountTxHandler{backend}}}, + {"account_tx", {.handler = AccountTxHandler{backend, etl}}}, {"amm_info", {.handler = AMMInfoHandler{backend, amendmentCenter}}}, {"book_changes", {.handler = BookChangesHandler{backend}}}, {"book_offers", {.handler = BookOffersHandler{backend, amendmentCenter}}}, @@ -138,4 +140,13 @@ ProductionHandlerProvider::isClioOnly(std::string const& command) const return handlerMap_.contains(command) && handlerMap_.at(command).isClioOnly; } +std::unordered_set +ProductionHandlerProvider::handlerNames() const +{ + std::unordered_set result; + for (auto const& [name, handler] : handlerMap_) + result.insert(name); + return result; +} + } // namespace rpc::impl diff --git a/src/rpc/common/impl/HandlerProvider.hpp b/src/rpc/common/impl/HandlerProvider.hpp index 89ea5661f..0b54be3c1 100644 --- a/src/rpc/common/impl/HandlerProvider.hpp +++ b/src/rpc/common/impl/HandlerProvider.hpp @@ -21,6 +21,8 @@ #include "data/AmendmentCenterInterface.hpp" #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/common/AnyHandler.hpp" #include "rpc/common/HandlerProvider.hpp" @@ -31,11 +33,8 @@ #include #include #include +#include -namespace etl { -class ETLService; -class LoadBalancer; -} // namespace etl namespace rpc { class Counters; } // namespace rpc @@ -55,8 +54,8 @@ public: util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& subscriptionManager, - std::shared_ptr const& balancer, - std::shared_ptr const& etl, + std::shared_ptr const& balancer, + std::shared_ptr const& etl, std::shared_ptr const& amendmentCenter, Counters const& counters ); @@ -69,6 +68,9 @@ public: bool isClioOnly(std::string const& command) const override; + + std::unordered_set + handlerNames() const; }; } // namespace rpc::impl diff --git a/src/rpc/handlers/AccountLines.cpp b/src/rpc/handlers/AccountLines.cpp index 5644e7daf..6459e27d6 100644 --- a/src/rpc/handlers/AccountLines.cpp +++ b/src/rpc/handlers/AccountLines.cpp @@ -86,8 +86,9 @@ AccountLinesHandler::addLine( bool const lineNoRipplePeer = (flags & (not viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple)) != 0u; bool const lineFreeze = (flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze)) != 0u; bool const lineFreezePeer = (flags & (not viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze)) != 0u; - bool const lineDeepFreeze = (flags & (viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighFreeze)) != 0u; - bool const lineDeepFreezePeer = (flags & (not viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighFreeze)) != 0u; + bool const lineDeepFreeze = (flags & (viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighDeepFreeze)) != 0u; + bool const lineDeepFreezePeer = + (flags & (not viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighDeepFreeze)) != 0u; ripple::STAmount const& saBalance = balance; ripple::STAmount const& saLimit = lineLimit; diff --git a/src/rpc/handlers/AccountTx.cpp b/src/rpc/handlers/AccountTx.cpp index 9b9c68adf..882d5a983 100644 --- a/src/rpc/handlers/AccountTx.cpp +++ b/src/rpc/handlers/AccountTx.cpp @@ -161,6 +161,18 @@ AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx) con auto const txKey = ctx.apiVersion < 2u ? JS(tx) : JS(tx_json); obj[JS(meta)] = std::move(meta); obj[txKey] = std::move(txn); + + // Put CTID into tx or tx_json + if (obj[JS(meta)].as_object().contains("TransactionIndex")) { + auto networkID = 0u; + if (auto const& etlState = etl_->getETLState(); etlState.has_value()) + networkID = etlState->networkID; + + auto const txnIdx = obj[JS(meta)].as_object().at("TransactionIndex").as_int64(); + if (auto const& ctid = rpc::encodeCTID(txnPlusMeta.ledgerSequence, txnIdx, networkID); ctid) + obj[txKey].as_object()[JS(ctid)] = ctid.value(); + } + obj[txKey].as_object()[JS(date)] = txnPlusMeta.date; obj[txKey].as_object()[JS(ledger_index)] = txnPlusMeta.ledgerSequence; diff --git a/src/rpc/handlers/AccountTx.hpp b/src/rpc/handlers/AccountTx.hpp index 0e620a8ca..14551c424 100644 --- a/src/rpc/handlers/AccountTx.hpp +++ b/src/rpc/handlers/AccountTx.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/JS.hpp" #include "rpc/common/JsonBool.hpp" @@ -55,6 +56,7 @@ namespace rpc { class AccountTxHandler { util::Logger log_{"RPC"}; std::shared_ptr sharedPtrBackend_; + std::shared_ptr etl_; public: static constexpr auto kLIMIT_MIN = 1; @@ -109,8 +111,13 @@ public: * @brief Construct a new AccountTxHandler object * * @param sharedPtrBackend The backend to use + * @param etl The ETL service to use */ - AccountTxHandler(std::shared_ptr const& sharedPtrBackend) : sharedPtrBackend_(sharedPtrBackend) + AccountTxHandler( + std::shared_ptr const& sharedPtrBackend, + std::shared_ptr const& etl + ) + : sharedPtrBackend_(sharedPtrBackend), etl_{etl} { } @@ -130,6 +137,7 @@ public: {JS(ledger_index), validation::CustomValidators::ledgerIndexValidator}, {JS(ledger_index_min), validation::Type{}}, {JS(ledger_index_max), validation::Type{}}, + {JS(ctid), validation::Type{}}, {JS(limit), validation::Type{}, validation::Min(1u), diff --git a/src/rpc/handlers/BookOffers.cpp b/src/rpc/handlers/BookOffers.cpp index f63c98be5..3c7420512 100644 --- a/src/rpc/handlers/BookOffers.cpp +++ b/src/rpc/handlers/BookOffers.cpp @@ -65,7 +65,7 @@ BookOffersHandler::process(Input input, Context const& ctx) const auto const book = std::get(bookMaybe); auto const bookKey = getBookBase(book); - // TODO: Add perfomance metrics if needed in future + // TODO: Add performance metrics if needed in future auto [offers, _] = sharedPtrBackend_->fetchBookOffers(bookKey, lgrInfo.seq, input.limit, ctx.yield); auto output = BookOffersHandler::Output{}; diff --git a/src/rpc/handlers/LedgerEntry.cpp b/src/rpc/handlers/LedgerEntry.cpp index c424135aa..ebd04e99d 100644 --- a/src/rpc/handlers/LedgerEntry.cpp +++ b/src/rpc/handlers/LedgerEntry.cpp @@ -185,6 +185,13 @@ LedgerEntryHandler::process(LedgerEntryHandler::Input input, Context const& ctx) ); auto const seq = input.permissionedDomain->at(JS(seq)).as_int64(); key = ripple::keylet::permissionedDomain(*account, seq).key; + } else if (input.delegate) { + auto const account = + ripple::parseBase58(boost::json::value_to(input.delegate->at(JS(account)))); + auto const authorize = + ripple::parseBase58(boost::json::value_to(input.delegate->at(JS(authorize))) + ); + key = ripple::keylet::delegate(*account, *authorize).key; } else { // Must specify 1 of the following fields to indicate what type if (ctx.apiVersion == 1) @@ -243,7 +250,7 @@ LedgerEntryHandler::composeKeyFromDirectory(boost::json::object const& directory if (directory.contains(JS(dir_root)) && directory.contains(JS(owner))) return Status{RippledError::rpcINVALID_PARAMS, "mayNotSpecifyBothDirRootAndOwner"}; - // at least one should availiable + // at least one should available if (!(directory.contains(JS(dir_root)) || directory.contains(JS(owner)))) return Status{RippledError::rpcINVALID_PARAMS, "missingOwnerOrDirRoot"}; @@ -302,7 +309,7 @@ tag_invoke(boost::json::value_to_tag, boost::json::va if (jsonObject.contains(JS(binary))) input.binary = jv.at(JS(binary)).as_bool(); - // check all the protential index + // check all the potential index static auto const kINDEX_FIELD_TYPE_MAP = std::unordered_map{ {JS(index), ripple::ltANY}, {JS(directory), ripple::ltDIR_NODE}, @@ -319,7 +326,8 @@ tag_invoke(boost::json::value_to_tag, boost::json::va {JS(oracle), ripple::ltORACLE}, {JS(credential), ripple::ltCREDENTIAL}, {JS(mptoken), ripple::ltMPTOKEN}, - {JS(permissioned_domain), ripple::ltPERMISSIONED_DOMAIN} + {JS(permissioned_domain), ripple::ltPERMISSIONED_DOMAIN}, + {JS(delegate), ripple::ltDELEGATE} }; auto const parseBridgeFromJson = [](boost::json::value const& bridgeJson) { @@ -408,6 +416,8 @@ tag_invoke(boost::json::value_to_tag, boost::json::va input.mptoken = jv.at(JS(mptoken)).as_object(); } else if (jsonObject.contains(JS(permissioned_domain))) { input.permissionedDomain = jv.at(JS(permissioned_domain)).as_object(); + } else if (jsonObject.contains(JS(delegate))) { + input.delegate = jv.at(JS(delegate)).as_object(); } if (jsonObject.contains("include_deleted")) diff --git a/src/rpc/handlers/LedgerEntry.hpp b/src/rpc/handlers/LedgerEntry.hpp index 9851ad746..54b5a5adb 100644 --- a/src/rpc/handlers/LedgerEntry.hpp +++ b/src/rpc/handlers/LedgerEntry.hpp @@ -110,6 +110,7 @@ public: std::optional createAccountClaimId; std::optional oracleNode; std::optional credential; + std::optional delegate; bool includeDeleted = false; }; @@ -392,6 +393,23 @@ public: }, }, }}}, + {JS(delegate), + meta::WithCustomError{ + validation::Type{}, Status(ClioError::RpcMalformedRequest) + }, + meta::IfType{kMALFORMED_REQUEST_HEX_STRING_VALIDATOR}, + meta::IfType{meta::Section{ + {JS(account), + meta::WithCustomError{validation::Required{}, Status(ClioError::RpcMalformedRequest)}, + meta::WithCustomError{ + validation::CustomValidators::accountBase58Validator, Status(ClioError::RpcMalformedAddress) + }}, + {JS(authorize), + meta::WithCustomError{validation::Required{}, Status(ClioError::RpcMalformedRequest)}, + meta::WithCustomError{ + validation::CustomValidators::accountBase58Validator, Status(ClioError::RpcMalformedAddress) + }} + }}}, {JS(ledger), check::Deprecated{}}, {"include_deleted", validation::Type{}}, }; diff --git a/src/rpc/handlers/LedgerIndex.hpp b/src/rpc/handlers/LedgerIndex.hpp index 7aa8969b3..c7f9f8fa3 100644 --- a/src/rpc/handlers/LedgerIndex.hpp +++ b/src/rpc/handlers/LedgerIndex.hpp @@ -36,7 +36,7 @@ namespace rpc { /** - * @brief The ledger_index method fetches the lastest closed ledger before the given date. + * @brief The ledger_index method fetches the latest closed ledger before the given date. * */ class LedgerIndexHandler { diff --git a/src/rpc/handlers/ServerInfo.hpp b/src/rpc/handlers/ServerInfo.hpp index 008450721..085066f7c 100644 --- a/src/rpc/handlers/ServerInfo.hpp +++ b/src/rpc/handlers/ServerInfo.hpp @@ -21,6 +21,8 @@ #include "data/BackendInterface.hpp" #include "data/DBHelpers.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/JS.hpp" @@ -49,10 +51,6 @@ #include #include -namespace etl { -class ETLService; -class LoadBalancer; -} // namespace etl namespace rpc { class Counters; } // namespace rpc @@ -62,18 +60,16 @@ namespace rpc { /** * @brief Contains common functionality for handling the `server_info` command * - * @tparam LoadBalancerType The type of the load balancer - * @tparam ETLServiceType The type of the ETL service * @tparam CountersType The type of the counters */ -template +template class BaseServerInfoHandler { static constexpr auto kBACKEND_COUNTERS_KEY = "backend_counters"; std::shared_ptr backend_; std::shared_ptr subscriptions_; - std::shared_ptr balancer_; - std::shared_ptr etl_; + std::shared_ptr balancer_; + std::shared_ptr etl_; std::reference_wrapper counters_; public: @@ -158,8 +154,8 @@ public: BaseServerInfoHandler( std::shared_ptr const& backend, std::shared_ptr const& subscriptions, - std::shared_ptr const& balancer, - std::shared_ptr const& etl, + std::shared_ptr const& balancer, + std::shared_ptr const& etl, CountersType const& counters ) : backend_(backend) @@ -352,6 +348,6 @@ private: * * For more details see: https://xrpl.org/server_info-clio.html */ -using ServerInfoHandler = BaseServerInfoHandler; +using ServerInfoHandler = BaseServerInfoHandler; } // namespace rpc diff --git a/src/rpc/handlers/Subscribe.cpp b/src/rpc/handlers/Subscribe.cpp index 2f140d9fb..6a302aca4 100644 --- a/src/rpc/handlers/Subscribe.cpp +++ b/src/rpc/handlers/Subscribe.cpp @@ -214,7 +214,7 @@ SubscribeHandler::subscribeToBooks( auto const [offers, _] = sharedPtrBackend_->fetchBookOffers(bookBase, rng->maxSequence, kFETCH_LIMIT, yield); - // the taker is not really uesed, same issue with + // the taker is not really used, same issue with // https://github.com/XRPLF/xrpl-dev-portal/issues/1818 auto const takerID = internalBook.taker ? accountFromStringStrict(*(internalBook.taker)) : beast::zero; diff --git a/src/rpc/handlers/Tx.hpp b/src/rpc/handlers/Tx.hpp index b05a64c1e..26da69a14 100644 --- a/src/rpc/handlers/Tx.hpp +++ b/src/rpc/handlers/Tx.hpp @@ -21,7 +21,7 @@ #include "data/BackendInterface.hpp" #include "data/Types.hpp" -#include "etl/ETLService.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/JS.hpp" #include "rpc/RPCHelpers.hpp" @@ -37,7 +37,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -52,14 +54,13 @@ namespace rpc { /** - * @brief Contains common functionality for handling the `tx` command + * @brief The tx method retrieves information on a single transaction, by its identifying hash. * - * @tparam ETLServiceType The type of the ETL service to use + * For more details see: https://xrpl.org/tx.html */ -template -class BaseTxHandler { +class TxHandler { std::shared_ptr sharedPtrBackend_; - std::shared_ptr etl_; + std::shared_ptr etl_; public: /** @@ -95,14 +96,14 @@ public: using Result = HandlerReturnType; /** - * @brief Construct a new BaseTxHandler object + * @brief Construct a new TxHandler object * * @param sharedPtrBackend The backend to use * @param etl The ETL service to use */ - BaseTxHandler( + TxHandler( std::shared_ptr const& sharedPtrBackend, - std::shared_ptr const& etl + std::shared_ptr const& etl ) : sharedPtrBackend_(sharedPtrBackend), etl_(etl) { @@ -183,7 +184,7 @@ public: dbResponse = sharedPtrBackend_->fetchTransaction(ripple::uint256{input.transaction->c_str()}, ctx.yield); } - auto output = BaseTxHandler::Output{.apiVersion = ctx.apiVersion}; + auto output = TxHandler::Output{.apiVersion = ctx.apiVersion}; if (!dbResponse) { if (rangeSupplied && input.transaction) // ranges not for ctid @@ -214,17 +215,15 @@ public: // input.transaction might be not available, get hash via tx object if (txn.contains(JS(hash))) output.hash = txn.at(JS(hash)).as_string(); + } - // append ctid here to mimic rippled 1.12 behavior: return ctid even binary=true - // rippled will change it in the future, ctid should be part of tx json which not available in binary - // mode - auto const txnIdx = boost::json::value_to(meta.at("TransactionIndex")); - if (txnIdx <= 0xFFFFU && dbResponse->ledgerSequence < 0x0FFF'FFFFUL && currentNetId && - *currentNetId <= 0xFFFFU) { - output.ctid = rpc::encodeCTID( - dbResponse->ledgerSequence, static_cast(txnIdx), static_cast(*currentNetId) - ); - } + // append ctid here to mimic rippled behavior + auto const txnIdx = boost::json::value_to(meta.at("TransactionIndex")); + if (txnIdx <= 0xFFFFU && dbResponse->ledgerSequence < 0x0FFF'FFFFUL && currentNetId && + *currentNetId <= 0xFFFFU) { + output.ctid = rpc::encodeCTID( + dbResponse->ledgerSequence, static_cast(txnIdx), static_cast(*currentNetId) + ); } output.date = dbResponse->date; @@ -281,12 +280,10 @@ private: if (output.tx) { obj[JS(tx_json)] = *output.tx; obj[JS(tx_json)].as_object()[JS(date)] = output.date; + if (output.ctid) + obj[JS(tx_json)].as_object()[JS(ctid)] = *output.ctid; + obj[JS(tx_json)].as_object()[JS(ledger_index)] = output.ledgerIndex; - // move ctid from tx_json to root - if (obj[JS(tx_json)].as_object().contains(JS(ctid))) { - obj[JS(ctid)] = obj[JS(tx_json)].as_object()[JS(ctid)]; - obj[JS(tx_json)].as_object().erase(JS(ctid)); - } // move hash from tx_json to root if (obj[JS(tx_json)].as_object().contains(JS(hash))) { obj[JS(hash)] = obj[JS(tx_json)].as_object()[JS(hash)]; @@ -320,7 +317,7 @@ private: friend Input tag_invoke(boost::json::value_to_tag, boost::json::value const& jv) { - auto input = BaseTxHandler::Input{}; + auto input = TxHandler::Input{}; auto const& jsonObject = jv.as_object(); if (jsonObject.contains(JS(transaction))) @@ -344,10 +341,4 @@ private: } }; -/** - * @brief The tx method retrieves information on a single transaction, by its identifying hash. - * - * For more details see: https://xrpl.org/tx.html - */ -using TxHandler = BaseTxHandler; } // namespace rpc diff --git a/src/rpc/handlers/VersionHandler.hpp b/src/rpc/handlers/VersionHandler.hpp index 515377b16..157079a32 100644 --- a/src/rpc/handlers/VersionHandler.hpp +++ b/src/rpc/handlers/VersionHandler.hpp @@ -22,7 +22,7 @@ #include "rpc/common/APIVersion.hpp" #include "rpc/common/Types.hpp" #include "rpc/common/impl/APIVersionParser.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include #include diff --git a/src/util/BlockingCache.hpp b/src/util/BlockingCache.hpp new file mode 100644 index 000000000..0deccdd87 --- /dev/null +++ b/src/util/BlockingCache.hpp @@ -0,0 +1,221 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "util/Assert.hpp" +#include "util/Mutex.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace util { + +/** + * @brief A thread-safe cache that blocks getting operations until the cache is updated + * + * @tparam ValueType The type of value to be cached + * @tparam ErrorType The type of error that can occur during updates + */ +template + requires(not std::same_as) +class BlockingCache { +public: + /** + * @brief Possible states of the cache + */ + enum class State { NoValue, Updating, HasValue }; + +private: + std::atomic state_{State::NoValue}; + util::Mutex, std::shared_mutex> value_; + boost::signals2::signal)> updateFinished_; + +public: + /** + * @brief Default constructor - creates an empty cache + */ + BlockingCache() = default; + + /** + * @brief Construct a cache with an initial value + * @param initialValue The value to initialize the cache with + */ + explicit BlockingCache(ValueType initialValue) : state_{State::HasValue}, value_(std::move(initialValue)) + { + } + + BlockingCache(BlockingCache&&) = delete; + BlockingCache(BlockingCache const&) = delete; + BlockingCache& + operator=(BlockingCache&&) = delete; + BlockingCache& + operator=(BlockingCache const&) = delete; + + /** + * @brief Function type for cache update operations + * @details Called when the cache needs to be populated or refreshed + */ + using Updater = std::function(boost::asio::yield_context)>; + + /** + * @brief Function type to verify if a value should be cached + * @details Returns true if the value should be stored in the cache + */ + using Verifier = std::function; + + /** + * @brief Asynchronously get a value from the cache, updating if necessary + * + * @param yield The asio yield context for coroutine suspension + * @param updater Function to generate a new value if needed + * @param verifier Function to validate whether a value should be cached + * @return std::expected The cached value or an error + * + * Depending on the current cache state, this will either: + * - Return the cached value if it's already present + * - Wait for an ongoing update to complete + * - Trigger a new update if the cache is empty + */ + [[nodiscard]] std::expected + asyncGet(boost::asio::yield_context yield, Updater updater, Verifier verifier) + { + switch (state_) { + case State::Updating: { + return wait(yield, std::move(updater), std::move(verifier)); + } + case State::HasValue: { + auto const value = value_.template lock(); + ASSERT(value->has_value(), "Value should be presented when the cache is full"); + return value->value(); + } + case State::NoValue: { + return update(yield, std::move(updater), std::move(verifier)); + } + }; + std::unreachable(); + } + + /** + * @brief Force an update of the cache value + * + * @param yield The ASIO yield context for coroutine suspension + * @param updater Function to generate a new value + * @param verifier Function to validate whether a value should be cached + * @return std::expected The new value or an error + * + * Initiates a cache update operation regardless of current state. + * If another update is already in progress, waits for it to complete. + */ + [[nodiscard]] std::expected + update(boost::asio::yield_context yield, Updater updater, Verifier verifier) + { + if (state_ == State::Updating) { + return asyncGet(yield, std::move(updater), std::move(verifier)); + } + state_ = State::Updating; + + auto const result = updater(yield); + auto const shouldBeCached = result.has_value() and verifier(result.value()); + + if (shouldBeCached) { + value_.lock().get() = result.value(); + state_ = State::HasValue; + } else { + state_ = State::NoValue; + value_.lock().get() = std::nullopt; + } + + updateFinished_(result); + return result; + } + + /** + * @brief Invalidates the currently cached value if present + * + * Clears the cache and sets its state to Empty. + * Has no effect if the cache is already empty or being updated. + */ + void + invalidate() + { + if (state_ == State::HasValue) { + state_ = State::NoValue; + value_.lock().get() = std::nullopt; + } + } + + /** + * @brief Returns the current state of the cache + * @return Current cache state (Empty, Updating, or Full) + */ + [[nodiscard]] State + state() const + { + return state_; + } + +private: + /** + * @brief Wait for an ongoing update to complete + * + * @param yield The ASIO yield context for coroutine suspension + * @param updater Function to generate a new value if needed + * @param verifier Function to validate whether a value should be cached + * @return std::expected The result of the ongoing update + * + * This method blocks the current coroutine until the ongoing update signals completion. + */ + std::expected + wait(boost::asio::yield_context yield, Updater updater, Verifier verifier) + { + boost::asio::steady_timer timer{yield.get_executor(), boost::asio::steady_timer::duration::max()}; + boost::system::error_code errorCode; + + std::optional> result; + boost::signals2::scoped_connection const slot = + updateFinished_.connect([yield, &timer, &result](std::expected value) { + boost::asio::spawn(yield, [&timer, &result, value = std::move(value)](auto&&) { + result = std::move(value); + timer.cancel(); + }); + }); + + if (state_ == State::Updating) { + timer.async_wait(yield[errorCode]); + ASSERT(result.has_value(), "There should be some value after waiting"); + return std::move(result).value(); + } + return asyncGet(yield, std::move(updater), std::move(verifier)); + } +}; + +} // namespace util diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 51d44765f..8b12f764a 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -4,7 +4,6 @@ target_sources( clio_util PRIVATE Assert.cpp build/Build.cpp - config/Config.cpp CoroutineGroup.cpp log/Logger.cpp prometheus/Http.cpp @@ -24,19 +23,20 @@ target_sources( ResponseExpirationCache.cpp SignalsHandler.cpp StopHelper.cpp + StringHash.cpp Taggable.cpp TerminationHandler.cpp TimeUtils.cpp TxUtils.cpp LedgerUtils.cpp - newconfig/Array.cpp - newconfig/ArrayView.cpp - newconfig/ConfigConstraints.cpp - newconfig/ConfigDefinition.cpp - newconfig/ConfigFileJson.cpp - newconfig/ObjectView.cpp - newconfig/Types.cpp - newconfig/ValueView.cpp + config/Array.cpp + config/ArrayView.cpp + config/ConfigConstraints.cpp + config/ConfigDefinition.cpp + config/ConfigFileJson.cpp + config/ObjectView.cpp + config/Types.cpp + config/ValueView.cpp ) # This must be above the target_link_libraries call otherwise backtrace doesn't work @@ -45,7 +45,14 @@ if ("${san}" STREQUAL "") endif () target_link_libraries( - clio_util PUBLIC Boost::headers fmt::fmt openssl::openssl xrpl::libxrpl Threads::Threads clio_options + clio_util + PUBLIC Boost::headers + fmt::fmt + openssl::openssl + xrpl::libxrpl + Threads::Threads + clio_options + clio_rpc_center ) # FIXME: needed on gcc-12, clang-16 and AppleClang for now (known boost 1.82 issue for some compilers) diff --git a/src/util/LedgerUtils.hpp b/src/util/LedgerUtils.hpp index 2c1916e0d..6e01e3e0e 100644 --- a/src/util/LedgerUtils.hpp +++ b/src/util/LedgerUtils.hpp @@ -80,7 +80,7 @@ public: } // namespace impl /** - * @brief A helper class that provides lists of different ledger type catagory. + * @brief A helper class that provides lists of different ledger type category. * */ class LedgerTypes { diff --git a/src/util/ResponseExpirationCache.cpp b/src/util/ResponseExpirationCache.cpp index 3b121e4f7..d5924f9f5 100644 --- a/src/util/ResponseExpirationCache.cpp +++ b/src/util/ResponseExpirationCache.cpp @@ -21,40 +21,26 @@ #include "util/Assert.hpp" +#include #include #include -#include -#include -#include +#include #include +#include #include namespace util { -void -ResponseExpirationCache::Entry::put(boost::json::object response) +ResponseExpirationCache::ResponseExpirationCache( + std::chrono::steady_clock::duration cacheTimeout, + std::unordered_set const& cmds +) + : cacheTimeout_(cacheTimeout) { - response_ = std::move(response); - lastUpdated_ = std::chrono::steady_clock::now(); -} - -std::optional -ResponseExpirationCache::Entry::get() const -{ - return response_; -} - -std::chrono::steady_clock::time_point -ResponseExpirationCache::Entry::lastUpdated() const -{ - return lastUpdated_; -} - -void -ResponseExpirationCache::Entry::invalidate() -{ - response_.reset(); + for (auto const& command : cmds) { + cache_.emplace(command, std::make_unique()); + } } bool @@ -63,38 +49,41 @@ ResponseExpirationCache::shouldCache(std::string const& cmd) return cache_.contains(cmd); } -std::optional -ResponseExpirationCache::get(std::string const& cmd) const +std::expected +ResponseExpirationCache::getOrUpdate( + boost::asio::yield_context yield, + std::string const& cmd, + Updater updater, + Verifier verifier +) { auto it = cache_.find(cmd); - if (it == cache_.end()) - return std::nullopt; + ASSERT(it != cache_.end(), "Can't get a value which is not in the cache"); - auto const& entry = it->second.lock(); - if (std::chrono::steady_clock::now() - entry->lastUpdated() > cacheTimeout_) - return std::nullopt; + auto& entry = it->second; + { + auto result = entry->asyncGet(yield, updater, verifier); + if (not result.has_value()) { + return std::unexpected{std::move(result).error()}; + } + if (std::chrono::steady_clock::now() - result->lastUpdated < cacheTimeout_) { + return std::move(result)->response; + } + } - return entry->get(); -} - -void -ResponseExpirationCache::put(std::string const& cmd, boost::json::object const& response) -{ - if (not shouldCache(cmd)) - return; - - ASSERT(cache_.contains(cmd), "Command is not in the cache: {}", cmd); - - auto entry = cache_[cmd].lock(); - entry->put(response); + // Force update due to cache timeout + auto result = entry->update(yield, std::move(updater), std::move(verifier)); + if (not result.has_value()) { + return std::unexpected{std::move(result).error()}; + } + return std::move(result)->response; } void ResponseExpirationCache::invalidate() { for (auto& [_, entry] : cache_) { - auto entryLock = entry.lock(); - entryLock->invalidate(); + entry->invalidate(); } } diff --git a/src/util/ResponseExpirationCache.hpp b/src/util/ResponseExpirationCache.hpp index e2cf5f231..61cbf6d8d 100644 --- a/src/util/ResponseExpirationCache.hpp +++ b/src/util/ResponseExpirationCache.hpp @@ -19,13 +19,15 @@ #pragma once -#include "util/Mutex.hpp" +#include "rpc/Errors.hpp" +#include "util/BlockingCache.hpp" +#include +#include #include #include -#include -#include +#include #include #include #include @@ -33,92 +35,88 @@ namespace util { /** - * @brief Cache of requests' responses with TTL support and configurable cachable commands + * @brief Cache of requests' responses with TTL support and configurable cacheable commands + * + * This class implements a time-based expiration cache for RPC responses. It allows + * caching responses for specified commands and automatically invalidates them after + * a configured timeout period. The cache uses BlockingCache internally to handle + * concurrent access and updates. */ class ResponseExpirationCache { +public: /** - * @brief A class to store a cache entry. + * @brief A data structure to store a cache entry with its timestamp */ - class Entry { - std::chrono::steady_clock::time_point lastUpdated_; - std::optional response_; - - public: - /** - * @brief Put a response into the cache - * - * @param response The response to store - */ - void - put(boost::json::object response); - - /** - * @brief Get the response from the cache - * - * @return The response - */ - std::optional - get() const; - - /** - * @brief Get the last time the cache was updated - * - * @return The last time the cache was updated - */ - std::chrono::steady_clock::time_point - lastUpdated() const; - - /** - * @brief Invalidate the cache entry - */ - void - invalidate(); + struct EntryData { + std::chrono::steady_clock::time_point lastUpdated; ///< When the entry was last updated + boost::json::object response; ///< The cached response data }; - std::chrono::steady_clock::duration cacheTimeout_; - std::unordered_map> cache_; + /** + * @brief A data structure to represent errors that can occur during an update of the cache + */ + struct Error { + rpc::Status status; ///< The status code and message of the error + boost::json::array warnings; ///< Any warnings related to the request - bool - shouldCache(std::string const& cmd); + bool + operator==(Error const&) const = default; + }; + + using CacheEntry = util::BlockingCache; + +private: + std::chrono::steady_clock::duration cacheTimeout_; + std::unordered_map> cache_; public: /** - * @brief Construct a new Cache object + * @brief Construct a new ResponseExpirationCache object * - * @param cacheTimeout The time for cache entries to expire - * @param cmds The commands that should be cached + * @param cacheTimeout The time period after which cached entries expire + * @param cmds The commands that should be cached (requests for other commands won't be cached) */ ResponseExpirationCache( std::chrono::steady_clock::duration cacheTimeout, std::unordered_set const& cmds - ) - : cacheTimeout_(cacheTimeout) - { - for (auto const& command : cmds) { - cache_.emplace(command, Entry{}); - } - } + ); /** - * @brief Get a response from the cache + * @brief Check if the given command should be cached * + * @param cmd The command to check + * @return true if the command should be cached, false otherwise + */ + bool + shouldCache(std::string const& cmd); + + using Updater = CacheEntry::Updater; + using Verifier = CacheEntry::Verifier; + + /** + * @brief Get a cached response or update the cache if necessary + * + * This method returns a cached response if it exists and hasn't expired. + * If the cache entry is expired or doesn't exist, it calls the updater to + * generate a new value. If multiple coroutines request the same entry + * simultaneously, only one updater will be called while others wait. + * + * @note cmd must be one of the commands that are cached. There is an ASSERT() inside the function + * + * @param yield Asio yield context for coroutine suspension * @param cmd The command to get the response for - * @return The response if it exists or std::nullopt otherwise + * @param updater Function to generate the response if not in cache or expired + * @param verifier Function to validate if a response should be cached + * @return The cached or newly generated response, or an error */ - [[nodiscard]] std::optional - get(std::string const& cmd) const; - - /** - * @brief Put a response into the cache if the request should be cached - * - * @param cmd The command to store the response for - * @param response The response to store - */ - void - put(std::string const& cmd, boost::json::object const& response); + [[nodiscard]] std::expected + getOrUpdate(boost::asio::yield_context yield, std::string const& cmd, Updater updater, Verifier verifier); /** * @brief Invalidate all entries in the cache + * + * This causes all cached entries to be cleared, forcing the next access + * to generate new responses. */ void invalidate(); diff --git a/src/util/SignalsHandler.cpp b/src/util/SignalsHandler.cpp index 3897ba014..9bd914652 100644 --- a/src/util/SignalsHandler.cpp +++ b/src/util/SignalsHandler.cpp @@ -20,8 +20,8 @@ #include "util/SignalsHandler.hpp" #include "util/Assert.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include #include diff --git a/src/util/SignalsHandler.hpp b/src/util/SignalsHandler.hpp index addf03344..2fb3545da 100644 --- a/src/util/SignalsHandler.hpp +++ b/src/util/SignalsHandler.hpp @@ -20,8 +20,8 @@ #pragma once #include "util/async/context/BasicExecutionContext.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include #include diff --git a/src/util/StrandedPriorityQueue.hpp b/src/util/StrandedPriorityQueue.hpp index 039e39809..cd605e58f 100644 --- a/src/util/StrandedPriorityQueue.hpp +++ b/src/util/StrandedPriorityQueue.hpp @@ -45,7 +45,7 @@ public: /** * @brief Construct a new priority queue on a strand * @param strand The strand to use - * @param limit The limit of items allowed simultaniously in the queue + * @param limit The limit of items allowed simultaneously in the queue */ StrandedPriorityQueue(util::async::AnyStrand&& strand, std::optional limit = std::nullopt) : strand_(std::move(strand)), limit_(limit.value_or(0uz)) diff --git a/src/util/StringHash.cpp b/src/util/StringHash.cpp new file mode 100644 index 000000000..3f26feb2a --- /dev/null +++ b/src/util/StringHash.cpp @@ -0,0 +1,46 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "util/StringHash.hpp" + +#include +#include +#include + +namespace util { + +size_t +StringHash::operator()(char const* str) const +{ + return hash_type{}(str); +} + +size_t +StringHash::operator()(std::string_view str) const +{ + return hash_type{}(str); +} + +size_t +StringHash::operator()(std::string const& str) const +{ + return hash_type{}(str); +} + +} // namespace util diff --git a/src/util/StringHash.hpp b/src/util/StringHash.hpp new file mode 100644 index 000000000..862c8ed86 --- /dev/null +++ b/src/util/StringHash.hpp @@ -0,0 +1,65 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include +#include +#include +#include + +namespace util { + +/** + * @brief A string hash functor that provides transparent hash operations for various string types. + * + * This hash functor can be used with unordered containers to enable heterogeneous lookups + * for different string-like types without unnecessary conversions. It supports C-style strings, + * string views, and standard strings. + */ +struct StringHash { + using hash_type = std::hash; + using is_transparent = void; ///< Enables heterogeneous lookup + + /** + * @brief Computes the hash of a C-style string. + * @param str Null-terminated C-style string to hash + * @return Size_t hash value + */ + std::size_t + operator()(char const* str) const; + + /** + * @brief Computes the hash of a string_view. + * @param str String view to hash + * @return Size_t hash value + */ + std::size_t + operator()(std::string_view str) const; + + /** + * @brief Computes the hash of a standard string. + * @param str String to hash + * @return Size_t hash value + */ + std::size_t + operator()(std::string const& str) const; +}; + +} // namespace util diff --git a/src/util/Taggable.hpp b/src/util/Taggable.hpp index 9092c6a1c..ea088ba9e 100644 --- a/src/util/Taggable.hpp +++ b/src/util/Taggable.hpp @@ -20,7 +20,7 @@ #pragma once #include "util/Assert.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include #include @@ -96,6 +96,19 @@ public: decorator.decorate(os); return os; } + + /** + * @brief Gets the string representation of the tag. + * + * @return The string representation of the tag + */ + std::string + toString() const + { + std::ostringstream oss; + decorate(oss); + return std::move(oss).str(); + } }; /** @@ -146,7 +159,7 @@ public: /** * @brief Specialization for a nop/null decorator. * - * This generates a pass-thru decorate member function which can be optimized away by the compiler. + * This generates a pass-through decorate member function which can be optimized away by the compiler. */ template <> class TagDecorator final : public BaseTagDecorator { diff --git a/src/util/TimeUtils.cpp b/src/util/TimeUtils.cpp index 85ce4d091..e517d57a8 100644 --- a/src/util/TimeUtils.cpp +++ b/src/util/TimeUtils.cpp @@ -19,6 +19,8 @@ #include "util/TimeUtils.hpp" +#include +#include #include #include @@ -38,6 +40,13 @@ systemTpFromUtcStr(std::string const& dateStr, std::string const& format) return std::chrono::system_clock::from_time_t(timegm(&timeStruct)); } +[[nodiscard]] std::string +systemTpToUtcStr(std::chrono::system_clock::time_point const& tp, std::string const& format) +{ + auto const formatWrapped = fmt::format("{{:{}}}", format); + return fmt::format(fmt::runtime(formatWrapped), std::chrono::floor(tp)); +} + [[nodiscard]] std::chrono::system_clock::time_point systemTpFromLedgerCloseTime(ripple::NetClock::time_point closeTime) { diff --git a/src/util/TimeUtils.hpp b/src/util/TimeUtils.hpp index 84ac5fee3..012479ca8 100644 --- a/src/util/TimeUtils.hpp +++ b/src/util/TimeUtils.hpp @@ -23,6 +23,7 @@ #include #include +#include namespace util { @@ -35,6 +36,16 @@ namespace util { [[nodiscard]] std::optional systemTpFromUtcStr(std::string const& dateStr, std::string const& format); +/** + * @brief Converts a system_clock time_point to a formatted UTC string. + * + * @param tp The time_point to convert. Must be a valid std::chrono::system_clock::time_point. + * @param format The format string that specifies the desired output format. + * @return A string representation of the time_point formatted according to the provided format. + */ +[[nodiscard]] std::string +systemTpToUtcStr(std::chrono::system_clock::time_point const& tp, std::string const& format); + /** * @brief Convert a ledger close time which is XRPL network clock to a system_clock::time_point. * @param closeTime The ledger close time to convert. diff --git a/src/util/async/Concepts.hpp b/src/util/async/Concepts.hpp index eb863699b..a49865318 100644 --- a/src/util/async/Concepts.hpp +++ b/src/util/async/Concepts.hpp @@ -170,7 +170,7 @@ template concept SomeStdDuration = requires { // Thank you Ed Catmur for this trick. // See https://stackoverflow.com/questions/74383254/concept-that-models-only-the-stdchrono-duration-types - []( // + []( // std::type_identity> ) {}(std::type_identity>()); }; @@ -180,7 +180,7 @@ concept SomeStdDuration = requires { */ template concept SomeStdOptional = requires { - []( // + []( // std::type_identity> ) {}(std::type_identity>()); }; diff --git a/src/util/async/README.md b/src/util/async/README.md index 24f4bd78c..13c86fddb 100644 --- a/src/util/async/README.md +++ b/src/util/async/README.md @@ -7,6 +7,7 @@ Clio uses threads intensively. Multiple parts of Clio were/are implemented by ru On the other hand, Clio also uses `Boost.Asio` for more complex tasks such as networking, scheduling RPC handlers, and even interacting with the database is done via Asio’s coroutines. There was a need for a simple yet powerful framework that will cover the following in a unified way: + - Exception/error handling and propagation - Ability to return a value of any type as a result of a successful operation - Cancellation (cooperative) of inflight operations @@ -28,80 +29,99 @@ At the core of async framework are the execution contexts. Each execution contex There are multiple execution contexts to choose from, each with their own pros and cons. #### CoroExecutionContext + This context wraps a thread pool and executes blocks of code by means of `boost::asio::spawn` which spawns coroutines. -Deep inside the framework it hides `boost::asio::yield_context` and automatically switches coroutine contexts everytime user’s code is checking `isStopRequested()` on the `StopToken` given to the user-provided lambda. +Deep inside the framework it hides `boost::asio::yield_context` and automatically switches coroutine contexts every time user’s code is checking `isStopRequested()` on the `StopToken` given to the user-provided lambda. The benefit is that both timers and async operations can work concurrently on a `CoroExecutionContext` even if internally the thread pool only has 1 thread. Users of this execution context should take care to split their work in reasonably sized batches to avoid incurring a performance penalty caused by switching coroutine contexts too often. However if the batches are too time consuming it may lead to slower cooperative cancellation. #### PoolExecutionContext + This context wraps a thread pool but executes blocks of code without using coroutines. Note: A downside of this execution context is that if there is only 1 thread in the thread pool, timers can not execute while the thread is busy executing user-provided code. It's up to the user of this execution context to decide how to deal with this and whether it's important for their use case. #### SyncExecutionContext + This is a fully synchronous execution context. It runs the scheduled operations right on the caller thread. By the time `execute([]{ … })` returns the Operation it’s guaranteed to be ready (i.e. value or error can be immediately queried with `.get()`). In order to support scheduled operations and timeout-based cancellation, this context schedules all timers on the SystemExecutionContext instead. #### SystemExecutionContext + This context of 1 thread is always readily available system-wide and can be used for + - fire and forget operations where it makes no sense to create an entirely new context for them - as an external context for scheduling timers (used by SyncExecutionContext automatically) ### Strand + Any execution context provides a convenient `makeStrand` member function which will return a strand object for the execution context. The strand can then be used with the same set of APIs that the execution context provides with the difference being that everything that is executed through a strand is guaranteed to be serially executed within the strand. This is a way to avoid the need for using a mutex or other explic synchronization mechanisms. ### Outcome + An outcome is like a `std::promise` to the operations that execute on the execution context. The framework will hold onto the outcome object internally and the user of the framework will only receive an operation object that is like the `std::future` to the outcome. The framework will set the final value or error through the outcome object so that the user can receive it on the operation side as a `std::expected`. ### Operation + There are several different operation types available. The one used will depend on the signature of the executable lambda passed by the user of this framework. #### Stoppable and non-stoppable operations + Stoppable operations can be cooperatively stopped via a stop token that is passed to the user-provided function/lambda. A stoppable operation is returned to the user if they specify a stop token as the first argument of the function/lambda for execution. Regular, non-stoppable operations, can not be stopped. A non-stoppable operation is returned to the user if they did not request a stop token as the first argument of the function/lambda for execution. #### Scheduled operations + Scheduled operations are wrappers on top of Stoppable and regular Operations and provide the functionality of a timer that needs to run out before the given block of code will finally be executed on the Execution Context. -Scheduled operations can be aborted by calling +Scheduled operations can be aborted by calling + - `cancel` - will only cancel the timer. If the timer already fired this will have no effect - `requestStop` - will stop the operation if it's already running or as soon as the timer runs out -- `abort` - will call `cancel` immediatelly followed by `requestStop` +- `abort` - will call `cancel` immediately followed by `requestStop` ### Error handling + By default, exceptions that happen during the execution of user-provided code are caught and returned in the error channel of `std::expected` as an instance of the `ExecutionError` struct. The user can then extract the error message by calling `what()` or directly accessing the `message` member. ### Returned value + If the user-provided lambda returns anything but `void`, the type and value will propagate through the operation object and can be received by calling `get` which will block until a value or an error is available. The `wait` member function can be used when the user just wants to wait for the value to become available but not necessarily getting at the value just yet. ### Type erasure + On top of the templated execution contexts, outcomes, operations, strands and stop tokens this framework provides the type-erased wrappers with (mostly) the same interface. #### AnyExecutionContext + This provides the same interface as any other execution context in this framework. Note: the original context is taken in by reference. See examples of use below. -#### AnyOperation +#### AnyOperation + Wraps any type of operations including regular, stoppable and scheduled. Since this wrapper does not know which operation type it's wrapping it only provides an `abort` member function that will call the correct underlying functions depending on the real type of the operation. If `abort` is called on a regular (non-stoppable and not scheduled) operation, the call will result in an assertion failure. ## Examples + This section provides some examples. For more examples take a look at `ExecutionContextBenchmarks`, `AsyncExecutionContextTests` and `AnyExecutionContextTests`. ### Regular operation + #### Awaiting and reading values + ```cpp auto res = ctx.execute([]() { return 42; }); EXPECT_EQ(res.get().value(), 42); @@ -111,12 +131,15 @@ auto res = ctx.execute([&value]() { value = 42; }); res.wait(); ASSERT_EQ(value, 42); -``` +``` ### Stoppable operation + #### Requesting stoppage + The stop token can be used via the `isStopRequested()` member function: -```cpp + +```cpp auto res = ctx.execute([](auto stopToken) { while (not stopToken.isStopRequested()) ; @@ -126,9 +149,10 @@ auto res = ctx.execute([](auto stopToken) { res.requestStop(); ``` - -Alternatively, the stop token is implicity convertible to `bool` so you can also use it like so: -```cpp + +Alternatively, the stop token is implicitly convertible to `bool` so you can also use it like so: + +```cpp auto res = ctx.execute([](auto stopRequested) { while (not stopRequested) ; @@ -140,8 +164,10 @@ res.requestStop(); ``` #### Automatic stoppage on timeout + By adding an optional timeout as the last arg to `execute` you can have the framework automatically call `requestStop()`: -```cpp + +```cpp auto res = ctx.execute([](auto stopRequested) { while (not stopRequested) ; @@ -153,7 +179,9 @@ auto res = ctx.execute([](auto stopRequested) { ``` ### Scheduled operation + #### Cancelling an outstanding operation + ```cpp auto res = ctx.scheduleAfter( 10ms, []([[maybe_unused]] auto stopRequested, auto cancelled) { @@ -162,11 +190,12 @@ auto res = ctx.scheduleAfter( } ); -res.cancel(); // or .abort() +res.cancel(); // or .abort() ``` #### Get value after stopping -```cpp + +```cpp auto res = ctx.scheduleAfter(1ms, [](auto stopRequested) { while (not stopRequested) ; @@ -178,6 +207,7 @@ res.requestStop(); ``` #### Handling an exception + ```cpp auto res = ctx.scheduleAfter(1s, []([[maybe_unused]] auto stopRequested, auto cancelled) { @@ -189,12 +219,14 @@ auto res = auto const err = res.get().error(); EXPECT_TRUE(err.message.ends_with("test")); EXPECT_TRUE(std::string{err}.ends_with("test")); -``` +``` + +### Strand -### Strand The APIs are basically the same as with the parent `ExecutionContext`. #### Computing a value on a strand + ```cpp auto strand = ctx.makeStrand(); auto res = strand.execute([] { return 42; }); @@ -202,30 +234,33 @@ auto res = strand.execute([] { return 42; }); EXPECT_EQ(res.get().value(), 42); ``` -### Type erasure +### Type erasure + #### Simple use + ```cpp auto ctx = CoroExecutionContext{4}; auto anyCtx = AnyExecutionContext{ctx}; auto op = anyCtx.execute([](auto stopToken) { while(not stopToken.isStopRequested()) - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(1s); }, 3s); ``` #### Aborting the operation + Erased operations only expose the `abort` member function that can be used to both cancel an outstanding and/or stop a running operation. ```cpp auto op = anyCtx.scheduleAfter(3s, [](auto stopToken, auto cancelled) { if (cancelled) return; - + while(not stopToken.isStopRequested()) - std::this_thread::sleep_for(1s); + std::this_thread::sleep_for(1s); }, 3s); -std::this_thread::sleep_for(2s); +std::this_thread::sleep_for(2s); op.abort(); // cancels the scheduled operation with 1s to spare ``` diff --git a/src/util/newconfig/Array.cpp b/src/util/config/Array.cpp similarity index 94% rename from src/util/newconfig/Array.cpp rename to src/util/config/Array.cpp index 46c2b31eb..4a07a0598 100644 --- a/src/util/newconfig/Array.cpp +++ b/src/util/config/Array.cpp @@ -17,12 +17,12 @@ */ //============================================================================== -#include "util/newconfig/Array.hpp" +#include "util/config/Array.hpp" #include "util/Assert.hpp" -#include "util/newconfig/ConfigValue.hpp" -#include "util/newconfig/Error.hpp" -#include "util/newconfig/Types.hpp" +#include "util/config/ConfigValue.hpp" +#include "util/config/Error.hpp" +#include "util/config/Types.hpp" #include #include diff --git a/src/util/newconfig/Array.hpp b/src/util/config/Array.hpp similarity index 97% rename from src/util/newconfig/Array.hpp rename to src/util/config/Array.hpp index 71e946aab..a1c6edca2 100644 --- a/src/util/newconfig/Array.hpp +++ b/src/util/config/Array.hpp @@ -19,9 +19,9 @@ #pragma once -#include "util/newconfig/ConfigValue.hpp" -#include "util/newconfig/Error.hpp" -#include "util/newconfig/Types.hpp" +#include "util/config/ConfigValue.hpp" +#include "util/config/Error.hpp" +#include "util/config/Types.hpp" #include #include diff --git a/src/util/newconfig/ArrayView.cpp b/src/util/config/ArrayView.cpp similarity index 88% rename from src/util/newconfig/ArrayView.cpp rename to src/util/config/ArrayView.cpp index 173cbdaef..5d0733ae4 100644 --- a/src/util/newconfig/ArrayView.cpp +++ b/src/util/config/ArrayView.cpp @@ -17,14 +17,14 @@ */ //============================================================================== -#include "util/newconfig/ArrayView.hpp" +#include "util/config/ArrayView.hpp" #include "util/Assert.hpp" -#include "util/newconfig/Array.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ConfigValue.hpp" -#include "util/newconfig/ObjectView.hpp" -#include "util/newconfig/ValueView.hpp" +#include "util/config/Array.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ConfigValue.hpp" +#include "util/config/ObjectView.hpp" +#include "util/config/ValueView.hpp" #include #include diff --git a/src/util/newconfig/ArrayView.hpp b/src/util/config/ArrayView.hpp similarity index 97% rename from src/util/newconfig/ArrayView.hpp rename to src/util/config/ArrayView.hpp index 521b4456a..dcdec2b1e 100644 --- a/src/util/newconfig/ArrayView.hpp +++ b/src/util/config/ArrayView.hpp @@ -20,9 +20,9 @@ #pragma once #include "util/Assert.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ObjectView.hpp" -#include "util/newconfig/ValueView.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ObjectView.hpp" +#include "util/config/ValueView.hpp" #include #include diff --git a/src/util/config/Config.cpp b/src/util/config/Config.cpp deleted file mode 100644 index 36e79eeba..000000000 --- a/src/util/config/Config.cpp +++ /dev/null @@ -1,210 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of clio: https://github.com/XRPLF/clio - Copyright (c) 2022, the clio developers. - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include "util/config/Config.hpp" - -#include "util/Assert.hpp" -#include "util/Constants.hpp" -#include "util/config/impl/Helpers.hpp" -#include "util/log/Logger.hpp" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace util { - -// Note: `store_(store)` MUST use `()` instead of `{}` otherwise gcc -// picks `initializer_list` constructor and anything passed becomes an -// array :-D -Config::Config(boost::json::value store) : store_(std::move(store)) -{ -} - -Config::operator bool() const noexcept -{ - return not store_.is_null(); -} - -bool -Config::contains(KeyType key) const -{ - return lookup(key).has_value(); -} - -std::optional -Config::lookup(KeyType key) const -{ - if (store_.is_null()) - return std::nullopt; - - std::reference_wrapper cur = std::cref(store_); - auto hasBrokenPath = false; - auto tokenized = impl::Tokenizer{key}; - std::string subkey{}; - - auto maybeSection = tokenized.next(); - while (maybeSection.has_value()) { - auto section = maybeSection.value(); - subkey += section; - - if (not hasBrokenPath) { - if (not cur.get().is_object()) - throw impl::StoreException("Not an object at '" + subkey + "'"); - if (not cur.get().as_object().contains(section)) { - hasBrokenPath = true; - } else { - cur = std::cref(cur.get().as_object().at(section)); - } - } - - subkey += kSEPARATOR; - maybeSection = tokenized.next(); - } - - if (hasBrokenPath) - return std::nullopt; - return std::make_optional(cur); -} - -std::optional -Config::maybeArray(KeyType key) const -{ - try { - auto maybeArr = lookup(key); - if (maybeArr && maybeArr->is_array()) { - auto& arr = maybeArr->as_array(); - ArrayType out; - out.reserve(arr.size()); - - std::ranges::transform(arr, std::back_inserter(out), [](auto&& element) { - return Config{std::forward(element)}; - }); - return std::make_optional(std::move(out)); - } - } catch (impl::StoreException const&) { // NOLINT(bugprone-empty-catch) - // ignore store error, but rethrow key errors - } - - return std::nullopt; -} - -Config::ArrayType -Config::array(KeyType key) const -{ - if (auto maybeArr = maybeArray(key); maybeArr) - return maybeArr.value(); - throw std::logic_error("No array found at '" + key + "'"); -} - -Config::ArrayType -Config::arrayOr(KeyType key, ArrayType fallback) const -{ - if (auto maybeArr = maybeArray(key); maybeArr) - return maybeArr.value(); - return fallback; -} - -Config::ArrayType -Config::arrayOrThrow(KeyType key, std::string_view err) const -{ - try { - return maybeArray(key).value(); - } catch (std::exception const&) { - throw std::runtime_error(std::string{err}); - } -} - -Config -Config::section(KeyType key) const -{ - auto maybeElement = lookup(key); - if (maybeElement && maybeElement->is_object()) - return Config{std::move(*maybeElement)}; - throw std::logic_error("No section found at '" + key + "'"); -} - -Config -Config::sectionOr(KeyType key, boost::json::object fallback) const -{ - auto maybeElement = lookup(key); - if (maybeElement && maybeElement->is_object()) - return Config{std::move(*maybeElement)}; - return Config{std::move(fallback)}; -} - -Config::ArrayType -Config::array() const -{ - if (not store_.is_array()) - throw std::logic_error("_self_ is not an array"); - - ArrayType out; - auto const& arr = store_.as_array(); - out.reserve(arr.size()); - - std::ranges::transform(arr, std::back_inserter(out), [](auto const& element) { return Config{element}; }); - return out; -} - -std::chrono::milliseconds -Config::toMilliseconds(float value) -{ - ASSERT(value >= 0.0f, "Floating point value of seconds must be non-negative, got: {}", value); - return std::chrono::milliseconds{std::lroundf(value * static_cast(util::kMILLISECONDS_PER_SECOND))}; -} - -Config -ConfigReader::open(std::filesystem::path path) -{ - try { - std::ifstream const in(path, std::ios::in | std::ios::binary); - if (in) { - std::stringstream contents; - contents << in.rdbuf(); - auto opts = boost::json::parse_options{}; - opts.allow_comments = true; - return Config{boost::json::parse(contents.str(), {}, opts)}; - } - } catch (std::exception const& e) { - LOG(util::LogService::error()) << "Could not read configuration file from '" << path.string() - << "': " << e.what(); - } - - return Config{}; -} - -} // namespace util diff --git a/src/util/config/Config.hpp b/src/util/config/Config.hpp deleted file mode 100644 index e7f35ba41..000000000 --- a/src/util/config/Config.hpp +++ /dev/null @@ -1,429 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of clio: https://github.com/XRPLF/clio - Copyright (c) 2022, the clio developers. - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#pragma once - -#include "util/config/impl/Helpers.hpp" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace util { - -/** - * @brief Convenience wrapper to query a JSON configuration file. - * - * Any custom data type can be supported by implementing the right `tag_invoke` - * for `boost::json::value_to`. - */ -class Config final { - boost::json::value store_; - static constexpr char kSEPARATOR = '.'; - -public: - using KeyType = std::string; - using ArrayType = std::vector; - using WriteCursorType = std::pair>, KeyType>; - - /** - * @brief Construct a new Config object. - * @param store boost::json::value that backs this instance - */ - explicit Config(boost::json::value store = {}); - - // - // Querying the store - // - - /** - * @brief Checks whether underlying store is not null. - * - * @return true If the store is null - * @return false If the store is not null - */ - operator bool() const noexcept; - - /** - * @brief Checks whether something exists under given key. - * - * @param key The key to check - * @return true If something exists under key - * @return false If nothing exists under key - * @throws std::logic_error If the key is of invalid format - */ - [[nodiscard]] bool - contains(KeyType key) const; - - // - // Key value access - // - - /** - * @brief Interface for fetching values by key that returns std::optional. - * - * Will attempt to fetch the value under the desired key. If the value - * exists and can be represented by the desired type Result then it will be - * returned wrapped in an optional. If the value exists but the conversion - * to Result is not possible - a runtime_error will be thrown. If the value - * does not exist under the specified key - std::nullopt is returned. - * - * @tparam Result The desired return type - * @param key The key to check - * @return Optional value of desired type - * @throws std::logic_error Thrown if conversion to Result is not possible - * or key is of invalid format - */ - template - [[nodiscard]] std::optional - maybeValue(KeyType key) const - { - auto maybeElement = lookup(key); - if (maybeElement) - return std::make_optional(checkedAs(key, *maybeElement)); - return std::nullopt; - } - - /** - * @brief Interface for fetching values by key. - * - * Will attempt to fetch the value under the desired key. If the value - * exists and can be represented by the desired type Result then it will be - * returned. If the value exists but the conversion - * to Result is not possible OR the value does not exist - a logic_error - * will be thrown. - * - * @tparam Result The desired return type - * @param key The key to check - * @return Value of desired type - * @throws std::logic_error Thrown if conversion to Result is not - * possible, value does not exist under specified key path or the key is of - * invalid format - */ - template - [[nodiscard]] Result - value(KeyType key) const - { - return maybeValue(key).value(); - } - - /** - * @brief Interface for fetching values by key with fallback. - * - * Will attempt to fetch the value under the desired key. If the value - * exists and can be represented by the desired type Result then it will be - * returned. If the value exists but the conversion - * to Result is not possible - a logic_error will be thrown. If the value - * does not exist under the specified key - user specified fallback is - * returned. - * - * @tparam Result The desired return type - * @param key The key to check - * @param fallback The fallback value - * @return Value of desired type - * @throws std::logic_error Thrown if conversion to Result is not possible - * or the key is of invalid format - */ - template - [[nodiscard]] Result - valueOr(KeyType key, Result fallback) const - { - try { - return maybeValue(key).value_or(fallback); - } catch (impl::StoreException const&) { - return fallback; - } - } - - /** - * @brief Interface for fetching values by key with custom error handling. - * - * Will attempt to fetch the value under the desired key. If the value - * exists and can be represented by the desired type Result then it will be - * returned. If the value exists but the conversion - * to Result is not possible OR the value does not exist - a runtime_error - * will be thrown with the user specified message. - * - * @tparam Result The desired return type - * @param key The key to check - * @param err The custom error message - * @return Value of desired type - * @throws std::runtime_error Thrown if conversion to Result is not possible - * or value does not exist under key - */ - template - [[nodiscard]] Result - valueOrThrow(KeyType key, std::string_view err) const - { - try { - return maybeValue(key).value(); - } catch (std::exception const&) { - throw std::runtime_error(std::string{err}); - } - } - - /** - * @brief Interface for fetching an array by key that returns std::optional. - * - * Will attempt to fetch an array under the desired key. If the array - * exists then it will be - * returned wrapped in an optional. If the array does not exist under the - * specified key - std::nullopt is returned. - * - * @param key The key to check - * @return Optional array - * @throws std::logic_error Thrown if the key is of invalid format - */ - [[nodiscard]] std::optional - maybeArray(KeyType key) const; - - /** - * @brief Interface for fetching an array by key. - * - * Will attempt to fetch an array under the desired key. If the array - * exists then it will be - * returned. If the array does not exist under the - * specified key an std::logic_error is thrown. - * - * @param key The key to check - * @return The array - * @throws std::logic_error Thrown if there is no array under the desired - * key or the key is of invalid format - */ - [[nodiscard]] ArrayType - array(KeyType key) const; - - /** - * @brief Interface for fetching an array by key with fallback. - * - * Will attempt to fetch an array under the desired key. If the array - * exists then it will be returned. - * If the array does not exist or another type is stored under the desired - * key - user specified fallback is returned. - * - * @param key The key to check - * @param fallback The fallback array - * @return The array - * @throws std::logic_error Thrown if the key is of invalid format - */ - [[nodiscard]] ArrayType - arrayOr(KeyType key, ArrayType fallback) const; - - /** - * @brief Interface for fetching an array by key with custom error handling. - * - * Will attempt to fetch an array under the desired key. If the array - * exists then it will be returned. - * If the array does not exist or another type is stored under the desired - * key - std::runtime_error is thrown with the user specified error message. - * - * @param key The key to check - * @param err The custom error message - * @return The array - * @throws std::runtime_error Thrown if there is no array under the desired - * key - */ - [[nodiscard]] ArrayType - arrayOrThrow(KeyType key, std::string_view err) const; - - /** - * @brief Interface for fetching a sub section by key. - * - * Will attempt to fetch an entire section under the desired key and return - * it as a Config instance. If the section does not exist or another type is - * stored under the desired key - std::logic_error is thrown. - * - * @param key The key to check - * @return Section represented as a separate instance of Config - * @throws std::logic_error Thrown if there is no section under the - * desired key or the key is of invalid format - */ - [[nodiscard]] Config - section(KeyType key) const; - - /** - * @brief Interface for fetching a sub section by key with a fallback object. - * - * Will attempt to fetch an entire section under the desired key and return - * it as a Config instance. If the section does not exist or another type is - * stored under the desired key - fallback object is used instead. - * - * @param key The key to check - * @param fallback The fallback object - * @return Section represented as a separate instance of Config - */ - [[nodiscard]] Config - sectionOr(KeyType key, boost::json::object fallback) const; - - // - // Direct self-value access - // - - /** - * @brief Interface for reading the value directly referred to by the - * instance. Wraps as std::optional. - * - * See @ref maybeValue(KeyType) const for how this works. - * @return Optional value - */ - template - [[nodiscard]] std::optional - maybeValue() const - { - if (store_.is_null()) - return std::nullopt; - return std::make_optional(checkedAs("_self_", store_)); - } - - /** - * @brief Interface for reading the value directly referred to by the - * instance. - * - * See @ref value(KeyType) const for how this works. - * @return The value - */ - template - [[nodiscard]] Result - value() const - { - return maybeValue().value(); - } - - /** - * @brief Interface for reading the value directly referred to by the - * instance with user-specified fallback. - * - * See @ref valueOr(KeyType, Result) const for how this works. - * @param fallback The fallback value - * @return The value - */ - template - [[nodiscard]] Result - valueOr(Result fallback) const - { - return maybeValue().valueOr(fallback); - } - - /** - * @brief Interface for reading the value directly referred to by the - * instance with user-specified error message. - * - * See @ref valueOrThrow(KeyType, std::string_view) const for how this - * works. - * @param err The custom error message - * @return The value - */ - template - [[nodiscard]] Result - valueOrThrow(std::string_view err) const - { - try { - return maybeValue().value(); - } catch (std::exception const&) { - throw std::runtime_error(std::string{err}); - } - } - - /** - * @brief Interface for reading the array directly referred to by the - * instance. - * - * See @ref array(KeyType) const for how this works. - * @return The array - */ - [[nodiscard]] ArrayType - array() const; - - /** - * @brief Method to convert a float seconds value to milliseconds. - * - * @param value The value to convert - * @return The value in milliseconds - */ - static std::chrono::milliseconds - toMilliseconds(float value); - -private: - template - [[nodiscard]] Return - checkedAs(KeyType key, boost::json::value const& value) const - { - using boost::json::value_to; - - auto errorIf = [&key, &value](bool condition) { - if (condition) { - throw std::runtime_error( - "Type for key '" + key + "' is '" + std::string{to_string(value.kind())} + - "' in JSON but requested '" + impl::typeName() + "'" - ); - } - }; - - if constexpr (std::is_same_v) { - errorIf(not value.is_bool()); - } else if constexpr (std::is_same_v) { - errorIf(not value.is_string()); - } else if constexpr (std::is_same_v or std::is_same_v) { - errorIf(not value.is_number()); - } else if constexpr (std::is_convertible_v || std::is_convertible_v) { - errorIf(not value.is_int64() && not value.is_uint64()); - } - - return value_to(value); - } - - std::optional - lookup(KeyType key) const; - - WriteCursorType - lookupForWrite(KeyType key); -}; - -/** - * @brief Simple configuration file reader. - * - * Reads the JSON file under specified path and creates a @ref Config object - * from its contents. - */ -class ConfigReader final { -public: - /** - * @brief Read in a configuration file - * - * @param path The path to the configuration file - * @return The configuration object - */ - static Config - open(std::filesystem::path path); -}; - -} // namespace util diff --git a/src/util/newconfig/ConfigConstraints.cpp b/src/util/config/ConfigConstraints.cpp similarity index 84% rename from src/util/newconfig/ConfigConstraints.cpp rename to src/util/config/ConfigConstraints.cpp index 2a0f8fb6e..3bf412423 100644 --- a/src/util/newconfig/ConfigConstraints.cpp +++ b/src/util/config/ConfigConstraints.cpp @@ -17,10 +17,11 @@ */ //============================================================================== -#include "util/newconfig/ConfigConstraints.hpp" +#include "util/config/ConfigConstraints.hpp" -#include "util/newconfig/Error.hpp" -#include "util/newconfig/Types.hpp" +#include "rpc/RPCCenter.hpp" +#include "util/config/Error.hpp" +#include "util/config/Types.hpp" #include @@ -103,4 +104,22 @@ PositiveDouble::checkValueImpl(Value const& num) const return Error{"Double number must be greater than or equal to 0"}; } +std::optional +RpcNameConstraint::checkTypeImpl(Value const& value) const +{ + if (not std::holds_alternative(value)) + return Error{"RPC command name must be a string"}; + return std::nullopt; +} + +std::optional +RpcNameConstraint::checkValueImpl(Value const& value) const +{ + auto const str = std::get(value); + if (not rpc::RPCCenter::isRpcName(str)) + return Error{"Invalid RPC command name"}; + + return std::nullopt; +} + } // namespace util::config diff --git a/src/util/newconfig/ConfigConstraints.hpp b/src/util/config/ConfigConstraints.hpp similarity index 86% rename from src/util/newconfig/ConfigConstraints.hpp rename to src/util/config/ConfigConstraints.hpp index 910dff1e5..995d6ebff 100644 --- a/src/util/newconfig/ConfigConstraints.hpp +++ b/src/util/config/ConfigConstraints.hpp @@ -20,9 +20,9 @@ #pragma once #include "rpc/common/APIVersion.hpp" +#include "util/config/Error.hpp" +#include "util/config/Types.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/Error.hpp" -#include "util/newconfig/Types.hpp" #include #include @@ -208,7 +208,7 @@ private: void print(std::ostream& stream) const override { - stream << fmt::format("The minimum value is `{}`. The maximum value is `{}", kPORT_MIN, kPORT_MAX); + stream << fmt::format("The minimum value is `{}`. The maximum value is `{}`.", kPORT_MIN, kPORT_MAX); } static constexpr uint32_t kPORT_MIN = 1; @@ -249,7 +249,7 @@ private: void print(std::ostream& stream) const override { - stream << "The value must be a valid IP address"; + stream << "The value must be a valid IP address."; } }; @@ -313,7 +313,13 @@ private: void print(std::ostream& stream) const override { - stream << fmt::format("The value must be one of the following: `{}`", fmt::join(arr_, ", ")); + std::string valuesStream; + std::ranges::for_each(arr_, [&valuesStream](std::string const& elem) { + valuesStream += fmt::format(" `{}`,", elem); + }); + // replace the last "," with "." + valuesStream.back() = '.'; + stream << fmt::format("The value must be one of the following:{}", valuesStream); } std::string_view key_; @@ -376,7 +382,7 @@ private: void print(std::ostream& stream) const override { - stream << fmt::format("The minimum value is `{}`. The maximum value is `{}`", min_, max_); + stream << fmt::format("The minimum value is `{}`. The maximum value is `{}`.", min_, max_); } NumType min_; @@ -417,7 +423,42 @@ private: void print(std::ostream& stream) const override { - stream << fmt::format("The value must be a positive double number"); + stream << "The value must be a positive double number."; + } +}; + +/** + * @brief A constraint to ensure the value is a valid RPC command name. + */ +class RpcNameConstraint final : public Constraint { +private: + /** + * @brief Check if the type of the value is correct for this specific constraint. + * + * @param value The type to be checked + * @return An Error object if the constraint is not met, nullopt otherwise + */ + [[nodiscard]] std::optional + checkTypeImpl(Value const& value) const override; + + /** + * @brief Check if the value is a valid RPC command name. + * + * @param value The value to check + * @return An Error object if the constraint is not met, nullopt otherwise + */ + [[nodiscard]] std::optional + checkValueImpl(Value const& value) const override; + + /** + * @brief Prints to the output stream for this specific constraint. + * + * @param stream The output stream + */ + void + print(std::ostream& stream) const override + { + stream << "Checks whether provided RPC name is valid"; } }; @@ -433,21 +474,17 @@ static constinit OneOf gValidateProcessingPolicy{"server.processing_policy", kPR static constinit PositiveDouble gValidatePositiveDouble{}; -static constinit NumberValueConstraint gValidateNumMarkers{1, 256}; -static constinit NumberValueConstraint gValidateIOThreads{1, std::numeric_limits::max()}; +static constinit NumberValueConstraint gValidateNumMarkers{1, 256}; +static constinit NumberValueConstraint gValidateNumCursors{0, std::numeric_limits::max()}; -static constinit NumberValueConstraint gValidateUint16{ - std::numeric_limits::min(), - std::numeric_limits::max() -}; +// replication factor can be 0 +static constinit NumberValueConstraint gValidateReplicationFactor{0, std::numeric_limits::max()}; -// log file size minimum is 1mb, log rotation time minimum is 1hr -static constinit NumberValueConstraint gValidateLogSize{1, std::numeric_limits::max()}; -static constinit NumberValueConstraint gValidateLogRotationTime{1, std::numeric_limits::max()}; -static constinit NumberValueConstraint gValidateUint32{ - std::numeric_limits::min(), - std::numeric_limits::max() -}; +static constinit NumberValueConstraint gValidateUint16{1, std::numeric_limits::max()}; + +static constinit NumberValueConstraint gValidateUint32{1, std::numeric_limits::max()}; +static constinit NumberValueConstraint gValidateNonNegativeUint32{0, std::numeric_limits::max()}; static constinit NumberValueConstraint gValidateApiVersion{rpc::kAPI_VERSION_MIN, rpc::kAPI_VERSION_MAX}; +static constinit RpcNameConstraint gRpcNameConstraint{}; } // namespace util::config diff --git a/src/util/newconfig/ConfigDefinition.cpp b/src/util/config/ConfigDefinition.cpp similarity index 95% rename from src/util/newconfig/ConfigDefinition.cpp rename to src/util/config/ConfigDefinition.cpp index a041d9412..76bd62bc6 100644 --- a/src/util/newconfig/ConfigDefinition.cpp +++ b/src/util/config/ConfigDefinition.cpp @@ -17,20 +17,20 @@ */ //============================================================================== -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/Assert.hpp" #include "util/Constants.hpp" #include "util/OverloadSet.hpp" -#include "util/newconfig/Array.hpp" -#include "util/newconfig/ArrayView.hpp" -#include "util/newconfig/ConfigConstraints.hpp" -#include "util/newconfig/ConfigFileInterface.hpp" -#include "util/newconfig/ConfigValue.hpp" -#include "util/newconfig/Error.hpp" -#include "util/newconfig/ObjectView.hpp" -#include "util/newconfig/Types.hpp" -#include "util/newconfig/ValueView.hpp" +#include "util/config/Array.hpp" +#include "util/config/ArrayView.hpp" +#include "util/config/ConfigConstraints.hpp" +#include "util/config/ConfigFileInterface.hpp" +#include "util/config/ConfigValue.hpp" +#include "util/config/Error.hpp" +#include "util/config/ObjectView.hpp" +#include "util/config/Types.hpp" +#include "util/config/ValueView.hpp" #include diff --git a/src/util/newconfig/ConfigDefinition.hpp b/src/util/config/ConfigDefinition.hpp similarity index 90% rename from src/util/newconfig/ConfigDefinition.hpp rename to src/util/config/ConfigDefinition.hpp index 030b8d160..558ccb7cb 100644 --- a/src/util/newconfig/ConfigDefinition.hpp +++ b/src/util/config/ConfigDefinition.hpp @@ -21,14 +21,14 @@ #include "rpc/common/APIVersion.hpp" #include "util/Assert.hpp" -#include "util/newconfig/Array.hpp" -#include "util/newconfig/ConfigConstraints.hpp" -#include "util/newconfig/ConfigFileInterface.hpp" -#include "util/newconfig/ConfigValue.hpp" -#include "util/newconfig/Error.hpp" -#include "util/newconfig/ObjectView.hpp" -#include "util/newconfig/Types.hpp" -#include "util/newconfig/ValueView.hpp" +#include "util/config/Array.hpp" +#include "util/config/ConfigConstraints.hpp" +#include "util/config/ConfigFileInterface.hpp" +#include "util/config/ConfigValue.hpp" +#include "util/config/Error.hpp" +#include "util/config/ObjectView.hpp" +#include "util/config/Types.hpp" +#include "util/config/ValueView.hpp" #include #include @@ -266,7 +266,7 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"database.cassandra.port", ConfigValue{ConfigType::Integer}.withConstraint(gValidatePort).optional()}, {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue("clio")}, {"database.cassandra.replication_factor", - ConfigValue{ConfigType::Integer}.defaultValue(3u).withConstraint(gValidateUint16)}, + ConfigValue{ConfigType::Integer}.defaultValue(3u).withConstraint(gValidateReplicationFactor)}, {"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()}, {"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000).withConstraint(gValidateUint32)}, @@ -293,7 +293,7 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()}, {"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, - + {"__ng_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, {"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidateIp)}}, {"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}}, {"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}}, @@ -314,6 +314,15 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(gValidateUint32)}, {"dos_guard.sweep_interval", ConfigValue{ConfigType::Double}.defaultValue(1.0).withConstraint(gValidatePositiveDouble)}, + {"dos_guard.__ng_default_weight", + ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(gValidateNonNegativeUint32)}, + {"dos_guard.__ng_weights.[].method", Array{ConfigValue{ConfigType::String}.withConstraint(gRpcNameConstraint)}}, + {"dos_guard.__ng_weights.[].weight", + Array{ConfigValue{ConfigType::Integer}.withConstraint(gValidateNonNegativeUint32)}}, + {"dos_guard.__ng_weights.[].weight_ledger_current", + Array{ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateNonNegativeUint32)}}, + {"dos_guard.__ng_weights.[].weight_ledger_validated", + Array{ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateNonNegativeUint32)}}, {"workers", ConfigValue{ConfigType::Integer} @@ -321,7 +330,7 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ .withConstraint(gValidateUint32)}, {"server.ip", ConfigValue{ConfigType::String}.withConstraint(gValidateIp)}, {"server.port", ConfigValue{ConfigType::Integer}.withConstraint(gValidatePort)}, - {"server.max_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(gValidateUint32)}, + {"server.max_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(gValidateUint32)}, {"server.local_admin", ConfigValue{ConfigType::Boolean}.optional()}, {"server.admin_password", ConfigValue{ConfigType::String}.optional()}, {"server.processing_policy", @@ -334,7 +343,7 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"prometheus.enabled", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, {"prometheus.compress_reply", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, - {"io_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(gValidateIOThreads)}, + {"io_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(gValidateUint16)}, {"subscription_workers", ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(gValidateUint32)}, @@ -342,9 +351,10 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"cache.num_diffs", ConfigValue{ConfigType::Integer}.defaultValue(32).withConstraint(gValidateUint16)}, {"cache.num_markers", ConfigValue{ConfigType::Integer}.defaultValue(48).withConstraint(gValidateUint16)}, - {"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(gValidateUint16)}, - {"cache.num_cursors_from_account", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(gValidateUint16) - }, + {"cache.num_cursors_from_diff", + ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(gValidateNumCursors)}, + {"cache.num_cursors_from_account", + ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(gValidateNumCursors)}, {"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512).withConstraint(gValidateUint16)}, {"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(gValidateLoadMode)}, @@ -364,13 +374,12 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"log_directory", ConfigValue{ConfigType::String}.optional()}, - {"log_rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048).withConstraint(gValidateLogSize)}, + {"log_rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048).withConstraint(gValidateUint32)}, - {"log_directory_max_size", - ConfigValue{ConfigType::Integer}.defaultValue(50 * 1024).withConstraint(gValidateLogSize)}, + {"log_directory_max_size", ConfigValue{ConfigType::Integer}.defaultValue(50 * 1024).withConstraint(gValidateUint32) + }, - {"log_rotation_hour_interval", - ConfigValue{ConfigType::Integer}.defaultValue(12).withConstraint(gValidateLogRotationTime)}, + {"log_rotation_hour_interval", ConfigValue{ConfigType::Integer}.defaultValue(12).withConstraint(gValidateUint32)}, {"log_tag_style", ConfigValue{ConfigType::String}.defaultValue("none").withConstraint(gValidateLogTag)}, @@ -378,8 +387,6 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, - {"txn_threshold", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(gValidateUint16)}, - {"start_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateUint32)}, {"finish_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateUint32)}, @@ -398,7 +405,6 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"migration.full_scan_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(gValidateUint32)}, {"migration.full_scan_jobs", ConfigValue{ConfigType::Integer}.defaultValue(4).withConstraint(gValidateUint32)}, {"migration.cursors_per_job", ConfigValue{ConfigType::Integer}.defaultValue(100).withConstraint(gValidateUint32)}}, - }; } // namespace util::config diff --git a/src/util/config/ConfigDescription.hpp b/src/util/config/ConfigDescription.hpp new file mode 100644 index 000000000..cee1167e5 --- /dev/null +++ b/src/util/config/ConfigDescription.hpp @@ -0,0 +1,299 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "util/Assert.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/Error.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace util::config { + +/** + * @brief All the config description are stored and extracted from this class + * + * Represents all the possible config description + */ +struct ClioConfigDescription { +public: + /** @brief Struct to represent a key-value pair*/ + struct KV { + std::string_view key; + std::string_view value; + }; + + /** + * @brief Constructs a new Clio Config Description based on pre-existing descriptions + * + * Config Keys and it's corresponding descriptions are all predefined. Used to generate markdown file + */ + constexpr ClioConfigDescription() = default; + + /** + * @brief Retrieves the description for a given key + * + * @param key The key to look up the description for + * @return The description associated with the key, or "Not Found" if the key does not exist + */ + [[nodiscard]] static constexpr std::string_view + get(std::string_view key) + { + auto const itr = std::ranges::find_if(kCONFIG_DESCRIPTION, [&](auto const& v) { return v.key == key; }); + ASSERT(itr != kCONFIG_DESCRIPTION.end(), "Key {} doesn't exist in config", key); + return itr->value; + } + + /** + * @brief Generate markdown file of all the clio config descriptions + * + * @param path The path location to generate the Config-description file + * @return An Error if generating markdown fails, otherwise nothing + */ + [[nodiscard]] static std::expected + generateConfigDescriptionToFile(std::filesystem::path path) + { + namespace fs = std::filesystem; + + // Validate the directory exists + auto const dir = path.parent_path(); + if (!dir.empty() && !fs::exists(dir)) { + return std::unexpected{ + fmt::format("Error: Directory '{}' does not exist or provided path is invalid", dir.string()) + }; + } + + std::ofstream file(path.string()); + if (!file.is_open()) { + return std::unexpected{fmt::format("Failed to create file '{}': {}", path.string(), std::strerror(errno))}; + } + + writeConfigDescriptionToFile(file); + file.close(); + + std::cout << "Markdown file generated successfully: " << path << "\n"; + return {}; + } + + /** + * @brief Writes to Config description to file + * + * @param file The config file to write to + */ + static void + writeConfigDescriptionToFile(std::ostream& file) + { + file << "# Clio Config Description\n\n"; + file << "This document provides a list of all available Clio configuration properties in detail.\n\n"; + file << "> [!NOTE]\n"; + file << "> Dot notation in configuration key names represents nested fields. For example, " + "**database.scylladb** refers to the _scylladb_ field inside the _database_ object. If a key name " + "includes \"[]\", it indicates that the nested field is an array (e.g., etl_sources.[]).\n\n"; + file << "## Configuration Details\n"; + + for (auto const& [key, val] : kCONFIG_DESCRIPTION) { + file << "\n### " << key << "\n\n"; + + // Every type of value is directed to operator<< in ConfigValue.hpp + // as ConfigValue is the one that holds all the info regarding the config values + if (key.contains("[]")) { + file << gClioConfig.asArray(key); + } else { + file << gClioConfig.getValueView(key); + } + file << "- **Description**: " << val << "\n"; + } + } + +private: + static constexpr auto kCONFIG_DESCRIPTION = std::array{ + KV{.key = "database.type", + .value = + "Specifies the type of database used for storing and retrieving data required by the Clio server. Both " + "ScyllaDB and Cassandra can serve as backends for Clio; however, this value must be set to `cassandra`." + }, + KV{.key = "database.cassandra.contact_points", + .value = "A list of IP addresses or hostnames for the initial cluster nodes (Cassandra or ScyllaDB) that " + "the client connects to when establishing a database connection. If you're running Clio locally, " + "set this value to `localhost` or `127.0.0.1`."}, + KV{.key = "database.cassandra.secure_connect_bundle", + .value = "The configuration file that contains the necessary credentials and connection details for " + "securely connecting to a Cassandra database cluster."}, + KV{.key = "database.cassandra.port", .value = "The port number used to connect to the Cassandra database."}, + KV{.key = "database.cassandra.keyspace", + .value = "The Cassandra keyspace to use for the database. If you don't provide a value, this is set to " + "`clio` by default."}, + KV{.key = "database.cassandra.replication_factor", + .value = "Represents the number of replicated nodes for ScyllaDB. For more details see [Fault Tolerance " + "Replication " + "Factor](https://university.scylladb.com/courses/scylla-essentials-overview/lessons/" + "high-availability/topic/fault-tolerance-replication-factor/)."}, + KV{.key = "database.cassandra.table_prefix", + .value = "An optional field to specify a prefix for the Cassandra database table names."}, + KV{.key = "database.cassandra.max_write_requests_outstanding", + .value = "Represents the maximum number of outstanding write requests. Write requests are API calls that " + "write to the database."}, + KV{.key = "database.cassandra.max_read_requests_outstanding", + .value = + "Maximum number of outstanding read requests. Read requests are API calls that read from the database."}, + KV{.key = "database.cassandra.threads", + .value = "Represents the number of threads that will be used for database operations."}, + KV{.key = "database.cassandra.core_connections_per_host", + .value = "The number of core connections per host for the Cassandra database."}, + KV{.key = "database.cassandra.queue_size_io", + .value = "Defines the queue size of the input/output (I/O) operations in Cassandra."}, + KV{.key = "database.cassandra.write_batch_size", + .value = "Represents the batch size for write operations in Cassandra."}, + KV{.key = "database.cassandra.connect_timeout", + .value = "The maximum amount of time in seconds that the system waits for a database connection to be " + "established."}, + KV{.key = "database.cassandra.request_timeout", + .value = "The maximum amount of time in seconds that the system waits for a request to be fetched from the " + "database."}, + KV{.key = "database.cassandra.username", .value = "The username used for authenticating with the database."}, + KV{.key = "database.cassandra.password", .value = "The password used for authenticating with the database."}, + KV{.key = "database.cassandra.certfile", + .value = "The path to the SSL/TLS certificate file used to establish a secure connection between the client " + "and the Cassandra database."}, + KV{.key = "allow_no_etl", .value = "If set to `True`, allows Clio to start without any ETL source."}, + KV{.key = "etl_sources.[].ip", .value = "The IP address of the ETL source."}, + KV{.key = "etl_sources.[].ws_port", .value = "The WebSocket port of the ETL source."}, + KV{.key = "etl_sources.[].grpc_port", .value = "The gRPC port of the ETL source."}, + KV{.key = "forwarding.cache_timeout", + .value = "Specifies the timeout duration (in seconds) for the forwarding cache used in `rippled` " + "communication. A value of `0` means disabling this feature."}, + KV{.key = "forwarding.request_timeout", + .value = + "Specifies the timeout duration (in seconds) for the forwarding request used in `rippled` communication." + }, + KV{.key = "rpc.cache_timeout", + .value = "Specifies the timeout duration (in seconds) for RPC cache response to timeout. A value of `0` " + "means disabling this feature."}, + KV{.key = "num_markers", .value = "Specifies the number of coroutines used to download the initial ledger."}, + KV{.key = "dos_guard.whitelist.[]", .value = "The list of IP addresses to whitelist for DOS protection."}, + KV{.key = "dos_guard.max_fetches", .value = "The maximum number of fetch operations allowed by DOS guard."}, + KV{.key = "dos_guard.max_connections", + .value = "The maximum number of concurrent connections for a specific IP address."}, + KV{.key = "dos_guard.max_requests", .value = "The maximum number of requests allowed for a specific IP address." + }, + KV{.key = "dos_guard.sweep_interval", .value = "Interval in seconds for DOS guard to sweep(clear) its state."}, + KV{.key = "workers", .value = "The number of threads used to process RPC requests."}, + KV{.key = "server.ip", .value = "The IP address of the Clio HTTP server."}, + KV{.key = "server.port", .value = "The port number of the Clio HTTP server."}, + KV{.key = "server.max_queue_size", + .value = + "The maximum size of the server's request queue. If set to `0`, this means there is no queue size limit." + }, + KV{.key = "server.local_admin", + .value = "Indicates if requests from `localhost` are allowed to call Clio admin-only APIs. Note that this " + "setting cannot be enabled " + "together with [server.admin_password](#serveradmin_password)."}, + KV{.key = "server.admin_password", + .value = "The password for Clio admin-only APIs. Note that this setting cannot be enabled together with " + "[server.local_admin](#serveradmin_password)."}, + KV{.key = "server.processing_policy", + .value = "For the `sequent` policy, requests from a single client connection are processed one by one, with " + "the next request read only after the previous one is processed. For the `parallel` policy, Clio " + "will accept all requests and process them in parallel, sending a reply for each request as soon " + "as it is ready."}, + KV{.key = "server.parallel_requests_limit", + .value = "This is an optional parameter, used only if the `processing_strategy` is `parallel`. It limits " + "the number of requests processed in parallel for a single client connection. If not specified, no " + "limit is enforced."}, + KV{.key = "server.ws_max_sending_queue_size", + .value = "Maximum queue size for sending subscription data to clients. This queue buffers data when a " + "client is slow to receive it, ensuring delivery once the client is ready."}, + KV{.key = "prometheus.enabled", .value = "Enables or disables Prometheus metrics."}, + KV{.key = "prometheus.compress_reply", .value = "Enables or disables compression of Prometheus responses."}, + KV{.key = "io_threads", .value = "The number of input/output (I/O) threads. The value cannot be less than `1`." + }, + KV{.key = "subscription_workers", + .value = "The number of worker threads or processes that are responsible for managing and processing " + "subscription-based tasks from `rippled`."}, + KV{.key = "graceful_period", + .value = "The number of milliseconds the server waits to shutdown gracefully. If Clio does not shutdown " + "gracefully after the specified value, it will be killed instead."}, + KV{.key = "cache.num_diffs", + .value = "The number of cursors generated is the number of changed (without counting deleted) objects in " + "the latest `cache.num_diffs` number of ledgers. Cursors are workers that load the ledger cache " + "from the position of markers concurrently. For more information, please read " + "[README.md](../src/etl/README.md)."}, + KV{.key = "cache.num_markers", + .value = "Specifies how many markers are placed randomly within the cache. These markers define the " + "positions on the ledger that will be loaded concurrently by the workers. The higher the number, " + "the more places within the cache we potentially cover."}, + KV{.key = "cache.num_cursors_from_diff", + .value = "`cache.num_cursors_from_diff` number of cursors are generated by looking at the number of changed " + "objects in the most recent ledger. If number of changed objects in current ledger is not enough, " + "it will keep reading previous ledgers until it hit `cache.num_cursors_from_diff`. If set to `0`, " + "the system defaults to generating cursors based on `cache.num_diffs`."}, + KV{.key = "cache.num_cursors_from_account", + .value = "`cache.num_cursors_from_diff` of cursors are generated by reading accounts in `account_tx` table. " + "If set to `0`, the system defaults to generating cursors based on `cache.num_diffs`."}, + KV{.key = "cache.page_fetch_size", .value = "The number of ledger objects to fetch concurrently per marker."}, + KV{.key = "cache.load", .value = "The strategy used for Cache loading."}, + KV{.key = "log_channels.[].channel", .value = "The name of the log channel."}, + KV{.key = "log_channels.[].log_level", .value = "The log level for the specific log channel."}, + KV{.key = "log_level", + .value = "The general logging level of Clio. This level is applied to all log channels that do not have an " + "explicitly defined logging level."}, + KV{.key = "log_format", + .value = "The format string for log messages. The format is described here: " + "."}, + KV{.key = "log_to_console", .value = "Enables or disables logging to the console."}, + KV{.key = "log_directory", .value = "The directory path for the log files."}, + KV{.key = "log_rotation_size", + .value = "The log rotation size in megabytes. When the log file reaches this particular size, a new log " + "file starts."}, + KV{.key = "log_directory_max_size", .value = "The maximum size of the log directory in megabytes."}, + KV{.key = "log_rotation_hour_interval", + .value = "Represents the interval (in hours) for log rotation. If the current log file reaches this value " + "in logging, a new log file starts."}, + KV{.key = "log_tag_style", + .value = + "Log tags are unique identifiers for log messages. `uint`/`int` starts logging from 0 and increments, " + "making it faster. In contrast, `uuid` generates a random unique identifier, which adds overhead."}, + KV{.key = "extractor_threads", .value = "Number of threads used to extract data from ETL source."}, + KV{.key = "read_only", .value = "Indicates if the server is allowed to write data to the database."}, + KV{.key = "start_sequence", + .value = "If specified, the ledger index Clio will start writing to the database from."}, + KV{.key = "finish_sequence", .value = "If specified, the final ledger that Clio will write to the database."}, + KV{.key = "ssl_cert_file", .value = "The path to the SSL certificate file."}, + KV{.key = "ssl_key_file", .value = "The path to the SSL key file."}, + KV{.key = "api_version.default", .value = "The default API version that the Clio server will run on."}, + KV{.key = "api_version.min", .value = "The minimum API version allowed to use."}, + KV{.key = "api_version.max", .value = "The maximum API version allowed to use."}, + KV{.key = "migration.full_scan_threads", .value = "The number of threads used to scan the table."}, + KV{.key = "migration.full_scan_jobs", .value = "The number of coroutines used to scan the table."}, + KV{.key = "migration.cursors_per_job", .value = "The number of cursors each job will scan."} + }; +}; + +} // namespace util::config diff --git a/src/util/newconfig/ConfigFileInterface.hpp b/src/util/config/ConfigFileInterface.hpp similarity index 96% rename from src/util/newconfig/ConfigFileInterface.hpp rename to src/util/config/ConfigFileInterface.hpp index 4749fe4d9..dfdd0faa6 100644 --- a/src/util/newconfig/ConfigFileInterface.hpp +++ b/src/util/config/ConfigFileInterface.hpp @@ -19,7 +19,7 @@ #pragma once -#include "util/newconfig/Types.hpp" +#include "util/config/Types.hpp" #include #include @@ -41,7 +41,7 @@ public: * @brief Retrieves the value of configValue. * * @param key The key of configuration. - * @return the value assosiated with key. + * @return the value associated with key. */ virtual Value getValue(std::string_view key) const = 0; diff --git a/src/util/newconfig/ConfigFileJson.cpp b/src/util/config/ConfigFileJson.cpp similarity index 98% rename from src/util/newconfig/ConfigFileJson.cpp rename to src/util/config/ConfigFileJson.cpp index a9d50fe52..349775317 100644 --- a/src/util/newconfig/ConfigFileJson.cpp +++ b/src/util/config/ConfigFileJson.cpp @@ -17,12 +17,12 @@ */ //============================================================================== -#include "util/newconfig/ConfigFileJson.hpp" +#include "util/config/ConfigFileJson.hpp" #include "util/Assert.hpp" -#include "util/newconfig/Array.hpp" -#include "util/newconfig/Error.hpp" -#include "util/newconfig/Types.hpp" +#include "util/config/Array.hpp" +#include "util/config/Error.hpp" +#include "util/config/Types.hpp" #include #include diff --git a/src/util/newconfig/ConfigFileJson.hpp b/src/util/config/ConfigFileJson.hpp similarity index 96% rename from src/util/newconfig/ConfigFileJson.hpp rename to src/util/config/ConfigFileJson.hpp index 11eb92e10..8eb72b554 100644 --- a/src/util/newconfig/ConfigFileJson.hpp +++ b/src/util/config/ConfigFileJson.hpp @@ -19,9 +19,9 @@ #pragma once -#include "util/newconfig/ConfigFileInterface.hpp" -#include "util/newconfig/Error.hpp" -#include "util/newconfig/Types.hpp" +#include "util/config/ConfigFileInterface.hpp" +#include "util/config/Error.hpp" +#include "util/config/Types.hpp" #include diff --git a/src/util/newconfig/ConfigFileYaml.hpp b/src/util/config/ConfigFileYaml.hpp similarity index 94% rename from src/util/newconfig/ConfigFileYaml.hpp rename to src/util/config/ConfigFileYaml.hpp index ac943ef53..c67a797b8 100644 --- a/src/util/newconfig/ConfigFileYaml.hpp +++ b/src/util/config/ConfigFileYaml.hpp @@ -19,8 +19,8 @@ #pragma once -#include "util/newconfig/ConfigFileInterface.hpp" -#include "util/newconfig/Types.hpp" +#include "util/config/ConfigFileInterface.hpp" +#include "util/config/Types.hpp" #include diff --git a/src/util/newconfig/ConfigValue.hpp b/src/util/config/ConfigValue.hpp similarity index 96% rename from src/util/newconfig/ConfigValue.hpp rename to src/util/config/ConfigValue.hpp index 3911d2651..1e47d6fdc 100644 --- a/src/util/newconfig/ConfigValue.hpp +++ b/src/util/config/ConfigValue.hpp @@ -21,9 +21,9 @@ #include "util/Assert.hpp" #include "util/OverloadSet.hpp" -#include "util/newconfig/ConfigConstraints.hpp" -#include "util/newconfig/Error.hpp" -#include "util/newconfig/Types.hpp" +#include "util/config/ConfigConstraints.hpp" +#include "util/config/Error.hpp" +#include "util/config/Types.hpp" #include @@ -184,7 +184,7 @@ public: /** * @brief Check if value is optional * - * @return if value is optiona, false otherwise + * @return if value is optional, false otherwise */ [[nodiscard]] bool constexpr hasValue() const { @@ -217,8 +217,10 @@ public: stream << "- **Type**: " << val.type() << "\n"; if (val.description_.has_value()) { stream << "- **Default value**: " << *val.description_ << "\n"; + } else if (val.hasValue()) { + stream << "- **Default value**: `" << *val.value_ << "`\n"; } else { - stream << "- **Default value**: " << (val.hasValue() ? *val.value_ : "None") << "\n"; + stream << "- **Default value**: None\n"; } stream << "- **Constraints**: "; diff --git a/src/util/newconfig/Error.hpp b/src/util/config/Error.hpp similarity index 100% rename from src/util/newconfig/Error.hpp rename to src/util/config/Error.hpp diff --git a/src/util/newconfig/ObjectView.cpp b/src/util/config/ObjectView.cpp similarity index 94% rename from src/util/newconfig/ObjectView.cpp rename to src/util/config/ObjectView.cpp index c8624333d..238c4210e 100644 --- a/src/util/newconfig/ObjectView.cpp +++ b/src/util/config/ObjectView.cpp @@ -17,12 +17,12 @@ */ //============================================================================== -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ObjectView.hpp" #include "util/Assert.hpp" -#include "util/newconfig/ArrayView.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ValueView.hpp" +#include "util/config/ArrayView.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ValueView.hpp" #include diff --git a/src/util/newconfig/ObjectView.hpp b/src/util/config/ObjectView.hpp similarity index 99% rename from src/util/newconfig/ObjectView.hpp rename to src/util/config/ObjectView.hpp index 42977d174..895175e3d 100644 --- a/src/util/newconfig/ObjectView.hpp +++ b/src/util/config/ObjectView.hpp @@ -19,7 +19,7 @@ #pragma once -#include "util/newconfig/ValueView.hpp" +#include "util/config/ValueView.hpp" #include #include diff --git a/src/util/newconfig/Types.cpp b/src/util/config/Types.cpp similarity index 98% rename from src/util/newconfig/Types.cpp rename to src/util/config/Types.cpp index b8a6330db..0e4ac22d5 100644 --- a/src/util/newconfig/Types.cpp +++ b/src/util/config/Types.cpp @@ -17,7 +17,7 @@ */ //============================================================================== -#include "util/newconfig/Types.hpp" +#include "util/config/Types.hpp" #include #include diff --git a/src/util/newconfig/Types.hpp b/src/util/config/Types.hpp similarity index 100% rename from src/util/newconfig/Types.hpp rename to src/util/config/Types.hpp diff --git a/src/util/newconfig/ValueView.cpp b/src/util/config/ValueView.cpp similarity index 96% rename from src/util/newconfig/ValueView.cpp rename to src/util/config/ValueView.cpp index 63b7eba6f..6a66013db 100644 --- a/src/util/newconfig/ValueView.cpp +++ b/src/util/config/ValueView.cpp @@ -17,11 +17,11 @@ */ //============================================================================== -#include "util/newconfig/ValueView.hpp" +#include "util/config/ValueView.hpp" #include "util/Assert.hpp" -#include "util/newconfig/ConfigValue.hpp" -#include "util/newconfig/Types.hpp" +#include "util/config/ConfigValue.hpp" +#include "util/config/Types.hpp" #include #include diff --git a/src/util/newconfig/ValueView.hpp b/src/util/config/ValueView.hpp similarity index 98% rename from src/util/newconfig/ValueView.hpp rename to src/util/config/ValueView.hpp index d23848542..280137f7d 100644 --- a/src/util/newconfig/ValueView.hpp +++ b/src/util/config/ValueView.hpp @@ -20,9 +20,9 @@ #pragma once #include "util/Assert.hpp" -#include "util/newconfig/ConfigConstraints.hpp" -#include "util/newconfig/ConfigValue.hpp" -#include "util/newconfig/Types.hpp" +#include "util/config/ConfigConstraints.hpp" +#include "util/config/ConfigValue.hpp" +#include "util/config/Types.hpp" #include diff --git a/src/util/config/impl/Helpers.hpp b/src/util/config/impl/Helpers.hpp deleted file mode 100644 index 7703db00e..000000000 --- a/src/util/config/impl/Helpers.hpp +++ /dev/null @@ -1,170 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of clio: https://github.com/XRPLF/clio - Copyright (c) 2022, the clio developers. - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace util::impl { - -/** - * @brief Thrown when a KeyPath related error occurs - */ -struct KeyException : public ::std::logic_error { - KeyException(::std::string msg) : ::std::logic_error{msg} - { - } -}; - -/** - * @brief Thrown when a Store (config's storage) related error occurs. - */ -struct StoreException : public ::std::logic_error { - StoreException(::std::string msg) : ::std::logic_error{msg} - { - } -}; - -/** - * @brief Simple string tokenizer. Used by @ref Config. - * - * @tparam KeyType The type of key to use - * @tparam Separator The separator character - */ -template -class Tokenizer final { - using opt_key_t = std::optional; - KeyType key_; - KeyType token_{}; - std::queue tokens_{}; - -public: - explicit Tokenizer(KeyType key) : key_{key} - { - if (key.empty()) - throw KeyException("Empty key"); - - for (auto const& c : key) { - if (c == Separator) { - saveToken(); - } else { - token_ += c; - } - } - - saveToken(); - } - - [[nodiscard]] opt_key_t - next() - { - if (tokens_.empty()) - return std::nullopt; - auto token = tokens_.front(); - tokens_.pop(); - return std::make_optional(std::move(token)); - } - -private: - void - saveToken() - { - if (token_.empty()) - throw KeyException("Empty token in key '" + key_ + "'."); - tokens_.push(std::move(token_)); - token_ = {}; - } -}; - -template -static constexpr char const* -typeName() -{ - return typeid(T).name(); -} - -template <> -constexpr char const* -typeName() -{ - return "uint64_t"; -} - -template <> -constexpr char const* -typeName() -{ - return "int64_t"; -} - -template <> -constexpr char const* -typeName() -{ - return "uint32_t"; -} - -template <> -constexpr char const* -typeName() -{ - return "int32_t"; -} - -template <> -constexpr char const* -typeName() -{ - return "bool"; -} - -template <> -constexpr char const* -typeName() -{ - return "std::string"; -} - -template <> -constexpr char const* -typeName() -{ - return "const char*"; -} - -template <> -constexpr char const* -typeName() -{ - return "double"; -} - -template <> -constexpr char const* -typeName() -{ - return "float"; -} - -}; // namespace util::impl diff --git a/src/util/log/Logger.cpp b/src/util/log/Logger.cpp index 3fcad094f..8e6604c00 100644 --- a/src/util/log/Logger.cpp +++ b/src/util/log/Logger.cpp @@ -22,9 +22,9 @@ #include "util/Assert.hpp" #include "util/BytesConverter.hpp" #include "util/SourceLocation.hpp" -#include "util/newconfig/ArrayView.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ArrayView.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ObjectView.hpp" #include #include diff --git a/src/util/log/Logger.hpp b/src/util/log/Logger.hpp index 6390b47d8..f0a10200d 100644 --- a/src/util/log/Logger.hpp +++ b/src/util/log/Logger.hpp @@ -276,7 +276,7 @@ public: LogService() = delete; /** - * @brief Global log core initialization from a @ref Config + * @brief Global log core initialization from a @ref config::ClioConfigDefinition * * @param config The configuration to use * @return Void on success, error message on failure @@ -285,7 +285,7 @@ public: init(config::ClioConfigDefinition const& config); /** - * @brief Globally accesible General logger at Severity::TRC severity + * @brief Globally accessible General logger at Severity::TRC severity * * @param loc The source location of the log message * @return The pump to use for logging @@ -297,7 +297,7 @@ public: } /** - * @brief Globally accesible General logger at Severity::DBG severity + * @brief Globally accessible General logger at Severity::DBG severity * * @param loc The source location of the log message * @return The pump to use for logging @@ -309,7 +309,7 @@ public: } /** - * @brief Globally accesible General logger at Severity::NFO severity + * @brief Globally accessible General logger at Severity::NFO severity * * @param loc The source location of the log message * @return The pump to use for logging @@ -321,7 +321,7 @@ public: } /** - * @brief Globally accesible General logger at Severity::WRN severity + * @brief Globally accessible General logger at Severity::WRN severity * * @param loc The source location of the log message * @return The pump to use for logging @@ -333,7 +333,7 @@ public: } /** - * @brief Globally accesible General logger at Severity::ERR severity + * @brief Globally accessible General logger at Severity::ERR severity * * @param loc The source location of the log message * @return The pump to use for logging @@ -345,7 +345,7 @@ public: } /** - * @brief Globally accesible General logger at Severity::FTL severity + * @brief Globally accessible General logger at Severity::FTL severity * * @param loc The source location of the log message * @return The pump to use for logging @@ -357,7 +357,7 @@ public: } /** - * @brief Globally accesible Alert logger + * @brief Globally accessible Alert logger * * @param loc The source location of the log message * @return The pump to use for logging diff --git a/src/util/newconfig/ConfigDescription.hpp b/src/util/newconfig/ConfigDescription.hpp deleted file mode 100644 index 52fbc6d06..000000000 --- a/src/util/newconfig/ConfigDescription.hpp +++ /dev/null @@ -1,261 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of clio: https://github.com/XRPLF/clio - Copyright (c) 2024, the clio developers. - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#pragma once - -#include "util/Assert.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/Error.hpp" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace util::config { - -/** - * @brief All the config description are stored and extracted from this class - * - * Represents all the possible config description - */ -struct ClioConfigDescription { -public: - /** @brief Struct to represent a key-value pair*/ - struct KV { - std::string_view key; - std::string_view value; - }; - - /** - * @brief Constructs a new Clio Config Description based on pre-existing descriptions - * - * Config Keys and it's corresponding descriptions are all predefined. Used to generate markdown file - */ - constexpr ClioConfigDescription() = default; - - /** - * @brief Retrieves the description for a given key - * - * @param key The key to look up the description for - * @return The description associated with the key, or "Not Found" if the key does not exist - */ - [[nodiscard]] static constexpr std::string_view - get(std::string_view key) - { - auto const itr = std::ranges::find_if(kCONFIG_DESCRIPTION, [&](auto const& v) { return v.key == key; }); - ASSERT(itr != kCONFIG_DESCRIPTION.end(), "Key {} doesn't exist in config", key); - return itr->value; - } - - /** - * @brief Generate markdown file of all the clio config descriptions - * - * @param path The path location to generate the Config-description file - * @return An Error if generating markdown fails, otherwise nothing - */ - [[nodiscard]] static std::expected - generateConfigDescriptionToFile(std::filesystem::path path) - { - namespace fs = std::filesystem; - - // Validate the directory exists - auto const dir = path.parent_path(); - if (!dir.empty() && !fs::exists(dir)) { - return std::unexpected{ - fmt::format("Error: Directory '{}' does not exist or provided path is invalid", dir.string()) - }; - } - - std::ofstream file(path.string()); - if (!file.is_open()) { - return std::unexpected{fmt::format("Failed to create file '{}': {}", path.string(), std::strerror(errno))}; - } - - writeConfigDescriptionToFile(file); - file.close(); - - std::cout << "Markdown file generated successfully: " << path << "\n"; - return {}; - } - - /** - * @brief Writes to Config description to file - * - * @param file The config file to write to - */ - static void - writeConfigDescriptionToFile(std::ostream& file) - { - file << "# Clio Config Description\n"; - file << "This file lists all Clio Configuration definitions in detail.\n\n"; - file << "## Configuration Details\n\n"; - - for (auto const& [key, val] : kCONFIG_DESCRIPTION) { - file << "### Key: " << key << "\n"; - - // Every type of value is directed to operator<< in ConfigValue.hpp - // as ConfigValue is the one that holds all the info regarding the config values - if (key.contains("[]")) { - file << gClioConfig.asArray(key); - } else { - file << gClioConfig.getValueView(key); - } - file << " - **Description**: " << val << "\n"; - } - file << "\n"; - } - -private: - static constexpr auto kCONFIG_DESCRIPTION = std::array{ - KV{.key = "database.type", - .value = "Type of database to use. We currently support Cassandra and Scylladb. We default to Scylladb."}, - KV{.key = "database.cassandra.contact_points", - .value = "A list of IP addresses or hostnames of the initial nodes (Cassandra/Scylladb cluster nodes) that " - "the client will connect to when establishing a connection with the database. If you're running " - "locally, it should be 'localhost' or 127.0.0.1"}, - KV{.key = "database.cassandra.secure_connect_bundle", - .value = "Configuration file that contains the necessary security credentials and connection details for " - "securely " - "connecting to a Cassandra database cluster."}, - KV{.key = "database.cassandra.port", .value = "Port number to connect to the database."}, - KV{.key = "database.cassandra.keyspace", .value = "Keyspace to use for the database."}, - KV{.key = "database.cassandra.replication_factor", - .value = "Number of replicated nodes for Scylladb. Visit this link for more details : " - "https://university.scylladb.com/courses/scylla-essentials-overview/lessons/high-availability/" - "topic/fault-tolerance-replication-factor/ "}, - KV{.key = "database.cassandra.table_prefix", .value = "Prefix for Database table names."}, - KV{.key = "database.cassandra.max_write_requests_outstanding", - .value = "Maximum number of outstanding write requests. Write requests are api calls that write to database " - }, - KV{.key = "database.cassandra.max_read_requests_outstanding", - .value = "Maximum number of outstanding read requests, which reads from database"}, - KV{.key = "database.cassandra.threads", .value = "Number of threads that will be used for database operations." - }, - KV{.key = "database.cassandra.core_connections_per_host", - .value = "Number of core connections per host for Cassandra."}, - KV{.key = "database.cassandra.queue_size_io", .value = "Queue size for I/O operations in Cassandra."}, - KV{.key = "database.cassandra.write_batch_size", .value = "Batch size for write operations in Cassandra."}, - KV{.key = "database.cassandra.connect_timeout", - .value = "The maximum amount of time in seconds the system will wait for a connection to be successfully " - "established " - "with the database."}, - KV{.key = "database.cassandra.request_timeout", - .value = - "The maximum amount of time in seconds the system will wait for a request to be fetched from database."}, - KV{.key = "database.cassandra.username", .value = "The username used for authenticating with the database."}, - KV{.key = "database.cassandra.password", .value = "The password used for authenticating with the database."}, - KV{.key = "database.cassandra.certfile", - .value = "The path to the SSL/TLS certificate file used to establish a secure connection between the client " - "and the " - "Cassandra database."}, - KV{.key = "allow_no_etl", .value = "If True, no ETL nodes will run with Clio."}, - KV{.key = "etl_sources.[].ip", .value = "IP address of the ETL source."}, - KV{.key = "etl_sources.[].ws_port", .value = "WebSocket port of the ETL source."}, - KV{.key = "etl_sources.[].grpc_port", .value = "gRPC port of the ETL source."}, - KV{.key = "forwarding.cache_timeout", - .value = "Timeout duration for the forwarding cache used in Rippled communication."}, - KV{.key = "forwarding.request_timeout", - .value = "Timeout duration for the forwarding request used in Rippled communication."}, - KV{.key = "rpc.cache_timeout", .value = "Timeout duration for RPC requests."}, - KV{.key = "num_markers", - .value = "The number of markers is the number of coroutines to download the initial ledger"}, - KV{.key = "dos_guard.whitelist.[]", .value = "List of IP addresses to whitelist for DOS protection."}, - KV{.key = "dos_guard.max_fetches", .value = "Maximum number of fetch operations allowed by DOS guard."}, - KV{.key = "dos_guard.max_connections", .value = "Maximum number of concurrent connections allowed by DOS guard." - }, - KV{.key = "dos_guard.max_requests", .value = "Maximum number of requests allowed by DOS guard."}, - KV{.key = "dos_guard.sweep_interval", .value = "Interval in seconds for DOS guard to sweep/clear its state."}, - KV{.key = "workers", .value = "Number of threads to process RPC requests."}, - KV{.key = "server.ip", .value = "IP address of the Clio HTTP server."}, - KV{.key = "server.port", .value = "Port number of the Clio HTTP server."}, - KV{.key = "server.max_queue_size", - .value = "Maximum size of the server's request queue. Value of 0 is no limit."}, - KV{.key = "server.local_admin", - .value = "Indicates if the server should run with admin privileges. Only one of local_admin or " - "admin_password can be set."}, - KV{.key = "server.admin_password", - .value = "Password for Clio admin-only APIs. Only one of local_admin or admin_password can be set."}, - KV{.key = "server.processing_policy", - .value = R"(Could be "sequent" or "parallel". For the sequent policy, requests from a single client - connection are processed one by one, with the next request read only after the previous one is processed. For the parallel policy, Clio will accept - all requests and process them in parallel, sending a reply for each request as soon as it is ready.)"}, - KV{.key = "server.parallel_requests_limit", - .value = - R"(Optional parameter, used only if processing_strategy `parallel`. It limits the number of requests for a single client connection that are processed in parallel. If not specified, the limit is infinite.)" - }, - KV{.key = "server.ws_max_sending_queue_size", .value = "Maximum size of the websocket sending queue."}, - KV{.key = "prometheus.enabled", .value = "Enable or disable Prometheus metrics."}, - KV{.key = "prometheus.compress_reply", .value = "Enable or disable compression of Prometheus responses."}, - KV{.key = "io_threads", .value = "Number of I/O threads. Value cannot be less than 1"}, - KV{.key = "subscription_workers", - .value = "The number of worker threads or processes that are responsible for managing and processing " - "subscription-based tasks from rippled"}, - KV{.key = "graceful_period", .value = "Number of milliseconds server will wait to shutdown gracefully."}, - KV{.key = "cache.num_diffs", .value = "Number of diffs to cache. For more info, consult readme.md in etc"}, - KV{.key = "cache.num_markers", .value = "Number of markers to cache."}, - KV{.key = "cache.num_cursors_from_diff", .value = "Num of cursors that are different."}, - KV{.key = "cache.num_cursors_from_account", .value = "Number of cursors from an account."}, - KV{.key = "cache.page_fetch_size", .value = "Page fetch size for cache operations."}, - KV{.key = "cache.load", .value = "Cache loading strategy ('sync' or 'async')."}, - KV{.key = "log_channels.[].channel", - .value = "Name of the log channel." - "'RPC', 'ETL', and 'Performance'"}, - KV{.key = "log_channels.[].log_level", - .value = "Log level for the specific log channel." - "`warning`, `error`, `fatal`"}, - KV{.key = "log_level", - .value = "General logging level of Clio. This level will be applied to all log channels that do not have an " - "explicitly defined logging level."}, - KV{.key = "log_format", .value = "Format string for log messages."}, - KV{.key = "log_to_console", .value = "Enable or disable logging to console."}, - KV{.key = "log_directory", .value = "Directory path for log files."}, - KV{.key = "log_rotation_size", - .value = - "Log rotation size in megabytes. When the log file reaches this particular size, a new log file starts." - }, - KV{.key = "log_directory_max_size", .value = "Maximum size of the log directory in megabytes."}, - KV{.key = "log_rotation_hour_interval", - .value = "Interval in hours for log rotation. If the current log file reaches this value in logging, a new " - "log file starts."}, - KV{.key = "log_tag_style", .value = "Style for log tags."}, - KV{.key = "extractor_threads", .value = "Number of extractor threads."}, - KV{.key = "read_only", .value = "Indicates if the server should have read-only privileges."}, - KV{.key = "txn_threshold", .value = "Transaction threshold value."}, - KV{.key = "start_sequence", .value = "Starting ledger index."}, - KV{.key = "finish_sequence", .value = "Ending ledger index."}, - KV{.key = "ssl_cert_file", .value = "Path to the SSL certificate file."}, - KV{.key = "ssl_key_file", .value = "Path to the SSL key file."}, - KV{.key = "api_version.default", .value = "Default API version Clio will run on."}, - KV{.key = "api_version.min", .value = "Minimum API version."}, - KV{.key = "api_version.max", .value = "Maximum API version."}, - KV{.key = "migration.full_scan_threads", .value = "The number of threads used to scan the table."}, - KV{.key = "migration.full_scan_jobs", .value = "The number of coroutines used to scan the table."}, - KV{.key = "migration.cursors_per_job", .value = "The number of cursors each coroutine will scan."} - }; -}; - -} // namespace util::config diff --git a/src/util/prometheus/Prometheus.cpp b/src/util/prometheus/Prometheus.cpp index 806e52a7e..dc9798e2e 100644 --- a/src/util/prometheus/Prometheus.cpp +++ b/src/util/prometheus/Prometheus.cpp @@ -20,7 +20,7 @@ #include "util/prometheus/Prometheus.hpp" #include "util/Assert.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/prometheus/Bool.hpp" #include "util/prometheus/Counter.hpp" #include "util/prometheus/Gauge.hpp" diff --git a/src/util/prometheus/Prometheus.hpp b/src/util/prometheus/Prometheus.hpp index d295d9556..15615c0ba 100644 --- a/src/util/prometheus/Prometheus.hpp +++ b/src/util/prometheus/Prometheus.hpp @@ -182,7 +182,7 @@ private: }; /** - * @brief Implemetation of PrometheusInterface + * @brief Implementation of PrometheusInterface * * @note When prometheus is disabled, all metrics will still counted but collection is disabled */ diff --git a/src/web/AdminVerificationStrategy.cpp b/src/web/AdminVerificationStrategy.cpp index f3b22af62..1902e2120 100644 --- a/src/web/AdminVerificationStrategy.cpp +++ b/src/web/AdminVerificationStrategy.cpp @@ -20,7 +20,7 @@ #include "web/AdminVerificationStrategy.hpp" #include "util/JsonUtils.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include #include diff --git a/src/web/AdminVerificationStrategy.hpp b/src/web/AdminVerificationStrategy.hpp index a2920eea1..20a199af2 100644 --- a/src/web/AdminVerificationStrategy.hpp +++ b/src/web/AdminVerificationStrategy.hpp @@ -19,7 +19,7 @@ #pragma once -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include #include diff --git a/src/web/CMakeLists.txt b/src/web/CMakeLists.txt index 251fda1bd..2b62bbe41 100644 --- a/src/web/CMakeLists.txt +++ b/src/web/CMakeLists.txt @@ -5,6 +5,7 @@ target_sources( PRIVATE AdminVerificationStrategy.cpp dosguard/DOSGuard.cpp dosguard/IntervalSweepHandler.cpp + dosguard/Weights.cpp dosguard/WhitelistHandler.cpp ng/Connection.cpp ng/impl/ErrorHandling.cpp diff --git a/src/web/RPCServerHandler.hpp b/src/web/RPCServerHandler.hpp index 632d9551e..38494fae5 100644 --- a/src/web/RPCServerHandler.hpp +++ b/src/web/RPCServerHandler.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/Factories.hpp" #include "rpc/JS.hpp" @@ -28,8 +29,9 @@ #include "util/JsonUtils.hpp" #include "util/Profiler.hpp" #include "util/Taggable.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "web/dosguard/DOSGuardInterface.hpp" #include "web/impl/ErrorHandling.hpp" #include "web/interface/ConnectionBase.hpp" @@ -58,13 +60,14 @@ namespace web { * * Note: see @ref web::SomeServerHandler concept */ -template +template class RPCServerHandler { std::shared_ptr const backend_; std::shared_ptr const rpcEngine_; - std::shared_ptr const etl_; + std::shared_ptr const etl_; util::TagDecoratorFactory const tagFactory_; rpc::impl::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed + std::reference_wrapper dosguard_; util::Logger log_{"RPC"}; util::Logger perfLog_{"Performance"}; @@ -77,18 +80,21 @@ public: * @param backend The backend to use * @param rpcEngine The RPC engine to use * @param etl The ETL to use + * @param dosguard The DOS guard service to use for request rate limiting */ RPCServerHandler( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& rpcEngine, - std::shared_ptr const& etl + std::shared_ptr const& etl, + web::dosguard::DOSGuardInterface& dosguard ) : backend_(backend) , rpcEngine_(rpcEngine) , etl_(etl) , tagFactory_(config) , apiVersionParser_(config.getObject("api_version")) + , dosguard_(dosguard) { } @@ -101,6 +107,11 @@ public: void operator()(std::string const& request, std::shared_ptr const& connection) { + if (not dosguard_.get().isOk(connection->clientIp)) { + connection->sendSlowDown(request); + return; + } + try { auto req = boost::json::parse(request).as_object(); LOG(perfLog_.debug()) << connection->tag() << "Adding to work queue"; @@ -108,6 +119,11 @@ public: if (not connection->upgraded and shouldReplaceParams(req)) req[JS(params)] = boost::json::array({boost::json::object{}}); + if (not dosguard_.get().request(connection->clientIp, req)) { + connection->sendSlowDown(request); + return; + } + if (!rpcEngine_->post( [this, request = std::move(req), connection](boost::asio::yield_context yield) mutable { handleRequest(yield, std::move(request), connection); @@ -195,8 +211,8 @@ private: auto [result, timeDiff] = util::timed([&]() { return rpcEngine_->buildResponse(*context); }); - auto us = std::chrono::duration(timeDiff); - rpc::logDuration(*context, us); + auto const us = std::chrono::duration(timeDiff); + rpc::logDuration(request, context->tag(), us); boost::json::object response; diff --git a/src/web/Server.hpp b/src/web/Server.hpp index 0b2fee4fb..b480b322f 100644 --- a/src/web/Server.hpp +++ b/src/web/Server.hpp @@ -120,7 +120,7 @@ public: } /** - * @brief A helper function that is called when any error ocurs. + * @brief A helper function that is called when any error occurs. * * @param ec The error code * @param message The message to include in the log @@ -326,7 +326,7 @@ using HttpServer = Server; /** * @brief A factory function that spawns a ready to use HTTP server. * - * @tparam HandlerType The tyep of handler to process the request + * @tparam HandlerType The type of handler to process the request * @param config The config to create server * @param ioc The server will run under this io_context * @param dosGuard The dos guard to protect the server diff --git a/src/web/dosguard/DOSGuard.cpp b/src/web/dosguard/DOSGuard.cpp index 40a28610a..d618235c5 100644 --- a/src/web/dosguard/DOSGuard.cpp +++ b/src/web/dosguard/DOSGuard.cpp @@ -20,12 +20,15 @@ #include "web/dosguard/DOSGuard.hpp" #include "util/Assert.hpp" +#include "util/config/ArrayView.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ValueView.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ArrayView.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ValueView.hpp" +#include "web/dosguard/WeightsInterface.hpp" #include "web/dosguard/WhitelistHandlerInterface.hpp" +#include + #include #include #include @@ -37,8 +40,13 @@ using namespace util::config; namespace web::dosguard { -DOSGuard::DOSGuard(ClioConfigDefinition const& config, WhitelistHandlerInterface const& whitelistHandler) +DOSGuard::DOSGuard( + ClioConfigDefinition const& config, + WhitelistHandlerInterface const& whitelistHandler, + WeightsInterface const& weights +) : whitelistHandler_{std::cref(whitelistHandler)} + , weights_(weights) , maxFetches_{config.get("dos_guard.max_fetches")} , maxConnCount_{config.get("dos_guard.max_connections")} , maxRequestCount_{config.get("dos_guard.max_requests")} @@ -59,11 +67,11 @@ DOSGuard::isOk(std::string const& ip) const noexcept { auto lock = mtx_.lock(); - if (lock->ipState.find(ip) != lock->ipState.end()) { - auto [transferredByte, requests] = lock->ipState.at(ip); + if (auto const it = lock->ipState.find(ip); it != lock->ipState.end()) { + auto const [transferredByte, requests] = it->second; if (transferredByte > maxFetches_ || requests > maxRequestCount_) { LOG(log_.warn()) << "Dosguard: Client surpassed the rate limit. ip = " << ip - << " Transfered Byte: " << transferredByte << "; Requests: " << requests; + << " Transferred Byte: " << transferredByte << "; Requests: " << requests; return false; } } @@ -108,21 +116,23 @@ DOSGuard::add(std::string const& ip, uint32_t numObjects) noexcept { auto lock = mtx_.lock(); - lock->ipState[ip].transferedByte += numObjects; + lock->ipState[ip].transferredByte += numObjects; } return isOk(ip); } [[maybe_unused]] bool -DOSGuard::request(std::string const& ip) noexcept +DOSGuard::request(std::string const& ip, boost::json::object const& request) { if (whitelistHandler_.get().isWhiteListed(ip)) return true; + auto const weight = weights_.get().requestWeight(request); + { auto lock = mtx_.lock(); - lock->ipState[ip].requestsCount++; + lock->ipState[ip].requestsCount += weight; } return isOk(ip); diff --git a/src/web/dosguard/DOSGuard.hpp b/src/web/dosguard/DOSGuard.hpp index 0fd83bfda..e4febe1f8 100644 --- a/src/web/dosguard/DOSGuard.hpp +++ b/src/web/dosguard/DOSGuard.hpp @@ -20,13 +20,15 @@ #pragma once #include "util/Mutex.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include "web/dosguard/DOSGuardInterface.hpp" +#include "web/dosguard/WeightsInterface.hpp" #include "web/dosguard/WhitelistHandlerInterface.hpp" #include #include +#include #include #include @@ -48,8 +50,8 @@ class DOSGuard : public DOSGuardInterface { * @brief Accumulated state per IP, state will be reset accordingly */ struct ClientState { - std::uint32_t transferedByte = 0; /**< Accumulated transferred byte */ - std::uint32_t requestsCount = 0; /**< Accumulated served requests count */ + std::uint32_t transferredByte = 0; /**< Accumulated transferred byte */ + std::uint32_t requestsCount = 0; /**< Accumulated served requests count */ }; struct State { @@ -59,6 +61,7 @@ class DOSGuard : public DOSGuardInterface { util::Mutex mtx_; std::reference_wrapper whitelistHandler_; + std::reference_wrapper weights_; std::uint32_t const maxFetches_; std::uint32_t const maxConnCount_; @@ -71,8 +74,13 @@ public: * * @param config Clio config * @param whitelistHandler Whitelist handler that checks whitelist for IP addresses + * @param weights API methods weights */ - DOSGuard(util::config::ClioConfigDefinition const& config, WhitelistHandlerInterface const& whitelistHandler); + DOSGuard( + util::config::ClioConfigDefinition const& config, + WhitelistHandlerInterface const& whitelistHandler, + WeightsInterface const& weights + ); /** * @brief Check whether an ip address is in the whitelist or not. @@ -133,11 +141,12 @@ public: * returned otherwise. * * @param ip + * @param request The request as json object * @return true * @return false */ [[maybe_unused]] bool - request(std::string const& ip) noexcept override; + request(std::string const& ip, boost::json::object const& request) override; /** * @brief Instantly clears all fetch counters added by @see add(std::string const&, uint32_t). diff --git a/src/web/dosguard/DOSGuardInterface.hpp b/src/web/dosguard/DOSGuardInterface.hpp index 1eee27b65..684087c9f 100644 --- a/src/web/dosguard/DOSGuardInterface.hpp +++ b/src/web/dosguard/DOSGuardInterface.hpp @@ -19,6 +19,8 @@ #pragma once +#include + #include #include #include @@ -99,12 +101,13 @@ public: * * * @param ip + * @param request The request as json object * @return If the total sums up to a value equal or larger than maxRequestCount_ * the operation is no longer allowed and false is returned; true is * returned otherwise. */ [[maybe_unused]] virtual bool - request(std::string const& ip) noexcept = 0; + request(std::string const& ip, boost::json::object const& request) = 0; }; } // namespace web::dosguard diff --git a/src/web/dosguard/IntervalSweepHandler.cpp b/src/web/dosguard/IntervalSweepHandler.cpp index 2049c0775..04427ccf8 100644 --- a/src/web/dosguard/IntervalSweepHandler.cpp +++ b/src/web/dosguard/IntervalSweepHandler.cpp @@ -19,7 +19,7 @@ #include "web/dosguard/IntervalSweepHandler.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include "web/dosguard/DOSGuardInterface.hpp" #include diff --git a/src/web/dosguard/IntervalSweepHandler.hpp b/src/web/dosguard/IntervalSweepHandler.hpp index 27cd2da4d..b80e1a0d0 100644 --- a/src/web/dosguard/IntervalSweepHandler.hpp +++ b/src/web/dosguard/IntervalSweepHandler.hpp @@ -20,7 +20,7 @@ #pragma once #include "util/Repeat.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include diff --git a/src/web/dosguard/Weights.cpp b/src/web/dosguard/Weights.cpp new file mode 100644 index 000000000..fcddd8bb1 --- /dev/null +++ b/src/web/dosguard/Weights.cpp @@ -0,0 +1,106 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "web/dosguard/Weights.hpp" + +#include "rpc/JS.hpp" +#include "util/Assert.hpp" +#include "util/config/ArrayView.hpp" +#include "util/config/ConfigDefinition.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace web::dosguard { + +Weights::Weights(size_t defaultWeight, std::unordered_map weights) + : defaultWeight_(defaultWeight), weights_(std::move_iterator(weights.begin()), std::move_iterator(weights.end())) +{ +} + +Weights +Weights::make(util::config::ClioConfigDefinition const& config) +{ + std::unordered_map weights; + auto const configWeights = config.getArray("dos_guard.__ng_weights"); + for (size_t i = 0; i < configWeights.size(); ++i) { + auto const w = configWeights.objectAt(i); + Weights::Entry const entry{ + .weight = w.get("weight"), + .weightLedgerCurrent = w.maybeValue("weight_ledger_current"), + .weightLedgerValidated = w.maybeValue("weight_ledger_validated"), + }; + weights.emplace(w.get("method"), entry); + } + return Weights{config.get("dos_guard.__ng_default_weight"), std::move(weights)}; +} + +size_t +Weights::requestWeight(boost::json::object const& request) const +{ + if (not((request.contains(JS(method)) and request.at(JS(method)).is_string()) or + (request.contains(JS(command)) and request.at(JS(command)).is_string()))) { + return defaultWeight_; + } + + std::string_view const cmd = + request.contains(JS(method)) ? request.at(JS(method)).as_string() : request.at(JS(command)).as_string(); + + auto it = weights_.find(cmd); + if (it == weights_.end()) { + return defaultWeight_; + } + + auto const& entry = it->second; + + boost::json::value const* ledgerIndex = nullptr; + if (request.contains(JS(ledger_index))) { + ledgerIndex = &request.at(JS(ledger_index)); + } else if (request.contains(JS(params))) { + ASSERT( + request.at(JS(params)).is_array() and not request.at(JS(params)).as_array().empty() and + request.at(JS(params)).as_array().at(0).is_object(), + "params should be [{{}}]" + ); + if (auto const& params = request.at(JS(params)).as_array().at(0).as_object(); + params.contains(JS(ledger_index))) { + ledgerIndex = ¶ms.at(JS(ledger_index)); + } + } + + if (ledgerIndex != nullptr and ledgerIndex->is_string()) { + auto const& ledgerIndexString = ledgerIndex->as_string(); + if (ledgerIndexString == JS(validated)) { + return entry.weightLedgerValidated.value_or(entry.weight); + } + if (ledgerIndexString == JS(current)) { + return entry.weightLedgerCurrent.value_or(entry.weight); + } + } + return entry.weight; +} + +} // namespace web::dosguard diff --git a/src/web/dosguard/Weights.hpp b/src/web/dosguard/Weights.hpp new file mode 100644 index 000000000..016a3eea9 --- /dev/null +++ b/src/web/dosguard/Weights.hpp @@ -0,0 +1,88 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "util/StringHash.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "web/dosguard/WeightsInterface.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace web::dosguard { + +/** + * @brief Implementation of WeightsInterface that manages command weights for DosGuard. + * + * This class provides a mechanism to assign different weights to API commands + * for the purpose of DOS protection calculations. Commands can have specific weights, + * or fall back to a default weight. + */ +class Weights : public WeightsInterface { +public: + /** + * @brief Structure representing weight configuration for a command. + * + * Contains the base weight and optional specialized weights for different ledger specifications. + */ + struct Entry { + size_t weight; + std::optional weightLedgerCurrent; + std::optional weightLedgerValidated; + }; + +private: + size_t defaultWeight_; + std::unordered_map> weights_; + +public: + /** + * @brief Construct a new Weights object + * + * @param defaultWeight The default weight to use when a command-specific weight is not defined + * @param weights Map of command names to their specific weights + */ + Weights(size_t defaultWeight, std::unordered_map weights); + + /** + * @brief Create a Weights object from configuration + * + * @param config The application configuration + * @return Weights instance initialized with values from configuration + */ + static Weights + make(util::config::ClioConfigDefinition const& config); + + /** + * @brief Get the weight assigned to a specific command + * + * @param request Json request + * @return size_t The weight value (specific weight if defined, otherwise default weight) + */ + size_t + requestWeight(boost::json::object const& request) const override; +}; + +} // namespace web::dosguard diff --git a/src/web/dosguard/WeightsInterface.hpp b/src/web/dosguard/WeightsInterface.hpp new file mode 100644 index 000000000..916a25f49 --- /dev/null +++ b/src/web/dosguard/WeightsInterface.hpp @@ -0,0 +1,48 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include + +#include + +namespace web::dosguard { + +/** + * @brief Interface for determining request weights in DOS protection. + * + * This interface defines the contract for classes that calculate weights for incoming + * requests, which is used for DOS protection mechanisms. + */ +class WeightsInterface { +public: + virtual ~WeightsInterface() = default; + + /** + * @brief Calculate the weight of a request. + * + * @param request The JSON object representing the request + * @return The calculated weight of the request + */ + virtual size_t + requestWeight(boost::json::object const& request) const = 0; +}; + +} // namespace web::dosguard diff --git a/src/web/dosguard/WhitelistHandler.hpp b/src/web/dosguard/WhitelistHandler.hpp index f7abbfd07..532c708fa 100644 --- a/src/web/dosguard/WhitelistHandler.hpp +++ b/src/web/dosguard/WhitelistHandler.hpp @@ -19,9 +19,9 @@ #pragma once -#include "util/newconfig/ArrayView.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ValueView.hpp" +#include "util/config/ArrayView.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ValueView.hpp" #include "web/Resolver.hpp" #include "web/dosguard/WhitelistHandlerInterface.hpp" diff --git a/src/web/impl/ErrorHandling.hpp b/src/web/impl/ErrorHandling.hpp index e35126ec4..d200810c6 100644 --- a/src/web/impl/ErrorHandling.hpp +++ b/src/web/impl/ErrorHandling.hpp @@ -78,8 +78,8 @@ public: case rpc::ClioError::RpcCommandNotString: connection_->send("method is not string", boost::beast::http::status::bad_request); break; - case rpc::ClioError::RpcParamsUnparseable: - connection_->send("params unparseable", boost::beast::http::status::bad_request); + case rpc::ClioError::RpcParamsUnparsable: + connection_->send("params unparsable", boost::beast::http::status::bad_request); break; // others are not applicable but we want a compilation error next time we add one diff --git a/src/web/impl/HttpBase.hpp b/src/web/impl/HttpBase.hpp index cfc257e80..5f3fa26cd 100644 --- a/src/web/impl/HttpBase.hpp +++ b/src/web/impl/HttpBase.hpp @@ -240,19 +240,7 @@ public: return sender_(httpResponse(http::status::bad_request, "text/html", "Expected a POST request")); } - // to avoid overwhelm work queue, the request limit check should be - // before posting to queue the web socket creation will be guarded via - // connection limit - if (!dosGuard_.get().request(clientIp)) { - // TODO: this looks like it could be useful to count too in the future - return sender_(httpResponse( - http::status::service_unavailable, - "text/plain", - boost::json::serialize(rpc::makeError(rpc::RippledError::rpcSLOW_DOWN)) - )); - } - - LOG(log_.info()) << tag() << "Received request from ip = " << clientIp << " - posting to WorkQueue"; + LOG(log_.info()) << tag() << "Received request from ip = " << clientIp; try { (*handler_)(req_.body(), derived().shared_from_this()); @@ -265,6 +253,16 @@ public: } } + void + sendSlowDown(std::string const&) override + { + sender_(httpResponse( + http::status::service_unavailable, + "text/plain", + boost::json::serialize(rpc::makeError(rpc::RippledError::rpcSLOW_DOWN)) + )); + } + /** * @brief Send a response to the client * The message length will be added to the DOSGuard, if the limit is reached, a warning will be added to the diff --git a/src/web/impl/WsBase.hpp b/src/web/impl/WsBase.hpp index 9be6b334a..0538a34e2 100644 --- a/src/web/impl/WsBase.hpp +++ b/src/web/impl/WsBase.hpp @@ -164,6 +164,12 @@ public: doWrite(); } + void + sendSlowDown(std::string const& request) override + { + sendError(rpc::RippledError::rpcSLOW_DOWN, request); + } + /** * @brief Send a message to the client * @param msg The message to send, it will keep the string alive until it is sent. It is useful when we have @@ -173,7 +179,8 @@ public: void send(std::shared_ptr msg) override { - boost::asio::dispatch( + // Note: post used instead of dispatch to guarantee async behavior of wsFail and maybeSendNext + boost::asio::post( derived().ws().get_executor(), [this, self = derived().shared_from_this(), msg = std::move(msg)]() { if (messages_.size() > maxSendingQueueSize_) { @@ -229,7 +236,7 @@ public: } /** - * @brief Accept the session asynchroniously + * @brief Accept the session asynchronously */ void run(http::request req) @@ -279,36 +286,33 @@ public: LOG(perfLog_.info()) << tag() << "Received request from ip = " << this->clientIp; - auto sendError = [this](auto error, std::string&& requestStr) { - auto e = rpc::makeError(error); - - try { - auto request = boost::json::parse(requestStr); - if (request.is_object() && request.as_object().contains("id")) - e["id"] = request.as_object().at("id"); - e["request"] = std::move(request); - } catch (std::exception const&) { - e["request"] = std::move(requestStr); - } - - this->send(std::make_shared(boost::json::serialize(e))); - }; - std::string requestStr{static_cast(buffer_.data().data()), buffer_.size()}; - // dosGuard served request++ and check ip address - if (!dosGuard_.get().request(clientIp)) { - // TODO: could be useful to count in counters in the future too - sendError(rpc::RippledError::rpcSLOW_DOWN, std::move(requestStr)); - } else { - try { - (*handler_)(requestStr, shared_from_this()); - } catch (std::exception const&) { - sendError(rpc::RippledError::rpcINTERNAL, std::move(requestStr)); - } + try { + (*handler_)(requestStr, shared_from_this()); + } catch (std::exception const&) { + sendError(rpc::RippledError::rpcINTERNAL, std::move(requestStr)); } doRead(); } + +private: + void + sendError(rpc::RippledError error, std::string requestStr) + { + auto e = rpc::makeError(error); + + try { + auto request = boost::json::parse(requestStr); + if (request.is_object() && request.as_object().contains("id")) + e["id"] = request.as_object().at("id"); + e["request"] = std::move(request); + } catch (std::exception const&) { + e["request"] = requestStr; + } + + this->send(std::make_shared(boost::json::serialize(e))); + } }; } // namespace web::impl diff --git a/src/web/interface/ConnectionBase.hpp b/src/web/interface/ConnectionBase.hpp index 839b1d9b0..894ce0dec 100644 --- a/src/web/interface/ConnectionBase.hpp +++ b/src/web/interface/ConnectionBase.hpp @@ -82,6 +82,13 @@ public: throw std::logic_error("web server can not send the shared payload"); } + /** + * @brief Send a "slow down" error response to the client. + * + * @param request The original request that triggered the rate limiting + */ + virtual void + sendSlowDown(std::string const& request) = 0; /** * @brief Get the subscription context for this connection. * diff --git a/src/web/ng/RPCServerHandler.hpp b/src/web/ng/RPCServerHandler.hpp index f8dbcb183..64db00627 100644 --- a/src/web/ng/RPCServerHandler.hpp +++ b/src/web/ng/RPCServerHandler.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/Factories.hpp" #include "rpc/JS.hpp" @@ -32,6 +33,7 @@ #include "util/Taggable.hpp" #include "util/log/Logger.hpp" #include "web/SubscriptionContextInterface.hpp" +#include "web/dosguard/DOSGuardInterface.hpp" #include "web/ng/Connection.hpp" #include "web/ng/Request.hpp" #include "web/ng/Response.hpp" @@ -64,11 +66,12 @@ namespace web::ng { * * Note: see @ref web::SomeServerHandler concept */ -template +template class RPCServerHandler { std::shared_ptr const backend_; std::shared_ptr const rpcEngine_; - std::shared_ptr const etl_; + std::shared_ptr const etl_; + std::reference_wrapper dosguard_; util::TagDecoratorFactory const tagFactory_; rpc::impl::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed @@ -83,16 +86,19 @@ public: * @param backend The backend to use * @param rpcEngine The RPC engine to use * @param etl The ETL to use + * @param dosguard The DOS guard service to use for request rate limiting */ RPCServerHandler( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& rpcEngine, - std::shared_ptr const& etl + std::shared_ptr const& etl, + dosguard::DOSGuardInterface& dosguard ) : backend_(backend) , rpcEngine_(rpcEngine) , etl_(etl) + , dosguard_(dosguard) , tagFactory_(config) , apiVersionParser_(config.getObject("api_version")) { @@ -115,6 +121,10 @@ public: boost::asio::yield_context yield ) { + if (not dosguard_.get().isOk(connectionMetadata.ip())) { + return makeSlowDownResponse(request, std::nullopt); + } + std::optional response; util::CoroutineGroup coroutineGroup{yield, 1}; auto const onTaskComplete = coroutineGroup.registerForeign(yield); @@ -141,18 +151,23 @@ public: } } else { auto parsedObject = std::move(parsedRequest).as_object(); - LOG(perfLog_.debug()) << connectionMetadata.tag() << "Adding to work queue"; - if (not connectionMetadata.wasUpgraded() and shouldReplaceParams(parsedObject)) - parsedObject[JS(params)] = boost::json::array({boost::json::object{}}); + if (not dosguard_.get().request(connectionMetadata.ip(), parsedObject)) { + response = makeSlowDownResponse(request, parsedObject); + } else { + LOG(perfLog_.debug()) << connectionMetadata.tag() << "Adding to work queue"; - response = handleRequest( - innerYield, - request, - std::move(parsedObject), - connectionMetadata, - std::move(subscriptionContext) - ); + if (not connectionMetadata.wasUpgraded() and shouldReplaceParams(parsedObject)) + parsedObject[JS(params)] = boost::json::array({boost::json::object{}}); + + response = handleRequest( + innerYield, + request, + std::move(parsedObject), + connectionMetadata, + std::move(subscriptionContext) + ); + } } } catch (std::exception const& ex) { LOG(perfLog_.error()) << connectionMetadata.tag() << "Caught exception: " << ex.what(); @@ -176,6 +191,11 @@ public: // Put the coroutine to sleep until the foreign task is done coroutineGroup.asyncWait(yield); ASSERT(response.has_value(), "Woke up coroutine without setting response"); + + if (not dosguard_.get().add(connectionMetadata.ip(), response->message().size())) { + response->setMessage(makeLoadWarning(*response)); + } + return std::move(response).value(); } @@ -203,7 +223,7 @@ private: auto const context = [&] { if (connectionMetadata.wasUpgraded()) { - ASSERT(subscriptionContext != nullptr, "Subscription context must exist for a WS connecton"); + ASSERT(subscriptionContext != nullptr, "Subscription context must exist for a WS connection"); return rpc::makeWsContext( yield, request, @@ -240,7 +260,7 @@ private: auto [result, timeDiff] = util::timed([&]() { return rpcEngine_->buildResponse(*context); }); auto us = std::chrono::duration(timeDiff); - rpc::logDuration(*context, us); + rpc::logDuration(request, context->tag(), us); boost::json::object response; @@ -315,6 +335,39 @@ private: } } + static Response + makeSlowDownResponse(Request const& request, std::optional requestJson) + { + auto error = rpc::makeError(rpc::RippledError::rpcSLOW_DOWN); + + if (not request.isHttp()) { + try { + if (not requestJson.has_value()) { + requestJson = boost::json::parse(request.message()); + } + if (requestJson->is_object() && requestJson->as_object().contains("id")) + error["id"] = requestJson->as_object().at("id"); + error["request"] = request.message(); + } catch (std::exception const&) { + error["request"] = request.message(); + } + } + return web::ng::Response{boost::beast::http::status::service_unavailable, error, request}; + } + + static boost::json::object + makeLoadWarning(Response const& response) + { + auto jsonResponse = boost::json::parse(response.message()).as_object(); + jsonResponse["warning"] = "load"; + if (jsonResponse.contains("warnings") && jsonResponse["warnings"].is_array()) { + jsonResponse["warnings"].as_array().push_back(rpc::makeWarning(rpc::WarnRpcRateLimit)); + } else { + jsonResponse["warnings"] = boost::json::array{rpc::makeWarning(rpc::WarnRpcRateLimit)}; + } + return jsonResponse; + } + bool shouldReplaceParams(boost::json::object const& req) const { diff --git a/src/web/ng/Server.cpp b/src/web/ng/Server.cpp index be3fe1064..a8bb6477e 100644 --- a/src/web/ng/Server.cpp +++ b/src/web/ng/Server.cpp @@ -21,9 +21,9 @@ #include "util/Assert.hpp" #include "util/Taggable.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ObjectView.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ObjectView.hpp" #include "web/ng/Connection.hpp" #include "web/ng/MessageHandler.hpp" #include "web/ng/ProcessingPolicy.hpp" diff --git a/src/web/ng/Server.hpp b/src/web/ng/Server.hpp index 188f6a883..1102f79cc 100644 --- a/src/web/ng/Server.hpp +++ b/src/web/ng/Server.hpp @@ -20,8 +20,8 @@ #pragma once #include "util/Taggable.hpp" +#include "util/config/ConfigDefinition.hpp" #include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include "web/ng/Connection.hpp" #include "web/ng/MessageHandler.hpp" #include "web/ng/ProcessingPolicy.hpp" diff --git a/src/web/ng/impl/ConnectionHandler.cpp b/src/web/ng/impl/ConnectionHandler.cpp index adfd8bc12..838c4bfb5 100644 --- a/src/web/ng/impl/ConnectionHandler.cpp +++ b/src/web/ng/impl/ConnectionHandler.cpp @@ -45,7 +45,6 @@ #include #include #include -#include #include namespace web::ng::impl { @@ -86,24 +85,6 @@ handleWsRequest( } // namespace -size_t -ConnectionHandler::StringHash::operator()(char const* str) const -{ - return hash_type{}(str); -} - -size_t -ConnectionHandler::StringHash::operator()(std::string_view str) const -{ - return hash_type{}(str); -} - -size_t -ConnectionHandler::StringHash::operator()(std::string const& str) const -{ - return hash_type{}(str); -} - ConnectionHandler::ConnectionHandler( ProcessingPolicy processingPolicy, std::optional maxParallelRequests, diff --git a/src/web/ng/impl/ConnectionHandler.hpp b/src/web/ng/impl/ConnectionHandler.hpp index cc1fc6417..e21b07bdd 100644 --- a/src/web/ng/impl/ConnectionHandler.hpp +++ b/src/web/ng/impl/ConnectionHandler.hpp @@ -20,6 +20,7 @@ #pragma once #include "util/StopHelper.hpp" +#include "util/StringHash.hpp" #include "util/Taggable.hpp" #include "util/log/Logger.hpp" #include "util/prometheus/Gauge.hpp" @@ -44,7 +45,6 @@ #include #include #include -#include #include namespace web::ng::impl { @@ -52,20 +52,7 @@ namespace web::ng::impl { class ConnectionHandler { public: using OnDisconnectHook = std::function; - - struct StringHash { - using hash_type = std::hash; - using is_transparent = void; - - std::size_t - operator()(char const* str) const; - std::size_t - operator()(std::string_view str) const; - std::size_t - operator()(std::string const& str) const; - }; - - using TargetToHandlerMap = std::unordered_map>; + using TargetToHandlerMap = std::unordered_map>; private: util::Logger log_{"WebServer"}; diff --git a/src/web/ng/impl/ErrorHandling.cpp b/src/web/ng/impl/ErrorHandling.cpp index f8c244289..6e9a0540f 100644 --- a/src/web/ng/impl/ErrorHandling.cpp +++ b/src/web/ng/impl/ErrorHandling.cpp @@ -93,8 +93,8 @@ ErrorHelper::makeError(rpc::Status const& err) const return Response{http::status::bad_request, "method is empty", rawRequest_}; case rpc::ClioError::RpcCommandNotString: return Response{http::status::bad_request, "method is not string", rawRequest_}; - case rpc::ClioError::RpcParamsUnparseable: - return Response{http::status::bad_request, "params unparseable", rawRequest_}; + case rpc::ClioError::RpcParamsUnparsable: + return Response{http::status::bad_request, "params unparsable", rawRequest_}; // others are not applicable but we want a compilation error next time we add one case rpc::ClioError::RpcUnknownOption: diff --git a/src/web/ng/impl/ServerSslContext.cpp b/src/web/ng/impl/ServerSslContext.cpp index f1f0e3bb3..03c992c7c 100644 --- a/src/web/ng/impl/ServerSslContext.cpp +++ b/src/web/ng/impl/ServerSslContext.cpp @@ -19,7 +19,7 @@ #include "web/ng/impl/ServerSslContext.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include #include diff --git a/src/web/ng/impl/ServerSslContext.hpp b/src/web/ng/impl/ServerSslContext.hpp index be272058d..4368a6a6f 100644 --- a/src/web/ng/impl/ServerSslContext.hpp +++ b/src/web/ng/impl/ServerSslContext.hpp @@ -19,7 +19,7 @@ #pragma once -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include diff --git a/tests/common/migration/TestMigrators.hpp b/tests/common/migration/TestMigrators.hpp index 03fd64cb3..88ea4de39 100644 --- a/tests/common/migration/TestMigrators.hpp +++ b/tests/common/migration/TestMigrators.hpp @@ -18,7 +18,7 @@ //============================================================================== #include "util/MockMigrationBackend.hpp" -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ObjectView.hpp" #include diff --git a/tests/common/util/MockBackend.hpp b/tests/common/util/MockBackend.hpp index 9aab18ee8..a6447d5da 100644 --- a/tests/common/util/MockBackend.hpp +++ b/tests/common/util/MockBackend.hpp @@ -23,10 +23,11 @@ #include "data/DBHelpers.hpp" #include "data/LedgerCache.hpp" #include "data/Types.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include #include +#include #include #include #include @@ -35,6 +36,7 @@ #include #include #include +#include #include struct MockBackend : public BackendInterface { @@ -181,6 +183,9 @@ struct MockBackend : public BackendInterface { (const, override) ); + using FetchClioNodeReturnType = std::expected>, std::string>; + MOCK_METHOD(FetchClioNodeReturnType, fetchClioNodesData, (boost::asio::yield_context yield), (const, override)); + MOCK_METHOD( std::optional, hardFetchLedgerRange, @@ -209,6 +214,8 @@ struct MockBackend : public BackendInterface { MOCK_METHOD(void, writeSuccessor, (std::string && key, std::uint32_t const, std::string&&), (override)); + MOCK_METHOD(void, writeNodeMessage, (boost::uuids::uuid const& uuid, std::string message), (override)); + MOCK_METHOD(void, startWrites, (), (const, override)); MOCK_METHOD(bool, isTooBusy, (), (const, override)); diff --git a/tests/common/util/MockBackendTestFixture.hpp b/tests/common/util/MockBackendTestFixture.hpp index 257774595..4e47e5c32 100644 --- a/tests/common/util/MockBackendTestFixture.hpp +++ b/tests/common/util/MockBackendTestFixture.hpp @@ -22,7 +22,7 @@ #include "data/BackendInterface.hpp" #include "util/LoggerFixtures.hpp" #include "util/MockBackend.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include diff --git a/tests/common/util/MockETLService.hpp b/tests/common/util/MockETLService.hpp index 618e5ab57..f398ad9cd 100644 --- a/tests/common/util/MockETLService.hpp +++ b/tests/common/util/MockETLService.hpp @@ -20,21 +20,21 @@ #pragma once #include "etl/ETLState.hpp" +#include "etlng/ETLServiceInterface.hpp" #include #include #include -#include #include #include -struct MockETLService { - MOCK_METHOD(boost::json::object, getInfo, (), (const)); - MOCK_METHOD(std::chrono::time_point, getLastPublish, (), (const)); - MOCK_METHOD(std::uint32_t, lastPublishAgeSeconds, (), (const)); - MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const)); - MOCK_METHOD(bool, isAmendmentBlocked, (), (const)); - MOCK_METHOD(bool, isCorruptionDetected, (), (const)); - MOCK_METHOD(std::optional, getETLState, (), (const)); +struct MockETLService : etlng::ETLServiceInterface { + MOCK_METHOD(void, run, (), (override)); + MOCK_METHOD(void, stop, (), (override)); + MOCK_METHOD(boost::json::object, getInfo, (), (const, override)); + MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const, override)); + MOCK_METHOD(bool, isAmendmentBlocked, (), (const, override)); + MOCK_METHOD(bool, isCorruptionDetected, (), (const, override)); + MOCK_METHOD(std::optional, getETLState, (), (const, override)); }; diff --git a/tests/common/util/MockETLServiceTestFixture.hpp b/tests/common/util/MockETLServiceTestFixture.hpp index 9009ab1fe..fef65a9ad 100644 --- a/tests/common/util/MockETLServiceTestFixture.hpp +++ b/tests/common/util/MockETLServiceTestFixture.hpp @@ -43,7 +43,7 @@ protected: /** * @brief Fixture with a "nice" ETLService mock. * - * Use @see MockETLServiceTestNaggy during development to get unset call expectation warnings from the embeded mock. + * Use @see MockETLServiceTestNaggy during development to get unset call expectation warnings from the embedded mock. * Once the test is ready and you are happy you can switch to this fixture to mute the warnings. */ using MockETLServiceTest = MockETLServiceTestBase<::testing::NiceMock>; diff --git a/tests/common/util/MockLedgerHeaderCache.hpp b/tests/common/util/MockLedgerHeaderCache.hpp new file mode 100644 index 000000000..6561b8dad --- /dev/null +++ b/tests/common/util/MockLedgerHeaderCache.hpp @@ -0,0 +1,35 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/LedgerHeaderCache.hpp" + +#include +#include + +#include + +struct MockLedgerHeaderCache { + MockLedgerHeaderCache() = default; + using CacheEntry = data::FetchLedgerCache::CacheEntry; + + MOCK_METHOD(void, put, (CacheEntry), ()); + MOCK_METHOD(std::optional, get, (), (const)); +}; diff --git a/tests/common/util/MockLoadBalancer.hpp b/tests/common/util/MockLoadBalancer.hpp index 12bcfea78..b9989252c 100644 --- a/tests/common/util/MockLoadBalancer.hpp +++ b/tests/common/util/MockLoadBalancer.hpp @@ -38,22 +38,6 @@ #include #include -struct MockLoadBalancer { - using RawLedgerObjectType = FakeLedgerObject; - - MOCK_METHOD(void, loadInitialLedger, (std::uint32_t, bool), ()); - MOCK_METHOD(std::optional, fetchLedger, (uint32_t, bool, bool), ()); - MOCK_METHOD(boost::json::value, toJson, (), (const)); - - using ForwardToRippledReturnType = std::expected; - MOCK_METHOD( - ForwardToRippledReturnType, - forwardToRippled, - (boost::json::object const&, std::optional const&, bool, boost::asio::yield_context), - (const) - ); -}; - struct MockNgLoadBalancer : etlng::LoadBalancerInterface { using RawLedgerObjectType = FakeLedgerObject; @@ -78,11 +62,14 @@ struct MockNgLoadBalancer : etlng::LoadBalancerInterface { MOCK_METHOD(boost::json::value, toJson, (), (const, override)); MOCK_METHOD(std::optional, getETLState, (), (noexcept, override)); - using ForwardToRippledReturnType = std::expected; + using ForwardToRippledReturnType = std::expected; MOCK_METHOD( ForwardToRippledReturnType, forwardToRippled, (boost::json::object const&, std::optional const&, bool, boost::asio::yield_context), (override) ); + MOCK_METHOD(void, stop, (boost::asio::yield_context), ()); }; + +using MockLoadBalancer = MockNgLoadBalancer; diff --git a/tests/common/util/MockMigrationBackendFixture.hpp b/tests/common/util/MockMigrationBackendFixture.hpp index f5370534b..217ac2b87 100644 --- a/tests/common/util/MockMigrationBackendFixture.hpp +++ b/tests/common/util/MockMigrationBackendFixture.hpp @@ -21,7 +21,7 @@ #include "util/LoggerFixtures.hpp" #include "util/MockMigrationBackend.hpp" -#include "util/newconfig/ConfigDefinition.hpp" +#include "util/config/ConfigDefinition.hpp" #include diff --git a/tests/common/util/MockPrometheus.hpp b/tests/common/util/MockPrometheus.hpp index a2d53883a..235cf792b 100644 --- a/tests/common/util/MockPrometheus.hpp +++ b/tests/common/util/MockPrometheus.hpp @@ -21,9 +21,9 @@ #include "util/Assert.hpp" #include "util/Concepts.hpp" -#include "util/newconfig/ConfigDefinition.hpp" -#include "util/newconfig/ConfigValue.hpp" -#include "util/newconfig/Types.hpp" +#include "util/config/ConfigDefinition.hpp" +#include "util/config/ConfigValue.hpp" +#include "util/config/Types.hpp" #include "util/prometheus/Bool.hpp" #include "util/prometheus/Counter.hpp" #include "util/prometheus/Gauge.hpp" diff --git a/tests/common/util/MockSource.hpp b/tests/common/util/MockSource.hpp index 58984d485..15542ff82 100644 --- a/tests/common/util/MockSource.hpp +++ b/tests/common/util/MockSource.hpp @@ -23,7 +23,7 @@ #include "etl/Source.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" -#include "util/newconfig/ObjectView.hpp" +#include "util/config/ObjectView.hpp" #include #include @@ -60,7 +60,7 @@ struct MockSource : etl::SourceBase { (uint32_t, bool, bool), (override) ); - MOCK_METHOD((std::pair, bool>), loadInitialLedger, (uint32_t, uint32_t, bool), (override)); + MOCK_METHOD((std::pair, bool>), loadInitialLedger, (uint32_t, uint32_t), (override)); using ForwardToRippledReturnType = std::expected; MOCK_METHOD( @@ -132,9 +132,9 @@ public: } std::pair, bool> - loadInitialLedger(uint32_t sequence, uint32_t maxLedger, bool getObjects) override + loadInitialLedger(uint32_t sequence, uint32_t maxLedger) override { - return mock_->loadInitialLedger(sequence, maxLedger, getObjects); + return mock_->loadInitialLedger(sequence, maxLedger); } std::expected diff --git a/tests/common/util/MockSourceNg.hpp b/tests/common/util/MockSourceNg.hpp new file mode 100644 index 000000000..6c59c5136 --- /dev/null +++ b/tests/common/util/MockSourceNg.hpp @@ -0,0 +1,245 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2023, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== +#pragma once + +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/Source.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/config/ObjectView.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct MockSourceNg : etlng::SourceBase { + MOCK_METHOD(void, run, (), (override)); + MOCK_METHOD(void, stop, (boost::asio::yield_context), (override)); + MOCK_METHOD(bool, isConnected, (), (const, override)); + MOCK_METHOD(void, setForwarding, (bool), (override)); + MOCK_METHOD(boost::json::object, toJson, (), (const, override)); + MOCK_METHOD(std::string, toString, (), (const, override)); + MOCK_METHOD(bool, hasLedger, (uint32_t), (const, override)); + MOCK_METHOD( + (std::pair), + fetchLedger, + (uint32_t, bool, bool), + (override) + ); + MOCK_METHOD( + (std::pair, bool>), + loadInitialLedger, + (uint32_t, uint32_t, etlng::InitialLoadObserverInterface&), + (override) + ); + + using ForwardToRippledReturnType = std::expected; + MOCK_METHOD( + ForwardToRippledReturnType, + forwardToRippled, + (boost::json::object const&, std::optional const&, std::string_view, boost::asio::yield_context), + (const, override) + ); +}; + +template