From 52ccf27aa3fd18e998b53b3f16df911a611f4406 Mon Sep 17 00:00:00 2001 From: tequ Date: Wed, 10 Dec 2025 18:32:03 +0900 Subject: [PATCH 1/5] Hook API Refactor1: whitelist api at `Enum.h` (#605) --- src/ripple/app/hook/Enum.h | 180 ++++++++++++++++++++----------------- 1 file changed, 100 insertions(+), 80 deletions(-) diff --git a/src/ripple/app/hook/Enum.h b/src/ripple/app/hook/Enum.h index 7d2e4613e..9728b1040 100644 --- a/src/ripple/app/hook/Enum.h +++ b/src/ripple/app/hook/Enum.h @@ -367,90 +367,110 @@ const uint8_t max_emit = 255; const uint8_t max_params = 16; const double fee_base_multiplier = 1.1f; +#define I32 0x7FU +#define I64 0x7EU + +#define HOOK_WRAP_PARAMS(...) __VA_ARGS__ +#define HOOK_API_DEFINITION(RETURN_TYPE, FUNCTION_NAME, PARAMS_TUPLE) \ + { \ +#FUNCTION_NAME, \ + { \ + RETURN_TYPE, HOOK_WRAP_PARAMS PARAMS_TUPLE \ + } \ + } + +using APIWhitelist = std::map>; + // RH NOTE: Find descriptions of api functions in ./impl/applyHook.cpp and // hookapi.h (include for hooks) this is a map of the api name to its return // code (vec[0] and its parameters vec[>0]) as wasm type codes -static const std::map> import_whitelist{ - {"_g", {0x7FU, 0x7FU, 0x7FU}}, - {"accept", {0x7EU, 0x7FU, 0x7FU, 0x7EU}}, - {"rollback", {0x7EU, 0x7FU, 0x7FU, 0x7EU}}, - {"util_raddr", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"util_accid", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"util_verify", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"util_sha512h", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"util_keylet", - {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"sto_validate", {0x7EU, 0x7FU, 0x7FU}}, - {"sto_subfield", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"sto_subarray", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"sto_emplace", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"sto_erase", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"etxn_burden", {0x7EU}}, - {"etxn_details", {0x7EU, 0x7FU, 0x7FU}}, - {"etxn_fee_base", {0x7EU, 0x7FU, 0x7FU}}, - {"etxn_reserve", {0x7EU, 0x7FU}}, - {"etxn_generation", {0x7EU}}, - {"etxn_nonce", {0x7EU, 0x7FU, 0x7FU}}, - {"emit", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"float_set", {0x7EU, 0x7FU, 0x7EU}}, - {"float_multiply", {0x7EU, 0x7EU, 0x7EU}}, - {"float_mulratio", {0x7EU, 0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"float_negate", {0x7EU, 0x7EU}}, - {"float_compare", {0x7EU, 0x7EU, 0x7EU, 0x7FU}}, - {"float_sum", {0x7EU, 0x7EU, 0x7EU}}, - {"float_sto", - {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7EU, 0x7FU}}, - {"float_sto_set", {0x7EU, 0x7FU, 0x7FU}}, - {"float_invert", {0x7EU, 0x7EU}}, - {"float_divide", {0x7EU, 0x7EU, 0x7EU}}, - {"float_one", {0x7EU}}, - {"float_mantissa", {0x7EU, 0x7EU}}, - {"float_sign", {0x7EU, 0x7EU}}, - {"float_int", {0x7EU, 0x7EU, 0x7FU, 0x7FU}}, - {"float_log", {0x7EU, 0x7EU}}, - {"float_root", {0x7EU, 0x7EU, 0x7FU}}, - {"fee_base", {0x7EU}}, - {"ledger_seq", {0x7EU}}, - {"ledger_last_time", {0x7EU}}, - {"ledger_last_hash", {0x7EU, 0x7FU, 0x7FU}}, - {"ledger_nonce", {0x7EU, 0x7FU, 0x7FU}}, - {"ledger_keylet", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"hook_account", {0x7EU, 0x7FU, 0x7FU}}, - {"hook_hash", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"hook_param_set", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"hook_param", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"hook_again", {0x7EU}}, - {"hook_skip", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"hook_pos", {0x7EU}}, - {"slot", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"slot_clear", {0x7EU, 0x7FU}}, - {"slot_count", {0x7EU, 0x7FU}}, - {"slot_set", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"slot_size", {0x7EU, 0x7FU}}, - {"slot_subarray", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"slot_subfield", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"slot_type", {0x7EU, 0x7FU, 0x7FU}}, - {"slot_float", {0x7EU, 0x7FU}}, - {"state_set", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"state_foreign_set", - {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"state", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"state_foreign", - {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"trace", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"trace_num", {0x7EU, 0x7FU, 0x7FU, 0x7EU}}, - {"trace_float", {0x7EU, 0x7FU, 0x7FU, 0x7EU}}, - {"otxn_burden", {0x7EU}}, - {"otxn_field", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"otxn_generation", {0x7EU}}, - {"otxn_id", {0x7EU, 0x7FU, 0x7FU, 0x7FU}}, - {"otxn_type", {0x7EU}}, - {"otxn_slot", {0x7EU, 0x7FU}}, - {"otxn_param", {0x7EU, 0x7FU, 0x7FU, 0x7FU, 0x7FU}}, - {"meta_slot", {0x7EU, 0x7FU}}}; +static const APIWhitelist import_whitelist{ + // clang-format off + HOOK_API_DEFINITION(I32, _g, (I32, I32)), + HOOK_API_DEFINITION(I64, accept, (I32, I32, I64)), + HOOK_API_DEFINITION(I64, rollback, (I32, I32, I64)), + HOOK_API_DEFINITION(I64, util_raddr, (I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, util_accid, (I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, util_verify, (I32, I32, I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, util_sha512h, (I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, util_keylet, (I32, I32, I32, I32, I32, I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, sto_validate, (I32, I32)), + HOOK_API_DEFINITION(I64, sto_subfield, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, sto_subarray, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, sto_emplace, (I32, I32, I32, I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, sto_erase, (I32, I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, etxn_burden, ()), + HOOK_API_DEFINITION(I64, etxn_details, (I32, I32)), + HOOK_API_DEFINITION(I64, etxn_fee_base, (I32, I32)), + HOOK_API_DEFINITION(I64, etxn_reserve, (I32)), + HOOK_API_DEFINITION(I64, etxn_generation, ()), + HOOK_API_DEFINITION(I64, etxn_nonce, (I32, I32)), + HOOK_API_DEFINITION(I64, emit, (I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, float_set, (I32, I64)), + HOOK_API_DEFINITION(I64, float_multiply, (I64, I64)), + HOOK_API_DEFINITION(I64, float_mulratio, (I64, I32, I32, I32)), + HOOK_API_DEFINITION(I64, float_negate, (I64)), + HOOK_API_DEFINITION(I64, float_compare, (I64, I64, I32)), + HOOK_API_DEFINITION(I64, float_sum, (I64, I64)), + HOOK_API_DEFINITION(I64, float_sto, (I32, I32, I32, I32, I32, I32, I64, I32)), + HOOK_API_DEFINITION(I64, float_sto_set, (I32, I32)), + HOOK_API_DEFINITION(I64, float_invert, (I64)), + HOOK_API_DEFINITION(I64, float_divide, (I64, I64)), + HOOK_API_DEFINITION(I64, float_one, ()), + HOOK_API_DEFINITION(I64, float_mantissa, (I64)), + HOOK_API_DEFINITION(I64, float_sign, (I64)), + HOOK_API_DEFINITION(I64, float_int, (I64, I32, I32)), + HOOK_API_DEFINITION(I64, float_log, (I64)), + HOOK_API_DEFINITION(I64, float_root, (I64, I32)), + HOOK_API_DEFINITION(I64, fee_base, ()), + HOOK_API_DEFINITION(I64, ledger_seq, ()), + HOOK_API_DEFINITION(I64, ledger_last_time, ()), + HOOK_API_DEFINITION(I64, ledger_last_hash, (I32, I32)), + HOOK_API_DEFINITION(I64, ledger_nonce, (I32, I32)), + HOOK_API_DEFINITION(I64, ledger_keylet, (I32, I32, I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, hook_account, (I32, I32)), + HOOK_API_DEFINITION(I64, hook_hash, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, hook_param_set, (I32, I32, I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, hook_param, (I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, hook_again, ()), + HOOK_API_DEFINITION(I64, hook_skip, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, hook_pos, ()), + HOOK_API_DEFINITION(I64, slot, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, slot_clear, (I32)), + HOOK_API_DEFINITION(I64, slot_count, (I32)), + HOOK_API_DEFINITION(I64, slot_set, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, slot_size, (I32)), + HOOK_API_DEFINITION(I64, slot_subarray, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, slot_subfield, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, slot_type, (I32, I32)), + HOOK_API_DEFINITION(I64, slot_float, (I32)), + HOOK_API_DEFINITION(I64, state_set, (I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, state_foreign_set, (I32, I32, I32, I32, I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, state, (I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, state_foreign, (I32, I32, I32, I32, I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, trace, (I32, I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, trace_num, (I32, I32, I64)), + HOOK_API_DEFINITION(I64, trace_float, (I32, I32, I64)), + HOOK_API_DEFINITION(I64, otxn_burden, ()), + HOOK_API_DEFINITION(I64, otxn_field, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, otxn_generation, ()), + HOOK_API_DEFINITION(I64, otxn_id, (I32, I32, I32)), + HOOK_API_DEFINITION(I64, otxn_type, ()), + HOOK_API_DEFINITION(I64, otxn_slot, (I32)), + HOOK_API_DEFINITION(I64, otxn_param, (I32, I32, I32, I32)), + HOOK_API_DEFINITION(I64, meta_slot, (I32)), + // clang-format on +}; // featureHooks1 -static const std::map> import_whitelist_1{ - {"xpop_slot", {0x7EU, 0x7FU, 0x7FU}}}; +static const APIWhitelist import_whitelist_1{ + // clang-format off + HOOK_API_DEFINITION(I64, xpop_slot, (I32, I32)), + // clang-format on +}; + +#undef HOOK_API_DEFINITION +#undef I32 +#undef I64 }; // namespace hook_api #endif From 374b361daa295570a5c8599fda77e863f442ddb9 Mon Sep 17 00:00:00 2001 From: tequ Date: Tue, 16 Dec 2025 13:16:36 +0900 Subject: [PATCH 2/5] Use Self hosted runner (#639) --- .github/actions/xahau-ga-build/action.yml | 17 ++++-- .../actions/xahau-ga-dependencies/action.yml | 19 +++++-- .github/workflows/build-in-docker.yml | 6 +-- .github/workflows/xahau-ga-nix.yml | 52 +++++++++++++------ 4 files changed, 67 insertions(+), 27 deletions(-) diff --git a/.github/actions/xahau-ga-build/action.yml b/.github/actions/xahau-ga-build/action.yml index c522fff79..a7bd7ace4 100644 --- a/.github/actions/xahau-ga-build/action.yml +++ b/.github/actions/xahau-ga-build/action.yml @@ -28,6 +28,10 @@ inputs: description: 'Cache version for invalidation' required: false default: '1' + gha_cache_enabled: + description: 'Whether to use actions/cache (disable for self-hosted with volume mounts)' + required: false + default: 'true' ccache_enabled: description: 'Whether to use ccache' required: false @@ -72,7 +76,7 @@ runs: echo "name=${SAFE_BRANCH}" >> $GITHUB_OUTPUT - name: Restore ccache directory for main branch - if: inputs.ccache_enabled == 'true' + if: inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' id: ccache-restore uses: ./.github/actions/xahau-ga-cache-restore with: @@ -84,7 +88,7 @@ runs: cache-type: ccache-main - name: Restore ccache directory for current branch - if: inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch + if: inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch id: ccache-restore-current-branch uses: ./.github/actions/xahau-ga-cache-restore with: @@ -103,6 +107,11 @@ runs: # Create cache directories mkdir -p ~/.ccache-main ~/.ccache-current + # Keep config separate from cache_dir so configs aren't swapped when CCACHE_DIR changes between steps + mkdir -p ~/.config/ccache + export CCACHE_CONFIGPATH="$HOME/.config/ccache/ccache.conf" + echo "CCACHE_CONFIGPATH=$CCACHE_CONFIGPATH" >> $GITHUB_ENV + # Configure ccache settings AFTER cache restore (prevents stale cached config) ccache --set-config=max_size=${{ inputs.ccache_max_size }} ccache --set-config=hash_dir=${{ inputs.ccache_hash_dir }} @@ -237,14 +246,14 @@ runs: run: ccache -s - name: Save ccache directory for main branch - if: success() && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name == inputs.main_branch + if: success() && inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name == inputs.main_branch uses: actions/cache/save@v4 with: path: ~/.ccache-main key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - name: Save ccache directory for current branch - if: success() && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch + if: success() && inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch uses: actions/cache/save@v4 with: path: ~/.ccache-current diff --git a/.github/actions/xahau-ga-dependencies/action.yml b/.github/actions/xahau-ga-dependencies/action.yml index 1e0c49945..6a491f9f2 100644 --- a/.github/actions/xahau-ga-dependencies/action.yml +++ b/.github/actions/xahau-ga-dependencies/action.yml @@ -17,8 +17,8 @@ inputs: description: 'Cache version for invalidation' required: false default: '1' - cache_enabled: - description: 'Whether to use caching' + gha_cache_enabled: + description: 'Whether to use actions/cache (disable for self-hosted with volume mounts)' required: false default: 'true' main_branch: @@ -64,7 +64,7 @@ runs: using: 'composite' steps: - name: Restore Conan cache - if: inputs.cache_enabled == 'true' + if: inputs.gha_cache_enabled == 'true' id: cache-restore-conan uses: ./.github/actions/xahau-ga-cache-restore with: @@ -76,6 +76,17 @@ runs: ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}- cache-type: Conan + - name: Configure Conan cache paths + if: inputs.gha_cache_enabled == 'false' + shell: bash + # For self-hosted runners, register cache paths to be used as volumes + # This allows the cache to be shared between containers + run: | + mkdir -p /.conan-cache/conan2 /.conan-cache/conan2_download /.conan-cache/conan2_sources + echo 'core.cache:storage_path=/.conan-cache/conan2' > ~/.conan2/global.conf + echo 'core.download:download_cache=/.conan-cache/conan2_download' >> ~/.conan2/global.conf + echo 'core.sources:download_cache=/.conan-cache/conan2_sources' >> ~/.conan2/global.conf + - name: Configure Conan shell: bash run: | @@ -152,7 +163,7 @@ runs: .. - name: Save Conan cache - if: success() && inputs.cache_enabled == 'true' && steps.cache-restore-conan.outputs.cache-hit != 'true' + if: success() && inputs.gha_cache_enabled == 'true' && steps.cache-restore-conan.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: ~/.conan2 diff --git a/.github/workflows/build-in-docker.yml b/.github/workflows/build-in-docker.yml index e57dda8b4..f46adfde9 100644 --- a/.github/workflows/build-in-docker.yml +++ b/.github/workflows/build-in-docker.yml @@ -33,7 +33,7 @@ jobs: fetch-depth: 2 # Only get the last 2 commits, to avoid fetching all history build: - runs-on: [self-hosted, vanity] + runs-on: [self-hosted, xahaud-build] needs: [checkout] defaults: run: @@ -74,7 +74,7 @@ jobs: fi tests: - runs-on: [self-hosted, vanity] + runs-on: [self-hosted, xahaud-build] needs: [build, checkout] defaults: run: @@ -84,7 +84,7 @@ jobs: run: /bin/bash docker-unit-tests.sh cleanup: - runs-on: [self-hosted, vanity] + runs-on: [self-hosted, xahaud-build] needs: [tests, checkout] if: always() steps: diff --git a/.github/workflows/xahau-ga-nix.yml b/.github/workflows/xahau-ga-nix.yml index 55c1d69f9..c63cbe1fe 100644 --- a/.github/workflows/xahau-ga-nix.yml +++ b/.github/workflows/xahau-ga-nix.yml @@ -14,7 +14,7 @@ concurrency: jobs: matrix-setup: - runs-on: ubuntu-latest + runs-on: [self-hosted, generic, 20.04] container: python:3-slim outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -176,7 +176,16 @@ jobs: build: needs: matrix-setup - runs-on: ubuntu-latest + runs-on: [self-hosted, generic, 20.04] + container: + image: ubuntu:24.04 + volumes: + - /home/runner/.conan-cache:/.conan-cache + - /home/runner/.ccache-main:/github/home/.ccache-main + - /home/runner/.ccache-current:/github/home/.ccache-current + defaults: + run: + shell: bash outputs: artifact_name: ${{ steps.set-artifact-name.outputs.artifact_name }} strategy: @@ -191,23 +200,22 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Get commit message - id: get-commit-message - uses: ./.github/actions/xahau-ga-get-commit-message - with: - event-name: ${{ github.event_name }} - head-commit-message: ${{ github.event.head_commit.message }} - pr-head-sha: ${{ github.event.pull_request.head.sha }} - - name: Install build dependencies run: | - sudo apt-get update - sudo apt-get install -y ninja-build ${{ matrix.cc }} ${{ matrix.cxx }} ccache + apt-get update + apt-get install -y software-properties-common + add-apt-repository ppa:ubuntu-toolchain-r/test -y + apt-get update + apt-get install -y python3 python-is-python3 pipx + pipx ensurepath + apt-get install -y cmake ninja-build ${{ matrix.cc }} ${{ matrix.cxx }} ccache + apt-get install -y perl # for openssl build + apt-get install -y libsqlite3-dev # for xahaud build # Install the specific GCC version needed for Clang if [ -n "${{ matrix.clang_gcc_toolchain }}" ]; then echo "=== Installing GCC ${{ matrix.clang_gcc_toolchain }} for Clang ===" - sudo apt-get install -y gcc-${{ matrix.clang_gcc_toolchain }} g++-${{ matrix.clang_gcc_toolchain }} libstdc++-${{ matrix.clang_gcc_toolchain }}-dev + apt-get install -y gcc-${{ matrix.clang_gcc_toolchain }} g++-${{ matrix.clang_gcc_toolchain }} libstdc++-${{ matrix.clang_gcc_toolchain }}-dev echo "=== GCC versions available after installation ===" ls -la /usr/lib/gcc/x86_64-linux-gnu/ | grep -E "^d" @@ -238,7 +246,7 @@ jobs: echo "Hiding GCC $version -> renaming to $counter (will be seen as GCC version $counter)" # Safety check: ensure target doesn't already exist if [ ! -e "/usr/lib/gcc/x86_64-linux-gnu/$counter" ]; then - sudo mv "$dir" "/usr/lib/gcc/x86_64-linux-gnu/$counter" + mv "$dir" "/usr/lib/gcc/x86_64-linux-gnu/$counter" else echo "ERROR: Cannot rename GCC $version - /usr/lib/gcc/x86_64-linux-gnu/$counter already exists" exit 1 @@ -262,11 +270,12 @@ jobs: # Install libc++ dev packages if using libc++ (not needed for libstdc++) if [ "${{ matrix.stdlib }}" = "libcxx" ]; then - sudo apt-get install -y libc++-${{ matrix.compiler_version }}-dev libc++abi-${{ matrix.compiler_version }}-dev + apt-get install -y libc++-${{ matrix.compiler_version }}-dev libc++abi-${{ matrix.compiler_version }}-dev fi # Install Conan 2 - pip install --upgrade "conan>=2.0,<3" + pipx install "conan>=2.0,<3" + echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Check environment run: | @@ -280,6 +289,14 @@ jobs: echo "---- Full Environment ----" env + - name: Get commit message + id: get-commit-message + uses: ./.github/actions/xahau-ga-get-commit-message + with: + event-name: ${{ github.event_name }} + head-commit-message: ${{ github.event.head_commit.message }} + pr-head-sha: ${{ github.event.pull_request.head.sha }} + - name: Install dependencies uses: ./.github/actions/xahau-ga-dependencies with: @@ -293,6 +310,7 @@ jobs: cc: ${{ matrix.cc }} cxx: ${{ matrix.cxx }} stdlib: ${{ matrix.stdlib }} + gha_cache_enabled: 'false' # Disable caching for self hosted runner - name: Build uses: ./.github/actions/xahau-ga-build @@ -307,6 +325,8 @@ jobs: main_branch: ${{ env.MAIN_BRANCH_NAME }} stdlib: ${{ matrix.stdlib }} clang_gcc_toolchain: ${{ matrix.clang_gcc_toolchain || '' }} + gha_cache_enabled: 'false' # Disable caching for self hosted runner + ccache_max_size: '100G' - name: Set artifact name id: set-artifact-name From f731bcfeba42a8585c60ea4956871c1625895e33 Mon Sep 17 00:00:00 2001 From: tequ Date: Tue, 16 Dec 2025 13:45:45 +0900 Subject: [PATCH 3/5] Increase ccache size from 10G to 100G in release-builder.sh for improved build performance (#643) --- release-builder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-builder.sh b/release-builder.sh index 672e68042..e3d55b86a 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -192,7 +192,7 @@ ENV PATH=/usr/local/bin:$PATH # Configure ccache and Conan 2 # NOTE: Using echo commands instead of heredocs because heredocs in Docker RUN commands are finnicky -RUN /hbb_exe/activate-exec bash -c "ccache -M 10G && \ +RUN /hbb_exe/activate-exec bash -c "ccache -M 100G && \ ccache -o cache_dir=/cache/ccache && \ ccache -o compiler_check=content && \ mkdir -p ~/.conan2 /cache/conan2 /cache/conan2_download /cache/conan2_sources && \ From 960f87857e210d88e631394b4fc75da3bbcfb57b Mon Sep 17 00:00:00 2001 From: tequ Date: Wed, 17 Dec 2025 08:43:25 +0900 Subject: [PATCH 4/5] Self hosted macos runner (#652) --- .github/actions/xahau-ga-build/action.yml | 63 +++------------- .../actions/xahau-ga-dependencies/action.yml | 31 +++----- .github/workflows/xahau-ga-macos.yml | 71 +++++++------------ .github/workflows/xahau-ga-nix.yml | 4 +- 4 files changed, 42 insertions(+), 127 deletions(-) diff --git a/.github/actions/xahau-ga-build/action.yml b/.github/actions/xahau-ga-build/action.yml index a7bd7ace4..247d3af85 100644 --- a/.github/actions/xahau-ga-build/action.yml +++ b/.github/actions/xahau-ga-build/action.yml @@ -75,37 +75,17 @@ runs: SAFE_BRANCH=$(echo "${{ github.ref_name }}" | tr -c 'a-zA-Z0-9_.-' '-') echo "name=${SAFE_BRANCH}" >> $GITHUB_OUTPUT - - name: Restore ccache directory for main branch - if: inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' - id: ccache-restore - uses: ./.github/actions/xahau-ga-cache-restore - with: - path: ~/.ccache-main - key: ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-${{ inputs.main_branch }} - restore-keys: | - ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}- - ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}- - cache-type: ccache-main - - - name: Restore ccache directory for current branch - if: inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch - id: ccache-restore-current-branch - uses: ./.github/actions/xahau-ga-cache-restore - with: - path: ~/.ccache-current - key: ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-${{ steps.safe-branch.outputs.name }} - restore-keys: | - ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-${{ inputs.main_branch }} - ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}- - ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}- - cache-type: ccache-current - - name: Configure ccache if: inputs.ccache_enabled == 'true' shell: bash run: | # Create cache directories - mkdir -p ~/.ccache-main ~/.ccache-current + mkdir -p ~/.ccache-cache + + # Keep config separate from cache_dir so configs aren't swapped when CCACHE_DIR changes between steps + mkdir -p ~/.config/ccache + export CCACHE_CONFIGPATH="$HOME/.config/ccache/ccache.conf" + echo "CCACHE_CONFIGPATH=$CCACHE_CONFIGPATH" >> $GITHUB_ENV # Keep config separate from cache_dir so configs aren't swapped when CCACHE_DIR changes between steps mkdir -p ~/.config/ccache @@ -116,20 +96,9 @@ runs: ccache --set-config=max_size=${{ inputs.ccache_max_size }} ccache --set-config=hash_dir=${{ inputs.ccache_hash_dir }} ccache --set-config=compiler_check=${{ inputs.ccache_compiler_check }} - - # Determine if we're on the main branch - if [ "${{ steps.safe-branch.outputs.name }}" = "${{ inputs.main_branch }}" ]; then - # Main branch: use main branch cache only - ccache --set-config=cache_dir="$HOME/.ccache-main" - echo "CCACHE_DIR=$HOME/.ccache-main" >> $GITHUB_ENV - echo "📦 Main branch: using ~/.ccache-main" - else - # Feature branch: use current branch cache with main as secondary (read-only fallback) - ccache --set-config=cache_dir="$HOME/.ccache-current" - ccache --set-config=secondary_storage="file:$HOME/.ccache-main" - echo "CCACHE_DIR=$HOME/.ccache-current" >> $GITHUB_ENV - echo "📦 Feature branch: using ~/.ccache-current with ~/.ccache-main as secondary" - fi + ccache --set-config=cache_dir="$HOME/.ccache-cache" + echo "CCACHE_DIR=$HOME/.ccache-cache" >> $GITHUB_ENV + echo "📦 using ~/.ccache-cache as ccache cache directory" # Print config for verification echo "=== ccache configuration ===" @@ -244,17 +213,3 @@ runs: if: inputs.ccache_enabled == 'true' shell: bash run: ccache -s - - - name: Save ccache directory for main branch - if: success() && inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name == inputs.main_branch - uses: actions/cache/save@v4 - with: - path: ~/.ccache-main - key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - - - name: Save ccache directory for current branch - if: success() && inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch - uses: actions/cache/save@v4 - with: - path: ~/.ccache-current - key: ${{ steps.ccache-restore-current-branch.outputs.cache-primary-key }} diff --git a/.github/actions/xahau-ga-dependencies/action.yml b/.github/actions/xahau-ga-dependencies/action.yml index 6a491f9f2..376448ff6 100644 --- a/.github/actions/xahau-ga-dependencies/action.yml +++ b/.github/actions/xahau-ga-dependencies/action.yml @@ -17,10 +17,6 @@ inputs: description: 'Cache version for invalidation' required: false default: '1' - gha_cache_enabled: - description: 'Whether to use actions/cache (disable for self-hosted with volume mounts)' - required: false - default: 'true' main_branch: description: 'Main branch name for restore keys' required: false @@ -63,18 +59,14 @@ outputs: runs: using: 'composite' steps: - - name: Restore Conan cache - if: inputs.gha_cache_enabled == 'true' - id: cache-restore-conan - uses: ./.github/actions/xahau-ga-cache-restore - with: - path: ~/.conan2 - # Note: compiler-id format is compiler-version-stdlib[-gccversion] - key: ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ hashFiles('**/conanfile.py') }}-${{ inputs.configuration }} - restore-keys: | - ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ hashFiles('**/conanfile.py') }}- - ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}- - cache-type: Conan + - name: Configure Conan cache paths + if: inputs.os == 'Linux' + shell: bash + run: | + mkdir -p /.conan-cache/conan2 /.conan-cache/conan2_download /.conan-cache/conan2_sources + echo 'core.cache:storage_path=/.conan-cache/conan2' > ~/.conan2/global.conf + echo 'core.download:download_cache=/.conan-cache/conan2_download' >> ~/.conan2/global.conf + echo 'core.sources:download_cache=/.conan-cache/conan2_sources' >> ~/.conan2/global.conf - name: Configure Conan cache paths if: inputs.gha_cache_enabled == 'false' @@ -161,10 +153,3 @@ runs: --build missing \ --settings build_type=${{ inputs.configuration }} \ .. - - - name: Save Conan cache - if: success() && inputs.gha_cache_enabled == 'true' && steps.cache-restore-conan.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: ~/.conan2 - key: ${{ steps.cache-restore-conan.outputs.cache-primary-key }} diff --git a/.github/workflows/xahau-ga-macos.yml b/.github/workflows/xahau-ga-macos.yml index 4ade81978..8615a773a 100644 --- a/.github/workflows/xahau-ga-macos.yml +++ b/.github/workflows/xahau-ga-macos.yml @@ -20,7 +20,7 @@ jobs: - Ninja configuration: - Debug - runs-on: macos-15 + runs-on: [self-hosted, macOS] env: build_dir: .build # Bump this number to invalidate all caches globally. @@ -30,61 +30,29 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Get commit message - id: get-commit-message - uses: ./.github/actions/xahau-ga-get-commit-message - with: - event-name: ${{ github.event_name }} - head-commit-message: ${{ github.event.head_commit.message }} - pr-head-sha: ${{ github.event.pull_request.head.sha }} - - - name: Install Conan + - name: Add Homebrew to PATH run: | - brew install conan - # Verify Conan 2 is installed - conan --version + echo "/opt/homebrew/bin" >> "$GITHUB_PATH" + echo "/opt/homebrew/sbin" >> "$GITHUB_PATH" - name: Install Coreutils run: | brew install coreutils echo "Num proc: $(nproc)" - - name: Install Ninja - if: matrix.generator == 'Ninja' - run: brew install ninja + # To isolate environments for each Runner, instead of installing globally with brew, + # use mise to isolate environments for each Runner directory. + - name: Setup toolchain (mise) + uses: jdx/mise-action@v2 + with: + install: true - - name: Install Python + - name: Install tools via mise run: | - if which python3 > /dev/null 2>&1; then - echo "Python 3 executable exists" - python3 --version - else - brew install python@3.12 - fi - # Create 'python' symlink if it doesn't exist (for tools expecting 'python') - if ! which python > /dev/null 2>&1; then - sudo ln -sf $(which python3) /usr/local/bin/python - fi - - - name: Install CMake - run: | - # Install CMake 3.x to match local dev environments - # With Conan 2 and the policy args passed to CMake, newer versions - # can have issues with dependencies that require cmake_minimum_required < 3.5 - brew uninstall cmake --ignore-dependencies 2>/dev/null || true - - # Download and install CMake 3.31.7 directly - curl -L https://github.com/Kitware/CMake/releases/download/v3.31.7/cmake-3.31.7-macos-universal.tar.gz -o cmake.tar.gz - tar -xzf cmake.tar.gz - - # Move the entire CMake.app to /Applications - sudo mv cmake-3.31.7-macos-universal/CMake.app /Applications/ - - echo "/Applications/CMake.app/Contents/bin" >> $GITHUB_PATH - /Applications/CMake.app/Contents/bin/cmake --version - - - name: Install ccache - run: brew install ccache + mise install + mise use cmake@3.23.1 python@3.12 pipx@latest conan@2 ninja@latest ccache@latest + mise reshim + echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" - name: Check environment run: | @@ -98,6 +66,14 @@ jobs: echo "---- Full Environment ----" env + - name: Get commit message + id: get-commit-message + uses: ./.github/actions/xahau-ga-get-commit-message + with: + event-name: ${{ github.event_name }} + head-commit-message: ${{ github.event.head_commit.message }} + pr-head-sha: ${{ github.event.pull_request.head.sha }} + - name: Detect compiler version id: detect-compiler run: | @@ -129,6 +105,7 @@ jobs: cache_version: ${{ env.CACHE_VERSION }} main_branch: ${{ env.MAIN_BRANCH_NAME }} stdlib: libcxx + ccache_max_size: '100G' - name: Test run: | diff --git a/.github/workflows/xahau-ga-nix.yml b/.github/workflows/xahau-ga-nix.yml index c63cbe1fe..6477d2cbe 100644 --- a/.github/workflows/xahau-ga-nix.yml +++ b/.github/workflows/xahau-ga-nix.yml @@ -181,8 +181,7 @@ jobs: image: ubuntu:24.04 volumes: - /home/runner/.conan-cache:/.conan-cache - - /home/runner/.ccache-main:/github/home/.ccache-main - - /home/runner/.ccache-current:/github/home/.ccache-current + - /home/runner/.ccache-cache:/github/home/.ccache-cache defaults: run: shell: bash @@ -325,7 +324,6 @@ jobs: main_branch: ${{ env.MAIN_BRANCH_NAME }} stdlib: ${{ matrix.stdlib }} clang_gcc_toolchain: ${{ matrix.clang_gcc_toolchain || '' }} - gha_cache_enabled: 'false' # Disable caching for self hosted runner ccache_max_size: '100G' - name: Set artifact name From 5a118a4e2bc1a04abcdec5f6cfb2c1fa8530696c Mon Sep 17 00:00:00 2001 From: Niq Dudfield Date: Wed, 17 Dec 2025 06:45:41 +0700 Subject: [PATCH 5/5] fix(logs): formatting fixes, color handling, and debug build defaults (#607) --- Builds/CMake/RippledCore.cmake | 24 +++++------ CMakeLists.txt | 19 +++----- src/ripple/basics/impl/Log.cpp | 6 ++- src/ripple/beast/utility/EnhancedLogging.h | 8 ++++ .../utility/src/beast_EnhancedLogging.cpp | 4 ++ .../beast/utility/src/beast_Journal.cpp | 43 ++++++++++++++++--- 6 files changed, 69 insertions(+), 35 deletions(-) diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake index 683032366..74ff58f60 100644 --- a/Builds/CMake/RippledCore.cmake +++ b/Builds/CMake/RippledCore.cmake @@ -48,13 +48,9 @@ target_sources (xrpl_core PRIVATE src/ripple/beast/net/impl/IPAddressV6.cpp src/ripple/beast/net/impl/IPEndpoint.cpp src/ripple/beast/utility/src/beast_Journal.cpp - src/ripple/beast/utility/src/beast_PropertyStream.cpp) - -# Conditionally add enhanced logging source when BEAST_ENHANCED_LOGGING is enabled -if(DEFINED BEAST_ENHANCED_LOGGING AND BEAST_ENHANCED_LOGGING) - target_sources(xrpl_core PRIVATE - src/ripple/beast/utility/src/beast_EnhancedLogging.cpp) -endif() + src/ripple/beast/utility/src/beast_PropertyStream.cpp + # Enhanced logging - compiles to empty when BEAST_ENHANCED_LOGGING is not defined + src/ripple/beast/utility/src/beast_EnhancedLogging.cpp) #[===============================[ core sources @@ -162,12 +158,16 @@ target_link_libraries (xrpl_core date::date Ripple::opts) -# Link date-tz library when enhanced logging is enabled -if(DEFINED BEAST_ENHANCED_LOGGING AND BEAST_ENHANCED_LOGGING) - if(TARGET date::date-tz) - target_link_libraries(xrpl_core PUBLIC date::date-tz) - endif() +# date-tz for enhanced logging (always linked, code is #ifdef guarded) +if(TARGET date::date-tz) + target_link_libraries(xrpl_core PUBLIC date::date-tz) endif() + +# BEAST_ENHANCED_LOGGING: enable for Debug builds OR when explicitly requested +# Uses generator expression so it works with multi-config generators (Xcode, VS, Ninja Multi-Config) +target_compile_definitions(xrpl_core PUBLIC + $<$,$>:BEAST_ENHANCED_LOGGING=1> +) #[=================================[ main/core headers installation #]=================================] diff --git a/CMakeLists.txt b/CMakeLists.txt index 9a31d4931..d9d42149e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,20 +37,11 @@ endif() #git set(SOURCE_ROOT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/src/") add_definitions(-DSOURCE_ROOT_PATH="${SOURCE_ROOT_PATH}") -# BEAST_ENHANCED_LOGGING option - adds file:line numbers and formatting to logs -# Default to ON for Debug builds, OFF for Release -if(CMAKE_BUILD_TYPE STREQUAL "Debug") - option(BEAST_ENHANCED_LOGGING "Include file and line numbers in log messages" ON) -else() - option(BEAST_ENHANCED_LOGGING "Include file and line numbers in log messages" OFF) -endif() - -if(BEAST_ENHANCED_LOGGING) - add_definitions(-DBEAST_ENHANCED_LOGGING=1) - message(STATUS "Log line numbers enabled") -else() - message(STATUS "Log line numbers disabled") -endif() +# BEAST_ENHANCED_LOGGING - adds file:line numbers and formatting to logs +# Automatically enabled for Debug builds via generator expression +# Can be explicitly controlled with -DBEAST_ENHANCED_LOGGING=ON/OFF +option(BEAST_ENHANCED_LOGGING "Include file and line numbers in log messages (auto: Debug=ON, Release=OFF)" OFF) +message(STATUS "BEAST_ENHANCED_LOGGING option: ${BEAST_ENHANCED_LOGGING}") if(thread_safety_analysis) add_compile_options(-Wthread-safety -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS -DRIPPLE_ENABLE_THREAD_SAFETY_ANNOTATIONS) diff --git a/src/ripple/basics/impl/Log.cpp b/src/ripple/basics/impl/Log.cpp index f4f52cdc5..ddde4584a 100644 --- a/src/ripple/basics/impl/Log.cpp +++ b/src/ripple/basics/impl/Log.cpp @@ -360,7 +360,8 @@ Logs::format( if (!partition.empty()) { #ifdef BEAST_ENHANCED_LOGGING - output += beast::detail::get_log_highlight_color(); + if (beast::detail::should_log_use_colors()) + output += beast::detail::get_log_highlight_color(); #endif output += partition + ":"; } @@ -392,7 +393,8 @@ Logs::format( } #ifdef BEAST_ENHANCED_LOGGING - output += "\033[0m"; + if (beast::detail::should_log_use_colors()) + output += "\033[0m"; #endif output += message; diff --git a/src/ripple/beast/utility/EnhancedLogging.h b/src/ripple/beast/utility/EnhancedLogging.h index 198e0b52f..626562f67 100644 --- a/src/ripple/beast/utility/EnhancedLogging.h +++ b/src/ripple/beast/utility/EnhancedLogging.h @@ -41,6 +41,14 @@ get_log_highlight_color(); constexpr const char* strip_source_root(const char* file) { + // Handle relative paths from build/ directory (common with ccache) + // e.g., "../src/ripple/..." -> "ripple/..." + if (file && file[0] == '.' && file[1] == '.' && file[2] == '/' && + file[3] == 's' && file[4] == 'r' && file[5] == 'c' && file[6] == '/') + { + return file + 7; // skip "../src/" + } + #ifdef SOURCE_ROOT_PATH constexpr const char* sourceRoot = SOURCE_ROOT_PATH; constexpr auto strlen_constexpr = [](const char* s) constexpr diff --git a/src/ripple/beast/utility/src/beast_EnhancedLogging.cpp b/src/ripple/beast/utility/src/beast_EnhancedLogging.cpp index c5f34bd88..f5a608dd0 100644 --- a/src/ripple/beast/utility/src/beast_EnhancedLogging.cpp +++ b/src/ripple/beast/utility/src/beast_EnhancedLogging.cpp @@ -17,6 +17,8 @@ */ //============================================================================== +#ifdef BEAST_ENHANCED_LOGGING + #include #include #include @@ -112,3 +114,5 @@ log_write_location_string(std::ostream& os, const char* file, int line) } // namespace detail } // namespace beast + +#endif // BEAST_ENHANCED_LOGGING diff --git a/src/ripple/beast/utility/src/beast_Journal.cpp b/src/ripple/beast/utility/src/beast_Journal.cpp index 037da0b76..1a86e5bbf 100644 --- a/src/ripple/beast/utility/src/beast_Journal.cpp +++ b/src/ripple/beast/utility/src/beast_Journal.cpp @@ -155,14 +155,43 @@ Journal::ScopedStream::~ScopedStream() #ifdef BEAST_ENHANCED_LOGGING // Add suffix if location is enabled - if (file_ && detail::should_show_location() && !s.empty() && s != "\n") + if (file_ && detail::should_show_location() && !s.empty()) { - std::ostringstream combined; - combined << s; - if (!s.empty() && s.back() != ' ') - combined << " "; - detail::log_write_location_string(combined, file_, line_); - s = combined.str(); + // Single optimized scan from the end + size_t const lastNonWhitespace = s.find_last_not_of(" \n\r\t"); + + // Skip if message is only whitespace (e.g., just "\n" or " \n\n") + if (lastNonWhitespace != std::string::npos) + { + // Count only the trailing newlines (tiny range) + size_t trailingNewlines = 0; + for (size_t i = lastNonWhitespace + 1; i < s.length(); ++i) + { + if (s[i] == '\n') + ++trailingNewlines; + } + + // Build location string once + std::ostringstream locStream; + detail::log_write_location_string(locStream, file_, line_); + std::string const location = locStream.str(); + + // Pre-allocate exact size → zero reallocations + size_t const finalSize = lastNonWhitespace + 1 + 1 + + location.length() + trailingNewlines; + + std::string result; + result.reserve(finalSize); + + // Direct string ops (no ostringstream overhead) + result.append(s, 0, lastNonWhitespace + 1); + result.push_back(' '); + result += location; + if (trailingNewlines > 0) + result.append(trailingNewlines, '\n'); + + s = std::move(result); // Move, no copy + } } #endif