From 374b361daa295570a5c8599fda77e863f442ddb9 Mon Sep 17 00:00:00 2001 From: tequ Date: Tue, 16 Dec 2025 13:16:36 +0900 Subject: [PATCH 1/2] Use Self hosted runner (#639) --- .github/actions/xahau-ga-build/action.yml | 17 ++++-- .../actions/xahau-ga-dependencies/action.yml | 19 +++++-- .github/workflows/build-in-docker.yml | 6 +-- .github/workflows/xahau-ga-nix.yml | 52 +++++++++++++------ 4 files changed, 67 insertions(+), 27 deletions(-) diff --git a/.github/actions/xahau-ga-build/action.yml b/.github/actions/xahau-ga-build/action.yml index c522fff79b..a7bd7ace4a 100644 --- a/.github/actions/xahau-ga-build/action.yml +++ b/.github/actions/xahau-ga-build/action.yml @@ -28,6 +28,10 @@ inputs: description: 'Cache version for invalidation' required: false default: '1' + gha_cache_enabled: + description: 'Whether to use actions/cache (disable for self-hosted with volume mounts)' + required: false + default: 'true' ccache_enabled: description: 'Whether to use ccache' required: false @@ -72,7 +76,7 @@ runs: echo "name=${SAFE_BRANCH}" >> $GITHUB_OUTPUT - name: Restore ccache directory for main branch - if: inputs.ccache_enabled == 'true' + if: inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' id: ccache-restore uses: ./.github/actions/xahau-ga-cache-restore with: @@ -84,7 +88,7 @@ runs: cache-type: ccache-main - name: Restore ccache directory for current branch - if: inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch + if: inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch id: ccache-restore-current-branch uses: ./.github/actions/xahau-ga-cache-restore with: @@ -103,6 +107,11 @@ runs: # Create cache directories mkdir -p ~/.ccache-main ~/.ccache-current + # Keep config separate from cache_dir so configs aren't swapped when CCACHE_DIR changes between steps + mkdir -p ~/.config/ccache + export CCACHE_CONFIGPATH="$HOME/.config/ccache/ccache.conf" + echo "CCACHE_CONFIGPATH=$CCACHE_CONFIGPATH" >> $GITHUB_ENV + # Configure ccache settings AFTER cache restore (prevents stale cached config) ccache --set-config=max_size=${{ inputs.ccache_max_size }} ccache --set-config=hash_dir=${{ inputs.ccache_hash_dir }} @@ -237,14 +246,14 @@ runs: run: ccache -s - name: Save ccache directory for main branch - if: success() && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name == inputs.main_branch + if: success() && inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name == inputs.main_branch uses: actions/cache/save@v4 with: path: ~/.ccache-main key: ${{ steps.ccache-restore.outputs.cache-primary-key }} - name: Save ccache directory for current branch - if: success() && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch + if: success() && inputs.gha_cache_enabled == 'true' && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch uses: actions/cache/save@v4 with: path: ~/.ccache-current diff --git a/.github/actions/xahau-ga-dependencies/action.yml b/.github/actions/xahau-ga-dependencies/action.yml index 1e0c499453..6a491f9f2d 100644 --- a/.github/actions/xahau-ga-dependencies/action.yml +++ b/.github/actions/xahau-ga-dependencies/action.yml @@ -17,8 +17,8 @@ inputs: description: 'Cache version for invalidation' required: false default: '1' - cache_enabled: - description: 'Whether to use caching' + gha_cache_enabled: + description: 'Whether to use actions/cache (disable for self-hosted with volume mounts)' required: false default: 'true' main_branch: @@ -64,7 +64,7 @@ runs: using: 'composite' steps: - name: Restore Conan cache - if: inputs.cache_enabled == 'true' + if: inputs.gha_cache_enabled == 'true' id: cache-restore-conan uses: ./.github/actions/xahau-ga-cache-restore with: @@ -76,6 +76,17 @@ runs: ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}- cache-type: Conan + - name: Configure Conan cache paths + if: inputs.gha_cache_enabled == 'false' + shell: bash + # For self-hosted runners, register cache paths to be used as volumes + # This allows the cache to be shared between containers + run: | + mkdir -p /.conan-cache/conan2 /.conan-cache/conan2_download /.conan-cache/conan2_sources + echo 'core.cache:storage_path=/.conan-cache/conan2' > ~/.conan2/global.conf + echo 'core.download:download_cache=/.conan-cache/conan2_download' >> ~/.conan2/global.conf + echo 'core.sources:download_cache=/.conan-cache/conan2_sources' >> ~/.conan2/global.conf + - name: Configure Conan shell: bash run: | @@ -152,7 +163,7 @@ runs: .. - name: Save Conan cache - if: success() && inputs.cache_enabled == 'true' && steps.cache-restore-conan.outputs.cache-hit != 'true' + if: success() && inputs.gha_cache_enabled == 'true' && steps.cache-restore-conan.outputs.cache-hit != 'true' uses: actions/cache/save@v4 with: path: ~/.conan2 diff --git a/.github/workflows/build-in-docker.yml b/.github/workflows/build-in-docker.yml index e57dda8b4d..f46adfde95 100644 --- a/.github/workflows/build-in-docker.yml +++ b/.github/workflows/build-in-docker.yml @@ -33,7 +33,7 @@ jobs: fetch-depth: 2 # Only get the last 2 commits, to avoid fetching all history build: - runs-on: [self-hosted, vanity] + runs-on: [self-hosted, xahaud-build] needs: [checkout] defaults: run: @@ -74,7 +74,7 @@ jobs: fi tests: - runs-on: [self-hosted, vanity] + runs-on: [self-hosted, xahaud-build] needs: [build, checkout] defaults: run: @@ -84,7 +84,7 @@ jobs: run: /bin/bash docker-unit-tests.sh cleanup: - runs-on: [self-hosted, vanity] + runs-on: [self-hosted, xahaud-build] needs: [tests, checkout] if: always() steps: diff --git a/.github/workflows/xahau-ga-nix.yml b/.github/workflows/xahau-ga-nix.yml index 55c1d69f9d..c63cbe1fe2 100644 --- a/.github/workflows/xahau-ga-nix.yml +++ b/.github/workflows/xahau-ga-nix.yml @@ -14,7 +14,7 @@ concurrency: jobs: matrix-setup: - runs-on: ubuntu-latest + runs-on: [self-hosted, generic, 20.04] container: python:3-slim outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -176,7 +176,16 @@ jobs: build: needs: matrix-setup - runs-on: ubuntu-latest + runs-on: [self-hosted, generic, 20.04] + container: + image: ubuntu:24.04 + volumes: + - /home/runner/.conan-cache:/.conan-cache + - /home/runner/.ccache-main:/github/home/.ccache-main + - /home/runner/.ccache-current:/github/home/.ccache-current + defaults: + run: + shell: bash outputs: artifact_name: ${{ steps.set-artifact-name.outputs.artifact_name }} strategy: @@ -191,23 +200,22 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Get commit message - id: get-commit-message - uses: ./.github/actions/xahau-ga-get-commit-message - with: - event-name: ${{ github.event_name }} - head-commit-message: ${{ github.event.head_commit.message }} - pr-head-sha: ${{ github.event.pull_request.head.sha }} - - name: Install build dependencies run: | - sudo apt-get update - sudo apt-get install -y ninja-build ${{ matrix.cc }} ${{ matrix.cxx }} ccache + apt-get update + apt-get install -y software-properties-common + add-apt-repository ppa:ubuntu-toolchain-r/test -y + apt-get update + apt-get install -y python3 python-is-python3 pipx + pipx ensurepath + apt-get install -y cmake ninja-build ${{ matrix.cc }} ${{ matrix.cxx }} ccache + apt-get install -y perl # for openssl build + apt-get install -y libsqlite3-dev # for xahaud build # Install the specific GCC version needed for Clang if [ -n "${{ matrix.clang_gcc_toolchain }}" ]; then echo "=== Installing GCC ${{ matrix.clang_gcc_toolchain }} for Clang ===" - sudo apt-get install -y gcc-${{ matrix.clang_gcc_toolchain }} g++-${{ matrix.clang_gcc_toolchain }} libstdc++-${{ matrix.clang_gcc_toolchain }}-dev + apt-get install -y gcc-${{ matrix.clang_gcc_toolchain }} g++-${{ matrix.clang_gcc_toolchain }} libstdc++-${{ matrix.clang_gcc_toolchain }}-dev echo "=== GCC versions available after installation ===" ls -la /usr/lib/gcc/x86_64-linux-gnu/ | grep -E "^d" @@ -238,7 +246,7 @@ jobs: echo "Hiding GCC $version -> renaming to $counter (will be seen as GCC version $counter)" # Safety check: ensure target doesn't already exist if [ ! -e "/usr/lib/gcc/x86_64-linux-gnu/$counter" ]; then - sudo mv "$dir" "/usr/lib/gcc/x86_64-linux-gnu/$counter" + mv "$dir" "/usr/lib/gcc/x86_64-linux-gnu/$counter" else echo "ERROR: Cannot rename GCC $version - /usr/lib/gcc/x86_64-linux-gnu/$counter already exists" exit 1 @@ -262,11 +270,12 @@ jobs: # Install libc++ dev packages if using libc++ (not needed for libstdc++) if [ "${{ matrix.stdlib }}" = "libcxx" ]; then - sudo apt-get install -y libc++-${{ matrix.compiler_version }}-dev libc++abi-${{ matrix.compiler_version }}-dev + apt-get install -y libc++-${{ matrix.compiler_version }}-dev libc++abi-${{ matrix.compiler_version }}-dev fi # Install Conan 2 - pip install --upgrade "conan>=2.0,<3" + pipx install "conan>=2.0,<3" + echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Check environment run: | @@ -280,6 +289,14 @@ jobs: echo "---- Full Environment ----" env + - name: Get commit message + id: get-commit-message + uses: ./.github/actions/xahau-ga-get-commit-message + with: + event-name: ${{ github.event_name }} + head-commit-message: ${{ github.event.head_commit.message }} + pr-head-sha: ${{ github.event.pull_request.head.sha }} + - name: Install dependencies uses: ./.github/actions/xahau-ga-dependencies with: @@ -293,6 +310,7 @@ jobs: cc: ${{ matrix.cc }} cxx: ${{ matrix.cxx }} stdlib: ${{ matrix.stdlib }} + gha_cache_enabled: 'false' # Disable caching for self hosted runner - name: Build uses: ./.github/actions/xahau-ga-build @@ -307,6 +325,8 @@ jobs: main_branch: ${{ env.MAIN_BRANCH_NAME }} stdlib: ${{ matrix.stdlib }} clang_gcc_toolchain: ${{ matrix.clang_gcc_toolchain || '' }} + gha_cache_enabled: 'false' # Disable caching for self hosted runner + ccache_max_size: '100G' - name: Set artifact name id: set-artifact-name From f731bcfeba42a8585c60ea4956871c1625895e33 Mon Sep 17 00:00:00 2001 From: tequ Date: Tue, 16 Dec 2025 13:45:45 +0900 Subject: [PATCH 2/2] Increase ccache size from 10G to 100G in release-builder.sh for improved build performance (#643) --- release-builder.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-builder.sh b/release-builder.sh index 672e68042a..e3d55b86a8 100755 --- a/release-builder.sh +++ b/release-builder.sh @@ -192,7 +192,7 @@ ENV PATH=/usr/local/bin:$PATH # Configure ccache and Conan 2 # NOTE: Using echo commands instead of heredocs because heredocs in Docker RUN commands are finnicky -RUN /hbb_exe/activate-exec bash -c "ccache -M 10G && \ +RUN /hbb_exe/activate-exec bash -c "ccache -M 100G && \ ccache -o cache_dir=/cache/ccache && \ ccache -o compiler_check=content && \ mkdir -p ~/.conan2 /cache/conan2 /cache/conan2_download /cache/conan2_sources && \