Merge branch 'dev' into ccache-100G

.github/actions/xahau-ga-build/action.yml

@@ -71,25 +71,56 @@ runs:
         SAFE_BRANCH=$(echo "${{ github.ref_name }}" | tr -c 'a-zA-Z0-9_.-' '-')
         echo "name=${SAFE_BRANCH}" >> $GITHUB_OUTPUT
 
+    - name: Restore ccache directory for main branch
+      if: inputs.ccache_enabled == 'true'
+      id: ccache-restore
+      uses: ./.github/actions/xahau-ga-cache-restore
+      with:
+        path: ~/.ccache-main
+        key: ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-${{ inputs.main_branch }}
+        restore-keys: |
+          ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-
+          ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-
+        cache-type: ccache-main
+
+    - name: Restore ccache directory for current branch
+      if: inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch
+      id: ccache-restore-current-branch
+      uses: ./.github/actions/xahau-ga-cache-restore
+      with:
+        path: ~/.ccache-current
+        key: ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-${{ steps.safe-branch.outputs.name }}
+        restore-keys: |
+          ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-${{ inputs.main_branch }}
+          ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-
+          ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-
+        cache-type: ccache-current
+
     - name: Configure ccache
       if: inputs.ccache_enabled == 'true'
       shell: bash
       run: |
         # Create cache directories
-        mkdir -p ~/.ccache-cache
-
-        # Keep config separate from cache_dir so configs aren't swapped when CCACHE_DIR changes between steps
-        mkdir -p ~/.config/ccache
-        export CCACHE_CONFIGPATH="$HOME/.config/ccache/ccache.conf"
-        echo "CCACHE_CONFIGPATH=$CCACHE_CONFIGPATH" >> $GITHUB_ENV
+        mkdir -p ~/.ccache-main ~/.ccache-current
 
         # Configure ccache settings AFTER cache restore (prevents stale cached config)
         ccache --set-config=max_size=${{ inputs.ccache_max_size }}
         ccache --set-config=hash_dir=${{ inputs.ccache_hash_dir }}
         ccache --set-config=compiler_check=${{ inputs.ccache_compiler_check }}
-        ccache --set-config=cache_dir="$HOME/.ccache-cache"
-        echo "CCACHE_DIR=$HOME/.ccache-cache" >> $GITHUB_ENV
-        echo "📦 using ~/.ccache-cache as ccache cache directory"
+
+        # Determine if we're on the main branch
+        if [ "${{ steps.safe-branch.outputs.name }}" = "${{ inputs.main_branch }}" ]; then
+          # Main branch: use main branch cache only
+          ccache --set-config=cache_dir="$HOME/.ccache-main"
+          echo "CCACHE_DIR=$HOME/.ccache-main" >> $GITHUB_ENV
+          echo "📦 Main branch: using ~/.ccache-main"
+        else
+          # Feature branch: use current branch cache with main as secondary (read-only fallback)
+          ccache --set-config=cache_dir="$HOME/.ccache-current"
+          ccache --set-config=secondary_storage="file:$HOME/.ccache-main"
+          echo "CCACHE_DIR=$HOME/.ccache-current" >> $GITHUB_ENV
+          echo "📦 Feature branch: using ~/.ccache-current with ~/.ccache-main as secondary"
+        fi
 
         # Print config for verification
         echo "=== ccache configuration ==="
@@ -204,3 +235,17 @@ runs:
       if: inputs.ccache_enabled == 'true'
       shell: bash
       run: ccache -s
+
+    - name: Save ccache directory for main branch
+      if: success() && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name == inputs.main_branch
+      uses: actions/cache/save@v4
+      with:
+        path: ~/.ccache-main
+        key: ${{ steps.ccache-restore.outputs.cache-primary-key }}
+
+    - name: Save ccache directory for current branch
+      if: success() && inputs.ccache_enabled == 'true' && steps.safe-branch.outputs.name != inputs.main_branch
+      uses: actions/cache/save@v4
+      with:
+        path: ~/.ccache-current
+        key: ${{ steps.ccache-restore-current-branch.outputs.cache-primary-key }}

.github/actions/xahau-ga-dependencies/action.yml

@@ -17,6 +17,10 @@ inputs:
     description: 'Cache version for invalidation'
     required: false
     default: '1'
+  cache_enabled:
+    description: 'Whether to use caching'
+    required: false
+    default: 'true'
   main_branch:
     description: 'Main branch name for restore keys'
     required: false
@@ -59,14 +63,18 @@ outputs:
 runs:
   using: 'composite'
   steps:
-    - name: Configure Conan cache paths
-      if: inputs.os == 'Linux'
-      shell: bash
-      run: |
-        mkdir -p /.conan-cache/conan2 /.conan-cache/conan2_download /.conan-cache/conan2_sources
-        echo 'core.cache:storage_path=/.conan-cache/conan2' > ~/.conan2/global.conf
-        echo 'core.download:download_cache=/.conan-cache/conan2_download' >> ~/.conan2/global.conf
-        echo 'core.sources:download_cache=/.conan-cache/conan2_sources' >> ~/.conan2/global.conf
+    - name: Restore Conan cache
+      if: inputs.cache_enabled == 'true'
+      id: cache-restore-conan
+      uses: ./.github/actions/xahau-ga-cache-restore
+      with:
+        path: ~/.conan2
+        # Note: compiler-id format is compiler-version-stdlib[-gccversion]
+        key: ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ hashFiles('**/conanfile.py') }}-${{ inputs.configuration }}
+        restore-keys: |
+          ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ hashFiles('**/conanfile.py') }}-
+          ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-
+        cache-type: Conan
 
     - name: Configure Conan
       shell: bash
@@ -142,3 +150,10 @@ runs:
           --build missing \
           --settings build_type=${{ inputs.configuration }} \
           ..
+
+    - name: Save Conan cache
+      if: success() && inputs.cache_enabled == 'true' && steps.cache-restore-conan.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        path: ~/.conan2
+        key: ${{ steps.cache-restore-conan.outputs.cache-primary-key }}

.github/workflows/build-in-docker.yml

@@ -33,7 +33,7 @@ jobs:
           fetch-depth: 2 # Only get the last 2 commits, to avoid fetching all history
 
   build:
-    runs-on: [self-hosted, xahaud-build]
+    runs-on: [self-hosted, vanity]
     needs: [checkout]
     defaults:
       run:
@@ -74,7 +74,7 @@ jobs:
           fi
 
   tests:
-    runs-on: [self-hosted, xahaud-build]
+    runs-on: [self-hosted, vanity]
     needs: [build, checkout]
     defaults:
       run:
@@ -84,7 +84,7 @@ jobs:
         run: /bin/bash docker-unit-tests.sh
 
   cleanup:
-    runs-on: [self-hosted, xahaud-build]
+    runs-on: [self-hosted, vanity]
     needs: [tests, checkout]
     if: always()
     steps:

.github/workflows/xahau-ga-macos.yml

@@ -20,7 +20,7 @@ jobs:
           - Ninja
         configuration:
           - Debug
-    runs-on: [self-hosted, macOS]
+    runs-on: macos-15
     env:
       build_dir: .build
       # Bump this number to invalidate all caches globally.
@@ -30,29 +30,61 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Add Homebrew to PATH
+      - name: Get commit message
+        id: get-commit-message
+        uses: ./.github/actions/xahau-ga-get-commit-message
+        with:
+          event-name: ${{ github.event_name }}
+          head-commit-message: ${{ github.event.head_commit.message }}
+          pr-head-sha: ${{ github.event.pull_request.head.sha }}
+
+      - name: Install Conan
         run: |
-          echo "/opt/homebrew/bin" >> "$GITHUB_PATH"
-          echo "/opt/homebrew/sbin" >> "$GITHUB_PATH"
+          brew install conan
+          # Verify Conan 2 is installed
+          conan --version
 
       - name: Install Coreutils
         run: |
           brew install coreutils
           echo "Num proc: $(nproc)"
 
-      # To isolate environments for each Runner, instead of installing globally with brew, 
-      # use mise to isolate environments for each Runner directory.
-      - name: Setup toolchain (mise)
-        uses: jdx/mise-action@v2
-        with:
-          install: true
+      - name: Install Ninja
+        if: matrix.generator == 'Ninja'
+        run: brew install ninja
 
-      - name: Install tools via mise
+      - name: Install Python
         run: |
-          mise install
-          mise use cmake@3.23.1 python@3.12 pipx@latest conan@2 ninja@latest ccache@latest
-          mise reshim
-          echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH"
+          if which python3 > /dev/null 2>&1; then
+              echo "Python 3 executable exists"
+              python3 --version
+          else
+              brew install python@3.12
+          fi
+          # Create 'python' symlink if it doesn't exist (for tools expecting 'python')
+          if ! which python > /dev/null 2>&1; then
+              sudo ln -sf $(which python3) /usr/local/bin/python
+          fi
+
+      - name: Install CMake
+        run: |
+          # Install CMake 3.x to match local dev environments
+          # With Conan 2 and the policy args passed to CMake, newer versions
+          # can have issues with dependencies that require cmake_minimum_required < 3.5
+          brew uninstall cmake --ignore-dependencies 2>/dev/null || true
+
+          # Download and install CMake 3.31.7 directly
+          curl -L https://github.com/Kitware/CMake/releases/download/v3.31.7/cmake-3.31.7-macos-universal.tar.gz -o cmake.tar.gz
+          tar -xzf cmake.tar.gz
+
+          # Move the entire CMake.app to /Applications
+          sudo mv cmake-3.31.7-macos-universal/CMake.app /Applications/
+
+          echo "/Applications/CMake.app/Contents/bin" >> $GITHUB_PATH
+          /Applications/CMake.app/Contents/bin/cmake --version
+
+      - name: Install ccache
+        run: brew install ccache
 
       - name: Check environment
         run: |
@@ -66,14 +98,6 @@ jobs:
           echo "---- Full Environment ----"
           env
 
-      - name: Get commit message
-        id: get-commit-message
-        uses: ./.github/actions/xahau-ga-get-commit-message
-        with:
-          event-name: ${{ github.event_name }}
-          head-commit-message: ${{ github.event.head_commit.message }}
-          pr-head-sha: ${{ github.event.pull_request.head.sha }}
-
       - name: Detect compiler version
         id: detect-compiler
         run: |
@@ -105,7 +129,6 @@ jobs:
           cache_version: ${{ env.CACHE_VERSION }}
           main_branch: ${{ env.MAIN_BRANCH_NAME }}
           stdlib: libcxx
-          ccache_max_size: '100G'
 
       - name: Test
         run: |

.github/workflows/xahau-ga-nix.yml

@@ -14,7 +14,7 @@ concurrency:
 
 jobs:
   matrix-setup:
-    runs-on: [self-hosted, generic, 20.04]
+    runs-on: ubuntu-latest
     container: python:3-slim
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
@@ -176,15 +176,7 @@ jobs:
 
   build:
     needs: matrix-setup
-    runs-on: [self-hosted, generic, 20.04]
-    container:
-      image: ubuntu:24.04
-      volumes:
-        - /home/runner/.conan-cache:/.conan-cache
-        - /home/runner/.ccache-cache:/github/home/.ccache-cache
-    defaults:
-      run:
-        shell: bash
+    runs-on: ubuntu-latest
     outputs:
       artifact_name: ${{ steps.set-artifact-name.outputs.artifact_name }}
     strategy:
@@ -199,22 +191,23 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Get commit message
+        id: get-commit-message
+        uses: ./.github/actions/xahau-ga-get-commit-message
+        with:
+          event-name: ${{ github.event_name }}
+          head-commit-message: ${{ github.event.head_commit.message }}
+          pr-head-sha: ${{ github.event.pull_request.head.sha }}
+
       - name: Install build dependencies
         run: |
-          apt-get update
-          apt-get install -y software-properties-common
-          add-apt-repository ppa:ubuntu-toolchain-r/test -y
-          apt-get update
-          apt-get install -y python3 python-is-python3 pipx
-          pipx ensurepath
-          apt-get install -y cmake ninja-build ${{ matrix.cc }} ${{ matrix.cxx }} ccache
-          apt-get install -y perl # for openssl build
-          apt-get install -y libsqlite3-dev # for xahaud build
+          sudo apt-get update
+          sudo apt-get install -y ninja-build ${{ matrix.cc }} ${{ matrix.cxx }} ccache
 
           # Install the specific GCC version needed for Clang
           if [ -n "${{ matrix.clang_gcc_toolchain }}" ]; then
             echo "=== Installing GCC ${{ matrix.clang_gcc_toolchain }} for Clang ==="
-            apt-get install -y gcc-${{ matrix.clang_gcc_toolchain }} g++-${{ matrix.clang_gcc_toolchain }} libstdc++-${{ matrix.clang_gcc_toolchain }}-dev
+            sudo apt-get install -y gcc-${{ matrix.clang_gcc_toolchain }} g++-${{ matrix.clang_gcc_toolchain }} libstdc++-${{ matrix.clang_gcc_toolchain }}-dev
 
             echo "=== GCC versions available after installation ==="
             ls -la /usr/lib/gcc/x86_64-linux-gnu/ | grep -E "^d"
@@ -245,7 +238,7 @@ jobs:
                   echo "Hiding GCC $version -> renaming to $counter (will be seen as GCC version $counter)"
                   # Safety check: ensure target doesn't already exist
                   if [ ! -e "/usr/lib/gcc/x86_64-linux-gnu/$counter" ]; then
-                    mv "$dir" "/usr/lib/gcc/x86_64-linux-gnu/$counter"
+                    sudo mv "$dir" "/usr/lib/gcc/x86_64-linux-gnu/$counter"
                   else
                     echo "ERROR: Cannot rename GCC $version - /usr/lib/gcc/x86_64-linux-gnu/$counter already exists"
                     exit 1
@@ -269,12 +262,11 @@ jobs:
 
           # Install libc++ dev packages if using libc++ (not needed for libstdc++)
           if [ "${{ matrix.stdlib }}" = "libcxx" ]; then
-            apt-get install -y libc++-${{ matrix.compiler_version }}-dev libc++abi-${{ matrix.compiler_version }}-dev
+            sudo apt-get install -y libc++-${{ matrix.compiler_version }}-dev libc++abi-${{ matrix.compiler_version }}-dev
           fi
 
           # Install Conan 2
-          pipx install "conan>=2.0,<3"
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
+          pip install --upgrade "conan>=2.0,<3"
 
       - name: Check environment
         run: |
@@ -288,14 +280,6 @@ jobs:
           echo "---- Full Environment ----"
           env
 
-      - name: Get commit message
-        id: get-commit-message
-        uses: ./.github/actions/xahau-ga-get-commit-message
-        with:
-          event-name: ${{ github.event_name }}
-          head-commit-message: ${{ github.event.head_commit.message }}
-          pr-head-sha: ${{ github.event.pull_request.head.sha }}
-
       - name: Install dependencies
         uses: ./.github/actions/xahau-ga-dependencies
         with:
@@ -323,7 +307,6 @@ jobs:
           main_branch: ${{ env.MAIN_BRANCH_NAME }}
           stdlib: ${{ matrix.stdlib }}
           clang_gcc_toolchain: ${{ matrix.clang_gcc_toolchain || '' }}
-          ccache_max_size: '100G'
 
       - name: Set artifact name
         id: set-artifact-name

release-builder.sh

@@ -192,7 +192,7 @@ ENV PATH=/usr/local/bin:$PATH
 
 # Configure ccache and Conan 2
 # NOTE: Using echo commands instead of heredocs because heredocs in Docker RUN commands are finnicky
-RUN /hbb_exe/activate-exec bash -c "ccache -M 10G && \
+RUN /hbb_exe/activate-exec bash -c "ccache -M 100G && \
     ccache -o cache_dir=/cache/ccache && \
     ccache -o compiler_check=content && \
     mkdir -p ~/.conan2 /cache/conan2 /cache/conan2_download /cache/conan2_sources && \