mirror of
				https://github.com/Xahau/xahaud.git
				synced 2025-11-04 10:45:50 +00:00 
			
		
		
		
	THE GREAT CULLING: Remove all OverlayFS and delta caching logic. After extensive investigation and testing, we determined that OverlayFS file-level layering is fundamentally incompatible with ccache's access patterns: - ccache opens files with O_RDWR → kernel must provide writable file handle - OverlayFS must copy files to upper layer immediately (can't wait) - Even with metacopy=on, metadata-only files still appear in upper layer - Result: ~366MB deltas instead of tiny incremental diffs The fundamental constraint: cannot have all three of: 1. Read-only lower layer (for base sharing) 2. Writable file handles (for O_RDWR) 3. Minimal deltas (for efficient caching) Changes: - Removed all OverlayFS mounting/unmounting logic - Removed workspace and registry tracking - Removed delta creation and restoration - Removed use-deltas parameter - Simplified to direct tar/extract workflow Before: 726 lines across cache actions After: 321 lines (-55% reduction) Benefits: - ✅ Simpler architecture (direct tar/extract) - ✅ More maintainable (less code, less complexity) - ✅ More reliable (fewer moving parts) - ✅ Same performance (base-only was already used) - ✅ Clear path forward (restic/borg for future optimization) Current state works great: - Build times: 20-30 min → 2-5 min (80% improvement) - Cache sizes: ~323-609 MB per branch (with zst compression) - S3 costs: acceptable for current volume If bandwidth costs become problematic, migrate to restic/borg for chunk-level deduplication (completely different architecture).
		
			
				
	
	
		
			170 lines
		
	
	
		
			5.7 KiB
		
	
	
	
		
			YAML
		
	
	
	
	
	
			
		
		
	
	
			170 lines
		
	
	
		
			5.7 KiB
		
	
	
	
		
			YAML
		
	
	
	
	
	
name: dependencies
 | 
						|
description: 'Installs build dependencies with caching'
 | 
						|
 | 
						|
inputs:
 | 
						|
  configuration:
 | 
						|
    description: 'Build configuration (Debug, Release, etc.)'
 | 
						|
    required: true
 | 
						|
  build_dir:
 | 
						|
    description: 'Directory to build dependencies in'
 | 
						|
    required: false
 | 
						|
    default: '.build'
 | 
						|
  compiler-id:
 | 
						|
    description: 'Unique identifier: compiler-version-stdlib[-gccversion] (e.g. clang-14-libstdcxx-gcc11, gcc-13-libstdcxx)'
 | 
						|
    required: false
 | 
						|
    default: ''
 | 
						|
  cache_version:
 | 
						|
    description: 'Cache version for invalidation'
 | 
						|
    required: false
 | 
						|
    default: '1'
 | 
						|
  cache_enabled:
 | 
						|
    description: 'Whether to use caching'
 | 
						|
    required: false
 | 
						|
    default: 'true'
 | 
						|
  main_branch:
 | 
						|
    description: 'Main branch name for restore keys'
 | 
						|
    required: false
 | 
						|
    default: 'dev'
 | 
						|
  os:
 | 
						|
    description: 'Operating system (Linux, Macos)'
 | 
						|
    required: false
 | 
						|
    default: 'Linux'
 | 
						|
  arch:
 | 
						|
    description: 'Architecture (x86_64, armv8)'
 | 
						|
    required: false
 | 
						|
    default: 'x86_64'
 | 
						|
  compiler:
 | 
						|
    description: 'Compiler type (gcc, clang, apple-clang)'
 | 
						|
    required: true
 | 
						|
  compiler_version:
 | 
						|
    description: 'Compiler version (11, 13, 14, etc.)'
 | 
						|
    required: true
 | 
						|
  cc:
 | 
						|
    description: 'C compiler executable (gcc-13, clang-14, etc.), empty for macOS'
 | 
						|
    required: false
 | 
						|
    default: ''
 | 
						|
  cxx:
 | 
						|
    description: 'C++ compiler executable (g++-14, clang++-14, etc.), empty for macOS'
 | 
						|
    required: false
 | 
						|
    default: ''
 | 
						|
  stdlib:
 | 
						|
    description: 'C++ standard library for Conan configuration (note: also in compiler-id)'
 | 
						|
    required: true
 | 
						|
    type: choice
 | 
						|
    options:
 | 
						|
      - libstdcxx
 | 
						|
      - libcxx
 | 
						|
  aws-access-key-id:
 | 
						|
    description: 'AWS Access Key ID for S3 cache storage'
 | 
						|
    required: true
 | 
						|
  aws-secret-access-key:
 | 
						|
    description: 'AWS Secret Access Key for S3 cache storage'
 | 
						|
    required: true
 | 
						|
 | 
						|
outputs:
 | 
						|
  cache-hit:
 | 
						|
    description: 'Whether there was a cache hit'
 | 
						|
    value: ${{ steps.cache-restore-conan.outputs.cache-hit }}
 | 
						|
 | 
						|
runs:
 | 
						|
  using: 'composite'
 | 
						|
  steps:
 | 
						|
 | 
						|
    - name: Restore Conan cache
 | 
						|
      if: inputs.cache_enabled == 'true'
 | 
						|
      id: cache-restore-conan
 | 
						|
      uses: ./.github/actions/xahau-actions-cache-restore
 | 
						|
      with:
 | 
						|
        path: ~/.conan2
 | 
						|
        # Note: compiler-id format is compiler-version-stdlib[-gccversion]
 | 
						|
        key: ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ hashFiles('**/conanfile.txt', '**/conanfile.py') }}-${{ inputs.configuration }}
 | 
						|
        restore-keys: |
 | 
						|
          ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ hashFiles('**/conanfile.txt', '**/conanfile.py') }}-
 | 
						|
          ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-
 | 
						|
        aws-access-key-id: ${{ inputs.aws-access-key-id }}
 | 
						|
        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
 | 
						|
 | 
						|
    - name: Configure Conan
 | 
						|
      shell: bash
 | 
						|
      run: |
 | 
						|
        # Create the default profile directory if it doesn't exist
 | 
						|
        mkdir -p ~/.conan2/profiles
 | 
						|
 | 
						|
        # Determine the correct libcxx based on stdlib parameter
 | 
						|
        if [ "${{ inputs.stdlib }}" = "libcxx" ]; then
 | 
						|
          LIBCXX="libc++"
 | 
						|
        else
 | 
						|
          LIBCXX="libstdc++11"
 | 
						|
        fi
 | 
						|
 | 
						|
        # Create profile with our specific settings
 | 
						|
        # This overwrites any cached profile to ensure fresh configuration
 | 
						|
        cat > ~/.conan2/profiles/default <<EOF
 | 
						|
        [settings]
 | 
						|
        arch=${{ inputs.arch }}
 | 
						|
        build_type=${{ inputs.configuration }}
 | 
						|
        compiler=${{ inputs.compiler }}
 | 
						|
        compiler.cppstd=20
 | 
						|
        compiler.libcxx=${LIBCXX}
 | 
						|
        compiler.version=${{ inputs.compiler_version }}
 | 
						|
        os=${{ inputs.os }}
 | 
						|
        EOF
 | 
						|
 | 
						|
        # Add buildenv and conf sections for Linux (not needed for macOS)
 | 
						|
        if [ "${{ inputs.os }}" = "Linux" ] && [ -n "${{ inputs.cc }}" ]; then
 | 
						|
          cat >> ~/.conan2/profiles/default <<EOF
 | 
						|
 | 
						|
        [buildenv]
 | 
						|
        CC=/usr/bin/${{ inputs.cc }}
 | 
						|
        CXX=/usr/bin/${{ inputs.cxx }}
 | 
						|
 | 
						|
        [conf]
 | 
						|
        tools.build:compiler_executables={"c": "/usr/bin/${{ inputs.cc }}", "cpp": "/usr/bin/${{ inputs.cxx }}"}
 | 
						|
        EOF
 | 
						|
        fi
 | 
						|
 | 
						|
        # Add macOS-specific conf if needed
 | 
						|
        if [ "${{ inputs.os }}" = "Macos" ]; then
 | 
						|
          cat >> ~/.conan2/profiles/default <<EOF
 | 
						|
 | 
						|
        [conf]
 | 
						|
        # Workaround for gRPC with newer Apple Clang
 | 
						|
        tools.build:cxxflags=["-Wno-missing-template-arg-list-after-template-kw"]
 | 
						|
        EOF
 | 
						|
        fi
 | 
						|
 | 
						|
        # Display profile for verification
 | 
						|
        conan profile show
 | 
						|
 | 
						|
    - name: Export custom recipes
 | 
						|
      shell: bash
 | 
						|
      run: |
 | 
						|
        conan export external/snappy --version 1.1.10 --user xahaud --channel stable
 | 
						|
        conan export external/soci --version 4.0.3 --user xahaud --channel stable
 | 
						|
        conan export external/wasmedge --version 0.11.2 --user xahaud --channel stable
 | 
						|
 | 
						|
    - name: Install dependencies
 | 
						|
      shell: bash
 | 
						|
      env:
 | 
						|
        CONAN_REQUEST_TIMEOUT: 180  # Increase timeout to 3 minutes for slow mirrors
 | 
						|
      run: |
 | 
						|
        # Create build directory
 | 
						|
        mkdir -p ${{ inputs.build_dir }}
 | 
						|
        cd ${{ inputs.build_dir }}
 | 
						|
        
 | 
						|
        # Install dependencies using conan
 | 
						|
        conan install \
 | 
						|
          --output-folder . \
 | 
						|
          --build missing \
 | 
						|
          --settings build_type=${{ inputs.configuration }} \
 | 
						|
          ..
 | 
						|
 | 
						|
    - name: Save Conan cache
 | 
						|
      if: always() && inputs.cache_enabled == 'true' && steps.cache-restore-conan.outputs.cache-hit != 'true'
 | 
						|
      uses: ./.github/actions/xahau-actions-cache-save
 | 
						|
      with:
 | 
						|
        path: ~/.conan2
 | 
						|
        key: ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ hashFiles('**/conanfile.txt', '**/conanfile.py') }}-${{ inputs.configuration }}
 | 
						|
        aws-access-key-id: ${{ inputs.aws-access-key-id }}
 | 
						|
        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
 |