mirror of
https://github.com/Xahau/xahaud.git
synced 2025-11-18 09:35:49 +00:00
Compare commits
35 Commits
actions-ca
...
nd-experim
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
83f6bc64e1 | ||
|
|
be6fad9692 | ||
|
|
b24e4647ba | ||
|
|
638cb0afe5 | ||
|
|
bd384e6bc1 | ||
|
|
4c546e5d91 | ||
|
|
28727b3f86 | ||
|
|
a4f96a435a | ||
|
|
d0f63cc2d1 | ||
|
|
2433bfe277 | ||
|
|
ef40a7f351 | ||
|
|
a4a4126bdc | ||
|
|
0559b6c418 | ||
|
|
f8d1a6f2b4 | ||
|
|
c46ede7c8f | ||
|
|
0e2bc365ea | ||
|
|
446bc76b69 | ||
|
|
a0c38a4fb3 | ||
|
|
631650f7eb | ||
|
|
0b31d8e534 | ||
|
|
ecf03f4afe | ||
|
|
b801c2837d | ||
|
|
1474e808cb | ||
|
|
457e633a81 | ||
|
|
7ea99caa19 | ||
|
|
3e5c15c172 | ||
|
|
52b4fb503c | ||
|
|
98123fa934 | ||
|
|
ce7b1c4f1d | ||
|
|
e062dcae58 | ||
|
|
a9d284fec1 | ||
|
|
065d0c3e07 | ||
|
|
4fda40b709 | ||
|
|
6014356d91 | ||
|
|
d790f97430 |
@@ -1,5 +1,4 @@
|
||||
name: 'Xahau Cache Restore (S3)'
|
||||
bump: 1
|
||||
description: 'Drop-in replacement for actions/cache/restore using S3 storage'
|
||||
|
||||
inputs:
|
||||
@@ -75,93 +74,14 @@ runs:
|
||||
echo "S3 bucket: s3://${S3_BUCKET}"
|
||||
echo ""
|
||||
|
||||
# Normalize target path (expand tilde)
|
||||
# Normalize target path (expand tilde and resolve to absolute path)
|
||||
if [[ "${TARGET_PATH}" == ~* ]]; then
|
||||
TARGET_PATH="${HOME}${TARGET_PATH:1}"
|
||||
fi
|
||||
|
||||
# Canonicalize path (Linux only - macOS realpath doesn't support -m)
|
||||
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||
TARGET_PATH=$(realpath -m "${TARGET_PATH}")
|
||||
fi
|
||||
|
||||
TARGET_PATH=$(realpath -m "${TARGET_PATH}")
|
||||
echo "Normalized target path: ${TARGET_PATH}"
|
||||
echo ""
|
||||
|
||||
# Debug: Show commit message
|
||||
echo "=========================================="
|
||||
echo "DEBUG: Cache clear tag detection"
|
||||
echo "=========================================="
|
||||
echo "Raw commit message:"
|
||||
echo "${XAHAU_GA_COMMIT_MSG}"
|
||||
echo ""
|
||||
echo "Searching for: [ci-ga-clear-cache] or [ci-ga-clear-cache:*]"
|
||||
echo ""
|
||||
|
||||
# Check for [ci-ga-clear-cache] tag in commit message (with optional search terms)
|
||||
# Examples:
|
||||
# [ci-ga-clear-cache] - Clear this job's cache
|
||||
# [ci-ga-clear-cache:ccache] - Clear only if key contains "ccache"
|
||||
# [ci-ga-clear-cache:gcc Debug] - Clear only if key contains both "gcc" AND "Debug"
|
||||
|
||||
# Extract search terms if present (e.g., "ccache" from "[ci-ga-clear-cache:ccache]")
|
||||
SEARCH_TERMS=$(echo "${XAHAU_GA_COMMIT_MSG}" | grep -o '\[ci-ga-clear-cache:[^]]*\]' | sed 's/\[ci-ga-clear-cache://;s/\]//' || echo "")
|
||||
|
||||
SHOULD_CLEAR=false
|
||||
|
||||
if [ -n "${SEARCH_TERMS}" ]; then
|
||||
# Search terms provided - check if THIS cache key matches ALL terms (AND logic)
|
||||
echo "🔍 [ci-ga-clear-cache:${SEARCH_TERMS}] detected"
|
||||
echo "Checking if cache key matches search terms..."
|
||||
echo " Cache key: ${CACHE_KEY}"
|
||||
echo " Search terms: ${SEARCH_TERMS}"
|
||||
echo ""
|
||||
|
||||
MATCHES=true
|
||||
for term in ${SEARCH_TERMS}; do
|
||||
if ! echo "${CACHE_KEY}" | grep -q "${term}"; then
|
||||
MATCHES=false
|
||||
echo " ✗ Key does not contain '${term}'"
|
||||
break
|
||||
else
|
||||
echo " ✓ Key contains '${term}'"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${MATCHES}" = "true" ]; then
|
||||
echo ""
|
||||
echo "✅ Cache key matches all search terms - will clear cache"
|
||||
SHOULD_CLEAR=true
|
||||
else
|
||||
echo ""
|
||||
echo "⏭️ Cache key doesn't match search terms - skipping cache clear"
|
||||
fi
|
||||
elif echo "${XAHAU_GA_COMMIT_MSG}" | grep -q '\[ci-ga-clear-cache\]'; then
|
||||
# No search terms - always clear this job's cache
|
||||
echo "🗑️ [ci-ga-clear-cache] detected in commit message"
|
||||
echo "Clearing cache for key: ${CACHE_KEY}"
|
||||
SHOULD_CLEAR=true
|
||||
fi
|
||||
|
||||
if [ "${SHOULD_CLEAR}" = "true" ]; then
|
||||
echo ""
|
||||
|
||||
# Delete base layer
|
||||
S3_BASE_KEY="s3://${S3_BUCKET}/${CACHE_KEY}-base.tar.zst"
|
||||
if aws s3 ls "${S3_BASE_KEY}" --region "${S3_REGION}" >/dev/null 2>&1; then
|
||||
echo "Deleting base layer: ${S3_BASE_KEY}"
|
||||
aws s3 rm "${S3_BASE_KEY}" --region "${S3_REGION}" 2>/dev/null || true
|
||||
echo "✓ Base layer deleted"
|
||||
else
|
||||
echo "ℹ️ No base layer found to delete"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "✅ Cache cleared successfully"
|
||||
echo "Build will proceed from scratch (bootstrap mode)"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Function to try restoring a cache key
|
||||
try_restore_key() {
|
||||
local key=$1
|
||||
@@ -254,7 +174,7 @@ runs:
|
||||
|
||||
echo ""
|
||||
echo "Downloading cache..."
|
||||
aws s3 cp "${S3_KEY}" "${TEMP_TARBALL}" --region "${S3_REGION}" --no-progress
|
||||
aws s3 cp "${S3_KEY}" "${TEMP_TARBALL}" --region "${S3_REGION}"
|
||||
|
||||
TARBALL_SIZE=$(du -h "${TEMP_TARBALL}" | cut -f1)
|
||||
echo "✓ Downloaded: ${TARBALL_SIZE}"
|
||||
@@ -95,7 +95,7 @@ runs:
|
||||
--body "${BASE_TARBALL}" \
|
||||
--tagging 'type=base' \
|
||||
--region "${S3_REGION}" \
|
||||
>/dev/null 2>&1
|
||||
>/dev/null
|
||||
|
||||
echo "✓ Uploaded: ${S3_BASE_KEY}"
|
||||
|
||||
16
.github/actions/xahau-ga-build/action.yml
vendored
16
.github/actions/xahau-ga-build/action.yml
vendored
@@ -80,7 +80,7 @@ runs:
|
||||
- name: Restore ccache directory
|
||||
if: inputs.ccache_enabled == 'true'
|
||||
id: ccache-restore
|
||||
uses: ./.github/actions/xahau-ga-cache-restore
|
||||
uses: ./.github/actions/xahau-actions-cache-restore
|
||||
with:
|
||||
path: ~/.ccache
|
||||
key: ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-${{ steps.safe-branch.outputs.name }}
|
||||
@@ -208,15 +208,7 @@ runs:
|
||||
shell: bash
|
||||
run: |
|
||||
cd ${{ inputs.build_dir }}
|
||||
|
||||
# Check for verbose build flag in commit message
|
||||
VERBOSE_FLAG=""
|
||||
if echo "${XAHAU_GA_COMMIT_MSG}" | grep -q '\[ci-ga-cmake-verbose\]'; then
|
||||
echo "🔊 [ci-ga-cmake-verbose] detected - enabling verbose output"
|
||||
VERBOSE_FLAG="-- -v"
|
||||
fi
|
||||
|
||||
cmake --build . --config ${{ inputs.configuration }} --parallel $(nproc) ${VERBOSE_FLAG}
|
||||
cmake --build . --config ${{ inputs.configuration }} --parallel $(nproc) -- -v
|
||||
|
||||
- name: Show ccache statistics
|
||||
if: inputs.ccache_enabled == 'true'
|
||||
@@ -224,8 +216,8 @@ runs:
|
||||
run: ccache -s
|
||||
|
||||
- name: Save ccache directory
|
||||
if: success() && inputs.ccache_enabled == 'true'
|
||||
uses: ./.github/actions/xahau-ga-cache-save
|
||||
if: always() && inputs.ccache_enabled == 'true'
|
||||
uses: ./.github/actions/xahau-actions-cache-save
|
||||
with:
|
||||
path: ~/.ccache
|
||||
key: ${{ runner.os }}-ccache-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ inputs.configuration }}-${{ steps.safe-branch.outputs.name }}
|
||||
|
||||
@@ -73,7 +73,7 @@ runs:
|
||||
- name: Restore Conan cache
|
||||
if: inputs.cache_enabled == 'true'
|
||||
id: cache-restore-conan
|
||||
uses: ./.github/actions/xahau-ga-cache-restore
|
||||
uses: ./.github/actions/xahau-actions-cache-restore
|
||||
with:
|
||||
path: ~/.conan2
|
||||
# Note: compiler-id format is compiler-version-stdlib[-gccversion]
|
||||
@@ -160,8 +160,8 @@ runs:
|
||||
..
|
||||
|
||||
- name: Save Conan cache
|
||||
if: success() && inputs.cache_enabled == 'true' && steps.cache-restore-conan.outputs.cache-hit != 'true'
|
||||
uses: ./.github/actions/xahau-ga-cache-save
|
||||
if: always() && inputs.cache_enabled == 'true' && steps.cache-restore-conan.outputs.cache-hit != 'true'
|
||||
uses: ./.github/actions/xahau-actions-cache-save
|
||||
with:
|
||||
path: ~/.conan2
|
||||
key: ${{ runner.os }}-conan-v${{ inputs.cache_version }}-${{ inputs.compiler-id }}-${{ hashFiles('**/conanfile.txt', '**/conanfile.py') }}-${{ inputs.configuration }}
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
name: 'Get Commit Message'
|
||||
description: 'Gets commit message for both push and pull_request events and sets XAHAU_GA_COMMIT_MSG env var'
|
||||
|
||||
inputs:
|
||||
event-name:
|
||||
description: 'The event name (push or pull_request)'
|
||||
required: true
|
||||
head-commit-message:
|
||||
description: 'The head commit message (for push events)'
|
||||
required: false
|
||||
default: ''
|
||||
pr-head-sha:
|
||||
description: 'The PR head SHA (for pull_request events)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Get commit message and set environment variable
|
||||
shell: python
|
||||
run: |
|
||||
import os
|
||||
import subprocess
|
||||
import secrets
|
||||
|
||||
event_name = "${{ inputs.event-name }}"
|
||||
pr_head_sha = "${{ inputs.pr-head-sha }}"
|
||||
|
||||
print("==========================================")
|
||||
print("Setting XAHAU_GA_COMMIT_MSG environment variable")
|
||||
print("==========================================")
|
||||
print(f"Event: {event_name}")
|
||||
|
||||
if event_name == 'push':
|
||||
# For push events, use the input directly
|
||||
message = """${{ inputs.head-commit-message }}"""
|
||||
print("Source: workflow input (github.event.head_commit.message)")
|
||||
elif event_name == 'pull_request':
|
||||
# For PR events, fetch the specific SHA
|
||||
print(f"Source: git show {pr_head_sha} (fetching PR head commit)")
|
||||
|
||||
# Fetch the PR head commit
|
||||
subprocess.run(
|
||||
['git', 'fetch', 'origin', pr_head_sha],
|
||||
check=True
|
||||
)
|
||||
|
||||
# Get commit message from the fetched SHA
|
||||
result = subprocess.run(
|
||||
['git', 'show', '-s', '--format=%B', pr_head_sha],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
message = result.stdout.strip()
|
||||
else:
|
||||
message = ""
|
||||
print(f"Warning: Unknown event type: {event_name}")
|
||||
|
||||
print(f"Commit message (first 100 chars): {message[:100]}")
|
||||
|
||||
# Write to GITHUB_ENV using heredoc with random delimiter (prevents injection attacks)
|
||||
# See: https://securitylab.github.com/resources/github-actions-untrusted-input/
|
||||
delimiter = f"EOF_{secrets.token_hex(16)}"
|
||||
|
||||
with open(os.environ['GITHUB_ENV'], 'a') as f:
|
||||
f.write(f'XAHAU_GA_COMMIT_MSG<<{delimiter}\n')
|
||||
f.write(message)
|
||||
f.write(f'\n{delimiter}\n')
|
||||
|
||||
print(f"✓ XAHAU_GA_COMMIT_MSG set (available to all subsequent steps)")
|
||||
print("==========================================")
|
||||
290
.github/workflows/test-cache-actions.yml.disabled
vendored
Normal file
290
.github/workflows/test-cache-actions.yml.disabled
vendored
Normal file
@@ -0,0 +1,290 @@
|
||||
name: Test Cache Actions (State Machine)
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["nd-experiment-overlayfs-*"]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
state_assertion:
|
||||
description: 'Expected state (optional, e.g. "2" to assert state 2)'
|
||||
required: false
|
||||
type: string
|
||||
default: '1'
|
||||
start_state:
|
||||
description: 'Force specific starting state (optional, e.g. "3" to start at state 3)'
|
||||
required: false
|
||||
type: string
|
||||
clear_cache:
|
||||
description: 'Clear cache before running'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test-cache-state-machine:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
CACHE_KEY: test-state-machine-${{ github.ref_name }}
|
||||
CACHE_DIR: /tmp/test-cache
|
||||
S3_BUCKET: xahaud-github-actions-cache-niq
|
||||
S3_REGION: us-east-1
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Parse Inputs (workflow_dispatch or commit message)
|
||||
id: parse-inputs
|
||||
run: |
|
||||
# Priority 1: workflow_dispatch inputs (manual trigger)
|
||||
STATE_ASSERTION="${{ inputs.state_assertion }}"
|
||||
START_STATE="${{ inputs.start_state }}"
|
||||
SHOULD_CLEAR="${{ inputs.clear_cache }}"
|
||||
|
||||
# Priority 2: commit message tags (push event)
|
||||
if [ "${{ github.event_name }}" = "push" ]; then
|
||||
COMMIT_MSG="${{ github.event.head_commit.message }}"
|
||||
|
||||
# Parse [state:N] assertion tag (optional, if not provided as input)
|
||||
if [ -z "${STATE_ASSERTION}" ] && echo "${COMMIT_MSG}" | grep -qE '\[state:[0-9]+\]'; then
|
||||
STATE_ASSERTION=$(echo "${COMMIT_MSG}" | grep -oE '\[state:[0-9]+\]' | grep -oE '[0-9]+')
|
||||
echo "State assertion found in commit: ${STATE_ASSERTION}"
|
||||
fi
|
||||
|
||||
# Parse [start-state:N] force tag (optional, if not provided as input)
|
||||
if [ -z "${START_STATE}" ] && echo "${COMMIT_MSG}" | grep -qE '\[start-state:[0-9]+\]'; then
|
||||
START_STATE=$(echo "${COMMIT_MSG}" | grep -oE '\[start-state:[0-9]+\]' | grep -oE '[0-9]+')
|
||||
echo "Start state found in commit: ${START_STATE}"
|
||||
fi
|
||||
|
||||
# Parse [ci-clear-cache] tag (if not provided as input)
|
||||
if [ "${SHOULD_CLEAR}" != "true" ] && echo "${COMMIT_MSG}" | grep -q '\[ci-clear-cache\]'; then
|
||||
SHOULD_CLEAR=true
|
||||
echo "Cache clear requested in commit"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Output final values
|
||||
echo "state_assertion=${STATE_ASSERTION}" >> "$GITHUB_OUTPUT"
|
||||
echo "start_state=${START_STATE}" >> "$GITHUB_OUTPUT"
|
||||
echo "should_clear=${SHOULD_CLEAR}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Log what we're using
|
||||
echo ""
|
||||
echo "Configuration:"
|
||||
[ -n "${STATE_ASSERTION}" ] && echo " State assertion: ${STATE_ASSERTION}"
|
||||
[ -n "${START_STATE}" ] && echo " Start state: ${START_STATE}"
|
||||
echo " Clear cache: ${SHOULD_CLEAR}"
|
||||
|
||||
- name: Check S3 State (Before Restore)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_ACCESS_KEY }}
|
||||
run: |
|
||||
echo "=========================================="
|
||||
echo "S3 State Check (Before Restore)"
|
||||
echo "=========================================="
|
||||
echo "Cache key: ${CACHE_KEY}"
|
||||
echo ""
|
||||
|
||||
# Check if base exists
|
||||
BASE_EXISTS=false
|
||||
if aws s3 ls "s3://${S3_BUCKET}/${CACHE_KEY}-base.tar.zst" --region "${S3_REGION}" >/dev/null 2>&1; then
|
||||
BASE_EXISTS=true
|
||||
fi
|
||||
echo "Base exists: ${BASE_EXISTS}"
|
||||
|
||||
# Count deltas
|
||||
DELTA_COUNT=$(aws s3 ls "s3://${S3_BUCKET}/" --region "${S3_REGION}" | grep "${CACHE_KEY}-delta-" | wc -l || echo "0")
|
||||
echo "Delta count: ${DELTA_COUNT}"
|
||||
|
||||
- name: Restore Cache
|
||||
uses: ./.github/actions/xahau-actions-cache-restore
|
||||
with:
|
||||
path: ${{ env.CACHE_DIR }}
|
||||
key: ${{ env.CACHE_KEY }}
|
||||
s3-bucket: ${{ env.S3_BUCKET }}
|
||||
s3-region: ${{ env.S3_REGION }}
|
||||
use-deltas: 'true'
|
||||
aws-access-key-id: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_ACCESS_KEY }}
|
||||
|
||||
- name: Auto-Detect State and Validate
|
||||
id: state
|
||||
env:
|
||||
STATE_ASSERTION: ${{ steps.parse-inputs.outputs.state_assertion }}
|
||||
START_STATE: ${{ steps.parse-inputs.outputs.start_state }}
|
||||
run: |
|
||||
echo "=========================================="
|
||||
echo "State Detection and Validation"
|
||||
echo "=========================================="
|
||||
|
||||
# Create cache directory if it doesn't exist
|
||||
mkdir -p "${CACHE_DIR}"
|
||||
|
||||
# Handle [start-state:N] - force specific state
|
||||
if [ -n "${START_STATE}" ]; then
|
||||
echo "🎯 [start-state:${START_STATE}] detected - forcing state setup"
|
||||
|
||||
# Clear cache and create state files 0 through START_STATE
|
||||
rm -f ${CACHE_DIR}/state*.txt 2>/dev/null || true
|
||||
for i in $(seq 0 ${START_STATE}); do
|
||||
echo "State ${i} - Forced at $(date)" > "${CACHE_DIR}/state${i}.txt"
|
||||
echo "Commit: ${{ github.sha }}" >> "${CACHE_DIR}/state${i}.txt"
|
||||
done
|
||||
|
||||
DETECTED_STATE=${START_STATE}
|
||||
echo "✓ Forced to state ${DETECTED_STATE}"
|
||||
else
|
||||
# Auto-detect state by counting state files
|
||||
STATE_FILES=$(ls ${CACHE_DIR}/state*.txt 2>/dev/null | wc -l)
|
||||
DETECTED_STATE=${STATE_FILES}
|
||||
echo "Auto-detected state: ${DETECTED_STATE} (${STATE_FILES} state files)"
|
||||
fi
|
||||
|
||||
# Show cache contents
|
||||
echo ""
|
||||
echo "Cache contents:"
|
||||
if [ -d "${CACHE_DIR}" ] && [ "$(ls -A ${CACHE_DIR})" ]; then
|
||||
ls -la "${CACHE_DIR}"
|
||||
else
|
||||
echo "(empty)"
|
||||
fi
|
||||
|
||||
# Validate [state:N] assertion if provided
|
||||
if [ -n "${STATE_ASSERTION}" ]; then
|
||||
echo ""
|
||||
echo "Validating assertion: [state:${STATE_ASSERTION}]"
|
||||
if [ "${DETECTED_STATE}" -ne "${STATE_ASSERTION}" ]; then
|
||||
echo "❌ ERROR: State mismatch!"
|
||||
echo " Expected (from [state:N]): ${STATE_ASSERTION}"
|
||||
echo " Detected (from cache): ${DETECTED_STATE}"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ Assertion passed: detected == expected (${DETECTED_STATE})"
|
||||
fi
|
||||
|
||||
# Output detected state for next steps
|
||||
echo "detected_state=${DETECTED_STATE}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
|
||||
- name: Simulate Build (State Transition)
|
||||
env:
|
||||
DETECTED_STATE: ${{ steps.state.outputs.detected_state }}
|
||||
run: |
|
||||
echo "=========================================="
|
||||
echo "Simulating Build (State Transition)"
|
||||
echo "=========================================="
|
||||
|
||||
# Calculate next state
|
||||
NEXT_STATE=$((DETECTED_STATE + 1))
|
||||
echo "Transitioning: State ${DETECTED_STATE} → State ${NEXT_STATE}"
|
||||
echo ""
|
||||
|
||||
# Create state file for next state
|
||||
STATE_FILE="${CACHE_DIR}/state${NEXT_STATE}.txt"
|
||||
echo "State ${NEXT_STATE} - Created at $(date)" > "${STATE_FILE}"
|
||||
echo "Commit: ${{ github.sha }}" >> "${STATE_FILE}"
|
||||
echo "Message: ${{ github.event.head_commit.message }}" >> "${STATE_FILE}"
|
||||
|
||||
echo "✓ Created ${STATE_FILE}"
|
||||
|
||||
# Show final cache state
|
||||
echo ""
|
||||
echo "Final cache contents:"
|
||||
ls -la "${CACHE_DIR}"
|
||||
|
||||
echo ""
|
||||
echo "State files:"
|
||||
cat ${CACHE_DIR}/state*.txt
|
||||
|
||||
- name: Save Cache
|
||||
uses: ./.github/actions/xahau-actions-cache-save
|
||||
with:
|
||||
path: ${{ env.CACHE_DIR }}
|
||||
key: ${{ env.CACHE_KEY }}
|
||||
s3-bucket: ${{ env.S3_BUCKET }}
|
||||
s3-region: ${{ env.S3_REGION }}
|
||||
use-deltas: 'true'
|
||||
aws-access-key-id: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_ACCESS_KEY }}
|
||||
|
||||
- name: Validate S3 State (After Save)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_ACCESS_KEY }}
|
||||
DETECTED_STATE: ${{ steps.state.outputs.detected_state }}
|
||||
run: |
|
||||
echo "=========================================="
|
||||
echo "S3 State Validation (After Save)"
|
||||
echo "=========================================="
|
||||
|
||||
# Calculate next state (what we just saved)
|
||||
NEXT_STATE=$((DETECTED_STATE + 1))
|
||||
echo "Saved state: ${NEXT_STATE}"
|
||||
echo ""
|
||||
|
||||
# Check if base exists
|
||||
if aws s3 ls "s3://${S3_BUCKET}/${CACHE_KEY}-base.tar.zst" --region "${S3_REGION}" >/dev/null 2>&1; then
|
||||
BASE_SIZE=$(aws s3 ls "s3://${S3_BUCKET}/${CACHE_KEY}-base.tar.zst" --region "${S3_REGION}" | awk '{print $3}')
|
||||
echo "✓ Base exists: ${CACHE_KEY}-base.tar.zst (${BASE_SIZE} bytes)"
|
||||
else
|
||||
echo "❌ ERROR: Base should exist after save"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# List deltas
|
||||
echo ""
|
||||
echo "Delta layers:"
|
||||
DELTAS=$(aws s3 ls "s3://${S3_BUCKET}/" --region "${S3_REGION}" | grep "${CACHE_KEY}-delta-" || echo "")
|
||||
if [ -n "${DELTAS}" ]; then
|
||||
echo "${DELTAS}"
|
||||
DELTA_COUNT=$(echo "${DELTAS}" | wc -l)
|
||||
else
|
||||
echo "(none)"
|
||||
DELTA_COUNT=0
|
||||
fi
|
||||
|
||||
# Validate S3 state
|
||||
echo ""
|
||||
if [ "${DETECTED_STATE}" -eq 0 ]; then
|
||||
# Saved state 1 from bootstrap (state 0 → 1)
|
||||
if [ "${DELTA_COUNT}" -ne 0 ]; then
|
||||
echo "⚠️ WARNING: Bootstrap (state 1) should have 0 deltas, found ${DELTA_COUNT}"
|
||||
else
|
||||
echo "✓ State 1 saved: base exists, 0 deltas"
|
||||
fi
|
||||
else
|
||||
# Saved delta (state N+1)
|
||||
if [ "${DELTA_COUNT}" -ne 1 ]; then
|
||||
echo "⚠️ WARNING: State ${NEXT_STATE} expects 1 delta (inline cleanup), found ${DELTA_COUNT}"
|
||||
echo "This might be OK if multiple builds ran concurrently"
|
||||
else
|
||||
echo "✓ State ${NEXT_STATE} saved: base + 1 delta (old deltas cleaned)"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "✅ State ${DETECTED_STATE} → ${NEXT_STATE} Complete!"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Next commit will auto-detect state ${NEXT_STATE}"
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " # Normal (auto-advance)"
|
||||
echo " git commit -m 'continue testing'"
|
||||
echo ""
|
||||
echo " # With assertion (validate state)"
|
||||
echo " git commit -m 'test delta [state:${NEXT_STATE}]'"
|
||||
echo ""
|
||||
echo " # Clear cache and restart"
|
||||
echo " git commit -m 'fresh start [ci-clear-cache]'"
|
||||
echo ""
|
||||
echo " # Jump to specific state"
|
||||
echo " git commit -m 'jump to state 3 [start-state:3]'"
|
||||
182
.github/workflows/test-overlayfs-delta.yml.disabled
vendored
Normal file
182
.github/workflows/test-overlayfs-delta.yml.disabled
vendored
Normal file
@@ -0,0 +1,182 @@
|
||||
name: Test OverlayFS Delta Extraction
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["*"]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test-overlayfs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# - name: Test encrypted secrets (decrypt test message)
|
||||
# run: |
|
||||
# echo "========================================"
|
||||
# echo "TESTING ENCRYPTED SECRETS"
|
||||
# echo "========================================"
|
||||
# echo ""
|
||||
# echo "Decrypting test message from .github/secrets/test-message.gpg"
|
||||
# echo "Using encryption key from GitHub Secrets..."
|
||||
# echo ""
|
||||
#
|
||||
# # Decrypt using key from GitHub Secrets
|
||||
# echo "${{ secrets.TEST_ENCRYPTION_KEY }}" | \
|
||||
# gpg --batch --yes --passphrase-fd 0 \
|
||||
# --decrypt .github/secrets/test-message.gpg
|
||||
#
|
||||
# echo ""
|
||||
# echo "========================================"
|
||||
# echo "If you see the success message above,"
|
||||
# echo "then encrypted secrets work! 🎉"
|
||||
# echo "========================================"
|
||||
# echo ""
|
||||
|
||||
- name: Setup OverlayFS layers
|
||||
run: |
|
||||
echo "=== Creating directory structure ==="
|
||||
mkdir -p /tmp/test/{base,delta,upper,work,merged}
|
||||
|
||||
echo "=== Creating base layer files ==="
|
||||
echo "base file 1" > /tmp/test/base/file1.txt
|
||||
echo "base file 2" > /tmp/test/base/file2.txt
|
||||
echo "base file 3" > /tmp/test/base/file3.txt
|
||||
mkdir -p /tmp/test/base/subdir
|
||||
echo "base subdir file" > /tmp/test/base/subdir/file.txt
|
||||
|
||||
echo "=== Base layer contents ==="
|
||||
find /tmp/test/base -type f -exec sh -c 'echo "{}:"; cat "{}"' \;
|
||||
|
||||
echo "=== Mounting OverlayFS ==="
|
||||
sudo mount -t overlay overlay \
|
||||
-o lowerdir=/tmp/test/base,upperdir=/tmp/test/upper,workdir=/tmp/test/work \
|
||||
/tmp/test/merged
|
||||
|
||||
echo "=== Mounted successfully ==="
|
||||
mount | grep overlay
|
||||
|
||||
- name: Verify merged view shows base files
|
||||
run: |
|
||||
echo "=== Contents of /merged (should show base files) ==="
|
||||
ls -R /tmp/test/merged
|
||||
find /tmp/test/merged -type f -exec sh -c 'echo "{}:"; cat "{}"' \;
|
||||
|
||||
- name: Make changes via merged layer
|
||||
run: |
|
||||
echo "=== Making changes via /merged ==="
|
||||
|
||||
# Overwrite existing file
|
||||
echo "MODIFIED file 2" > /tmp/test/merged/file2.txt
|
||||
echo "Modified file2.txt"
|
||||
|
||||
# Create new file
|
||||
echo "NEW file 4" > /tmp/test/merged/file4.txt
|
||||
echo "Created new file4.txt"
|
||||
|
||||
# Create new directory with file
|
||||
mkdir -p /tmp/test/merged/newdir
|
||||
echo "NEW file in new dir" > /tmp/test/merged/newdir/newfile.txt
|
||||
echo "Created newdir/newfile.txt"
|
||||
|
||||
# Add file to existing directory
|
||||
echo "NEW file in existing subdir" > /tmp/test/merged/subdir/newfile.txt
|
||||
echo "Created subdir/newfile.txt"
|
||||
|
||||
echo "=== Changes complete ==="
|
||||
|
||||
- name: Show the delta (upperdir)
|
||||
run: |
|
||||
echo "========================================"
|
||||
echo "THE DELTA (only changes in /upper):"
|
||||
echo "========================================"
|
||||
|
||||
if [ -z "$(ls -A /tmp/test/upper)" ]; then
|
||||
echo "Upper directory is empty - no changes detected"
|
||||
else
|
||||
echo "Upper directory structure:"
|
||||
ls -R /tmp/test/upper
|
||||
echo ""
|
||||
echo "Upper directory files with content:"
|
||||
find /tmp/test/upper -type f -exec sh -c 'echo "---"; echo "FILE: {}"; cat "{}"; echo ""' \;
|
||||
|
||||
echo "========================================"
|
||||
echo "SIZE OF DELTA:"
|
||||
du -sh /tmp/test/upper
|
||||
echo "========================================"
|
||||
fi
|
||||
|
||||
- name: Compare base vs upper vs merged
|
||||
run: |
|
||||
echo "========================================"
|
||||
echo "COMPARISON:"
|
||||
echo "========================================"
|
||||
|
||||
echo "BASE layer (original, untouched):"
|
||||
ls -la /tmp/test/base/
|
||||
echo ""
|
||||
|
||||
echo "UPPER layer (DELTA - only changes):"
|
||||
ls -la /tmp/test/upper/
|
||||
echo ""
|
||||
|
||||
echo "MERGED layer (unified view = base + upper):"
|
||||
ls -la /tmp/test/merged/
|
||||
echo ""
|
||||
|
||||
echo "========================================"
|
||||
echo "PROOF: Upper dir contains ONLY the delta!"
|
||||
echo "========================================"
|
||||
|
||||
- name: Simulate tarball creation (what we'd upload)
|
||||
run: |
|
||||
echo "=== Creating tarball of delta ==="
|
||||
tar -czf /tmp/delta.tar.gz -C /tmp/test/upper .
|
||||
|
||||
echo "Delta tarball size:"
|
||||
ls -lh /tmp/delta.tar.gz
|
||||
|
||||
echo ""
|
||||
echo "Delta tarball contents:"
|
||||
tar -tzf /tmp/delta.tar.gz
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "This is what we'd upload to S3/rsync!"
|
||||
echo "Only ~few KB instead of entire cache!"
|
||||
echo "========================================"
|
||||
|
||||
- name: Upload delta to S3 (actual test!)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.XAHAUD_GITHUB_ACTIONS_CACHE_NIQ_AWS_ACCESS_KEY }}
|
||||
run: |
|
||||
echo "========================================"
|
||||
echo "UPLOADING TO S3"
|
||||
echo "========================================"
|
||||
|
||||
# Upload the delta tarball
|
||||
aws s3 cp /tmp/delta.tar.gz \
|
||||
s3://xahaud-github-actions-cache-niq/hello-world-first-test.tar.gz \
|
||||
--region us-east-1
|
||||
|
||||
echo ""
|
||||
echo "✅ Successfully uploaded to S3!"
|
||||
echo "File: s3://xahaud-github-actions-cache-niq/hello-world-first-test.tar.gz"
|
||||
echo ""
|
||||
|
||||
# Verify it exists
|
||||
echo "Verifying upload..."
|
||||
aws s3 ls s3://xahaud-github-actions-cache-niq/hello-world-first-test.tar.gz --region us-east-1
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "S3 upload test complete! 🚀"
|
||||
echo "========================================"
|
||||
|
||||
- name: Cleanup
|
||||
if: always()
|
||||
run: |
|
||||
echo "=== Unmounting OverlayFS ==="
|
||||
sudo umount /tmp/test/merged || true
|
||||
@@ -30,14 +30,6 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Get commit message
|
||||
id: get-commit-message
|
||||
uses: ./.github/actions/xahau-ga-get-commit-message
|
||||
with:
|
||||
event-name: ${{ github.event_name }}
|
||||
head-commit-message: ${{ github.event.head_commit.message }}
|
||||
pr-head-sha: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Install Conan
|
||||
run: |
|
||||
brew install conan
|
||||
8
.github/workflows/xahau-ga-nix.yml
vendored
8
.github/workflows/xahau-ga-nix.yml
vendored
@@ -162,14 +162,6 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Get commit message
|
||||
id: get-commit-message
|
||||
uses: ./.github/actions/xahau-ga-get-commit-message
|
||||
with:
|
||||
event-name: ${{ github.event_name }}
|
||||
head-commit-message: ${{ github.event.head_commit.message }}
|
||||
pr-head-sha: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Install build dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
|
||||
@@ -72,15 +72,15 @@ It generates many files of [results](results):
|
||||
desired as described above. In a perfect repo, this file will be
|
||||
empty.
|
||||
This file is committed to the repo, and is used by the [levelization
|
||||
Github workflow](../../.github/workflows/levelization.yml) to validate
|
||||
Github workflow](../../.github/workflows/levelization.yml.disabled) to validate
|
||||
that nothing changed.
|
||||
* [`ordering.txt`](results/ordering.txt): A list showing relationships
|
||||
between modules where there are no loops as they actually exist, as
|
||||
opposed to how they are desired as described above.
|
||||
This file is committed to the repo, and is used by the [levelization
|
||||
Github workflow](../../.github/workflows/levelization.yml) to validate
|
||||
Github workflow](../../.github/workflows/levelization.yml.disabled) to validate
|
||||
that nothing changed.
|
||||
* [`levelization.yml`](../../.github/workflows/levelization.yml)
|
||||
* [`levelization.yml`](../../.github/workflows/levelization.yml.disabled)
|
||||
Github Actions workflow to test that levelization loops haven't
|
||||
changed. Unfortunately, if changes are detected, it can't tell if
|
||||
they are improvements or not, so if you have resolved any issues or
|
||||
|
||||
Reference in New Issue
Block a user