Compare commits

..

8 Commits

Author SHA1 Message Date
Niq Dudfield
fa4ad19f9b Merge branch 'dev' into nd-use-github-actions-cache-with-bug-fixes-from-s3-implementation-2025-11-24 2025-11-25 08:12:47 +07:00
Nicholas Dudfield
ee71cd74cf refactor: use GitHub API instead of git fetch for PR commit messages 2025-11-25 08:09:02 +07:00
Nicholas Dudfield
e2b7b18c1d fix: fetch PR commit message via GitHub API for matrix-setup [ci-nix-full-matrix]
The matrix-setup job was using github.event.head_commit.message which
is empty for PR events. Now fetches via API using the PR head SHA.
2025-11-25 08:02:26 +07:00
Nicholas Dudfield
525f96aa43 chore: bump macos cache version to 3 2025-11-25 07:48:14 +07:00
Nicholas Dudfield
430155587e chore: upgrade macos runner to xlarge (M2)
Upgrades macOS CI runner from macos-15 to macos-15-xlarge for faster
builds with more CPU cores.

Changes:
- runs-on: macos-15-xlarge (was: macos-15)

Benefits:
- M2 processor with more cores
- Faster compilation times
- Better ccache performance
2025-11-24 15:24:27 +07:00
Nicholas Dudfield
49c1e337e8 refactor: inline cache clearing into restore action
Consolidates cache clearing and restore into a single self-contained
action, eliminating the need for a separate clear-cache action.

Changes:
- Inline cache clearing logic directly into xahau-ga-cache-restore
- Delete xahau-ga-clear-cache action (no longer needed)
- Add additional-clear-keys input for clearing multiple keys
- Remove nd-experiment-overlayfs branch from workflow triggers

Benefits:
- Single action does everything (clear + restore)
- Simpler architecture (one action instead of two)
- Self-contained and easier to maintain
2025-11-24 15:21:22 +07:00
Nicholas Dudfield
c5972875c4 refactor: create generic cache restore action (DRY)
Consolidates cache clearing + restore logic into a single reusable
action to eliminate key synthesis duplication.

Changes:
- New xahau-ga-cache-restore action (wraps clear + restore)
- Integrates cache clearing directly into restore operation
- Eliminates separate cache clearing steps
- Cache keys built once in restore action (not duplicated)

Usage:
- ccache: two restore calls (main + current branch dual-cache)
- Conan: one restore call (single cache)
- Saves use actions/cache/save@v4 directly (no wrapper needed)

Benefits:
- DRY: cache keys no longer synthesized in multiple places
- Cleaner: cache clearing automatic during restore
- Simpler: one action call instead of two steps per cache
2025-11-24 15:15:44 +07:00
Nicholas Dudfield
f8bdb57f2e feat: improve ci caching with dual-cache system and api-based clearing
Implements enhanced caching strategy for GitHub Actions CI workflows
with efficient cache management and commit-message-driven controls.

Dual-cache system:
- Main branch: uses ~/.ccache-main (read-write)
- Feature branches: use ~/.ccache-current (read-write) + ~/.ccache-main
  as secondary_storage (read-only fallback)
- Feature branches only save new compilation artifacts, inheriting
  everything else from main branch cache
- Eliminates cache duplication across branches

Cache clearing via GitHub API:
- New xahau-ga-clear-cache reusable action
- [ci-ga-clear-cache] - Clear all caches for this job
- [ci-ga-clear-cache:ccache] - Clear only if key contains "ccache"
- [ci-ga-clear-cache:conan] - Clear only if key contains "conan"
- [ci-ga-clear-cache:gcc Debug] - Clear if key contains both (AND logic)
- Uses gh CLI with github.token (no additional credentials)
- Cache deletion happens before restore step

ccache improvements:
- Wrapper toolchain prevents Conan from overriding ccache settings
- Single cache directory approach with proper secondary_storage config
- Configuration applied AFTER cache restore (prevents stale config)
- Verbose build flag: [ci-ga-cmake-verbose]

Conan improvements:
- Configuration applied AFTER cache restore (prevents stale profile)
- Removed ineffective branch comparison logic for cache saves
- Fixed cache keys to use conanfile.py (not conanfile.txt)

Breaking changes:
- Cache version bumped to v3 (invalidates old caches)

Fixes #620 (ccache caches were empty)
Fixes #618 (conan not caching on feature branches)
2025-11-24 15:01:31 +07:00
3 changed files with 19 additions and 44 deletions

View File

@@ -20,7 +20,7 @@ jobs:
- Ninja - Ninja
configuration: configuration:
- Debug - Debug
runs-on: macos-15 runs-on: macos-15-xlarge
env: env:
build_dir: .build build_dir: .build
# Bump this number to invalidate all caches globally. # Bump this number to invalidate all caches globally.

View File

@@ -19,14 +19,6 @@ jobs:
outputs: outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }} matrix: ${{ steps.set-matrix.outputs.matrix }}
steps: steps:
- name: escape double quotes
id: escape
shell: bash
env:
PR_TITLE: ${{ github.event.pull_request.title }}
run: |
ESCAPED_PR_TITLE="${PR_TITLE//\"/\\\"}"
echo "title=${ESCAPED_PR_TITLE}" >> "$GITHUB_OUTPUT"
- name: Generate build matrix - name: Generate build matrix
id: set-matrix id: set-matrix
shell: python shell: python
@@ -110,7 +102,7 @@ jobs:
ref = "${{ github.ref }}" ref = "${{ github.ref }}"
base_ref = "${{ github.base_ref }}" # For PRs, this is the target branch base_ref = "${{ github.base_ref }}" # For PRs, this is the target branch
event_name = "${{ github.event_name }}" event_name = "${{ github.event_name }}"
pr_title = """${{ steps.escape.outputs.title }}""" pr_title = """${{ github.event.pull_request.title }}"""
pr_head_sha = "${{ github.event.pull_request.head.sha }}" pr_head_sha = "${{ github.event.pull_request.head.sha }}"
# Get commit message - for PRs, fetch via API since head_commit.message is empty # Get commit message - for PRs, fetch via API since head_commit.message is empty

View File

@@ -484,61 +484,44 @@ OverlayImpl::start()
m_peerFinder->setConfig(config); m_peerFinder->setConfig(config);
m_peerFinder->start(); m_peerFinder->start();
auto addIps = [this](std::vector<std::string> ips, bool fixed) { auto addIps = [&](std::vector<std::string> bootstrapIps) -> void {
beast::Journal const& j = app_.journal("Overlay"); beast::Journal const& j = app_.journal("Overlay");
for (auto& ip : ips) for (auto& ip : bootstrapIps)
{ {
std::size_t pos = ip.find('#'); std::size_t pos = ip.find('#');
if (pos != std::string::npos) if (pos != std::string::npos)
ip.erase(pos); ip.erase(pos);
JLOG(j.trace()) JLOG(j.trace()) << "Found boostrap IP: " << ip;
<< "Found " << (fixed ? "fixed" : "bootstrap") << " IP: " << ip;
} }
m_resolver.resolve( m_resolver.resolve(
ips, bootstrapIps,
[this, fixed]( [&](std::string const& name,
std::string const& name,
std::vector<beast::IP::Endpoint> const& addresses) { std::vector<beast::IP::Endpoint> const& addresses) {
std::vector<std::string> ips;
ips.reserve(addresses.size());
beast::Journal const& j = app_.journal("Overlay"); beast::Journal const& j = app_.journal("Overlay");
std::string const base("config: ");
std::vector<beast::IP::Endpoint> eps;
eps.reserve(addresses.size());
for (auto const& addr : addresses) for (auto const& addr : addresses)
{ {
auto ep = addr.port() == 0 ? addr.at_port(DEFAULT_PEER_PORT) std::string addrStr = addr.port() == 0
: addr; ? to_string(addr.at_port(DEFAULT_PEER_PORT))
JLOG(j.trace()) : to_string(addr);
<< "Parsed " << (fixed ? "fixed" : "bootstrap") JLOG(j.trace()) << "Parsed boostrap IP: " << addrStr;
<< " IP: " << ep; ips.push_back(addrStr);
eps.push_back(ep);
} }
if (eps.empty()) std::string const base("config: ");
return; if (!ips.empty())
m_peerFinder->addFallbackStrings(base + name, ips);
if (fixed)
{
m_peerFinder->addFixedPeer(base + name, eps);
}
else
{
std::vector<std::string> strs;
strs.reserve(eps.size());
for (auto const& ep : eps)
strs.push_back(to_string(ep));
m_peerFinder->addFallbackStrings(base + name, strs);
}
}); });
}; };
if (!app_.config().IPS.empty()) if (!app_.config().IPS.empty())
addIps(app_.config().IPS, false); addIps(app_.config().IPS);
if (!app_.config().IPS_FIXED.empty()) if (!app_.config().IPS_FIXED.empty())
addIps(app_.config().IPS_FIXED, true); addIps(app_.config().IPS_FIXED);
auto const timer = std::make_shared<Timer>(*this); auto const timer = std::make_shared<Timer>(*this);
std::lock_guard lock(mutex_); std::lock_guard lock(mutex_);