ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-07-23 23:20:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' into bthomee/version

Browse Source
This commit is contained in:
Ayaz Salikhov
2026-07-15 21:34:12 +01:00
committed by GitHub
parent e9311e5ea2 781ab723af
commit 60ac072dac
6 changed files with 143 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
.github/workflows/on-pr.yml vendored
View File

@@ -25,18 +25,11 @@ on:
- unlabeled
concurrency:
# Use a per-ref group so a newer run (a push, or a change to a label below)
# supersedes the in-progress one for that ref. Label events we don't act on get
# their own unique group (per run id) instead, keeping them out of the shared
# group so real builds keep running. Keep this list in sync with `should-run`.
group: >-
${{ github.workflow }}-${{ github.ref }}${{
((github.event.action == 'labeled' || github.event.action == 'unlabeled')
&& github.event.label.name != 'Ready to merge'
&& github.event.label.name != 'DraftRunCI'
&& github.event.label.name != 'Full CI build')
&& format('-{0}', github.run_id) || ''
}}
# A single per-ref group with cancel-in-progress means any newer run (a push
# or a label change) supersedes the in-progress one for that ref. Keeping
# exactly one authoritative run per ref ensures a fast do-nothing run can never
# mask a real build's checks.
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
defaults:
@@ -44,21 +37,17 @@ defaults:
shell: bash
jobs:
# This job determines whether the rest of the workflow should run. It runs
# when the PR is not a draft (which should also cover merge-group) or has the
# 'DraftRunCI' or 'Full CI build' label. For label events it only runs when the
# label added or removed is one we act on ('Ready to merge', 'DraftRunCI' or
# 'Full CI build'), so unrelated label changes do not trigger a redundant run.
# This job determines whether the rest of the workflow should run at all,
# based on the current set of labels: it runs when the PR is not a draft
# (which should also cover merge-group) or has the 'DraftRunCI' or
# 'Full CI build' label. Whether a build then happens, and whether it is the
# minimal or full matrix, is decided further below and in the strategy matrix.
should-run:
if: >-
${{
((github.event.action != 'labeled' && github.event.action != 'unlabeled')
|| github.event.label.name == 'Ready to merge'
|| github.event.label.name == 'DraftRunCI'
|| github.event.label.name == 'Full CI build')
&& (!github.event.pull_request.draft
|| contains(github.event.pull_request.labels.*.name, 'DraftRunCI')
|| contains(github.event.pull_request.labels.*.name, 'Full CI build'))
!github.event.pull_request.draft
|| contains(github.event.pull_request.labels.*.name, 'DraftRunCI')
|| contains(github.event.pull_request.labels.*.name, 'Full CI build')
}}
runs-on: ubuntu-latest
steps:

41
include/xrpl/basics/TaggedCache.ipp
View File

@@ -3,6 +3,9 @@
#include <xrpl/basics/IntrusivePointer.ipp>
#include <xrpl/basics/Log.h> // IWYU pragma: keep
#include <xrpl/basics/TaggedCache.h>
#include <xrpl/basics/scope.h>
#include <algorithm>
namespace xrpl {
@@ -601,8 +604,42 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
std::vector<key_type> v;
{
std::scoped_lock const lock(mutex_);
v.reserve(cache_.size());
// Keep track of how many iterations are needed. Exit the loop if the number of retries gets
// absurd. (Note that if this somehow ever happens, one more allocation will be done under
// lock, which is undesirable, but really should be almost impossible.)
std::size_t allocationIterations = 0;
std::unique_lock lock(mutex_);
for (auto size = cache_.size(); v.capacity() < size && allocationIterations < 20;
size = cache_.size())
{
ScopeUnlock const unlock(lock);
if (allocationIterations > 0)
{
JLOG(journal_.info())
<< "getKeys(): Cache grew beyond allocated capacity after "
<< allocationIterations << " prior attempt(s). Have " << v.capacity()
<< ", need " << size << ". Retrying allocation";
}
// Allocate the current size plus a little extra, in case the cache grows while
// allocating. Each time another allocation is needed, the extra also gets bigger until
// it ultimately doubles the size + 1.
constexpr std::size_t baseShift = 5;
auto const bufferOffset = std::min(allocationIterations, std::size_t{baseShift});
auto const bufferShift = baseShift - bufferOffset;
size += (size >> bufferShift) + 1;
v.reserve(size);
++allocationIterations;
}
if (v.capacity() < cache_.size())
{
// LCOV_EXCL_START
UNREACHABLE("xrpl::TaggedCache::getKeys(): failed to allocate sufficient capacity");
v.reserve(cache_.size());
// LCOV_EXCL_STOP
}
XRPL_ASSERT(lock.owns_lock(), "xrpl::TaggedCache::getKeys(): owns lock");
XRPL_ASSERT(
v.capacity() >= cache_.size(), "xrpl::TaggedCache::getKeys(): sufficient capacity");
for (auto const& _ : cache_)
v.push_back(_.first);
}

13
include/xrpl/nodestore/DatabaseRotating.h
View File

@@ -41,6 +41,19 @@ public:
std::unique_ptr<NodeStore::Backend>&& newBackend,
std::function<void(std::string const& writableName, std::string const& archiveName)> const&
f) = 0;
/**
* Marks an online-delete rotation as in progress (or completed).
*
* While in flight, a read served by the archive backend is copied
* forward into the writable backend even for ordinary
* (duplicate == false) fetches: the archive is about to be deleted,
* and a node body canonicalized into caches during the rotation
* window would otherwise survive only in RAM once the archive is
* dropped.
*/
virtual void
setRotationInFlight(bool inFlight) = 0;
};
} // namespace xrpl::NodeStore

12
include/xrpl/nodestore/detail/DatabaseRotatingImp.h
View File

@@ -9,6 +9,7 @@
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Scheduler.h>
#include <atomic>
#include <cstdint>
#include <functional>
#include <memory>
@@ -69,11 +70,22 @@ public:
void
sweep() override;
void
setRotationInFlight(bool inFlight) override;
private:
std::shared_ptr<Backend> writableBackend_;
std::shared_ptr<Backend> archiveBackend_;
mutable std::mutex mutex_;
// True between SHAMapStore starting the cache-freshen phase and the
// completion of rotate(). While true, archive hits on ordinary
// (duplicate == false) fetches are copied forward into the writable
// backend; copyForwardCount_ tallies them per rotation for the
// summary line logged at swap.
std::atomic<bool> rotationInFlight_{false};
std::atomic<std::uint64_t> copyForwardCount_{0};
std::shared_ptr<NodeObject>
fetchNodeObject(uint256 const& hash, std::uint32_t, FetchReport& fetchReport, bool duplicate)
override;

32
src/libxrpl/nodestore/DatabaseRotatingImp.cpp
View File

@@ -13,6 +13,7 @@
#include <xrpl/nodestore/Scheduler.h>
#include <xrpl/nodestore/Types.h>
#include <atomic>
#include <cstdint>
#include <exception>
#include <functional>
@@ -52,6 +53,7 @@ DatabaseRotatingImp::rotate(
// callback finishes. Only then will the archive directory be
// deleted.
std::shared_ptr<NodeStore::Backend> oldArchiveBackend;
std::uint64_t copyForwards = 0;
{
std::scoped_lock const lock(mutex_);
@@ -62,11 +64,28 @@ DatabaseRotatingImp::rotate(
newArchiveBackendName = archiveBackend_->getName();
writableBackend_ = std::move(newBackend);
copyForwards = copyForwardCount_.exchange(0, std::memory_order_relaxed);
}
if (copyForwards > 0)
{
JLOG(j_.warn()) << "Rotating: copied forward " << copyForwards
<< " archive-served reads into the writable backend "
"during the rotation window";
}
f(newWritableBackendName, newArchiveBackendName);
}
void
DatabaseRotatingImp::setRotationInFlight(bool inFlight)
{
rotationInFlight_.store(inFlight, std::memory_order_release);
JLOG(j_.debug()) << "Rotating: copy-forward on archive reads "
<< (inFlight ? "enabled" : "disabled");
}
std::string
DatabaseRotatingImp::getName() const
{
@@ -177,9 +196,18 @@ DatabaseRotatingImp::fetchNodeObject(
writable = writableBackend_;
}
// Update writable backend with data from the archive backend
if (duplicate)
// Update writable backend with data from the archive backend.
// While a rotation is in flight, ordinary (duplicate == false)
// reads served by the archive are copied forward too: the
// archive is about to be deleted, and a body canonicalized
// into the cache after the freshen getKeys() snapshot would
// otherwise survive only in RAM once the archive is dropped.
if (duplicate || rotationInFlight_.load(std::memory_order_acquire))
{
if (!duplicate)
copyForwardCount_.fetch_add(1, std::memory_order_relaxed);
writable->store(nodeObject);
}
}
}

37
src/xrpld/app/misc/SHAMapStoreImp.cpp
View File

@@ -16,9 +16,11 @@
#include <xrpl/ledger/Ledger.h>
#include <xrpl/nodestore/Database.h>
#include <xrpl/nodestore/Manager.h>
#include <xrpl/nodestore/NodeObject.h>
#include <xrpl/nodestore/Scheduler.h>
#include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
#include <xrpl/protocol/Protocol.h>
#include <xrpl/protocol/Serializer.h>
#include <xrpl/server/NetworkOPs.h>
#include <xrpl/server/State.h>
#include <xrpl/shamap/SHAMapMissingNode.h>
@@ -254,8 +256,24 @@ bool
SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node)
{
// Copy a single record from node to dbRotating_
dbRotating_->fetchNodeObject(
auto obj = dbRotating_->fetchNodeObject(
node.getHash().asUInt256(), 0, NodeStore::FetchType::Synchronous, true);
if (!obj)
{
XRPL_ASSERT(node.cowid() == 0, "SHAMapStoreImp::copyNode : rescued node must be clean");
// Reachable from the validated state map in memory, but present in
// neither backend: its only on-disk copy lived in a backend removed by
// an earlier rotation, and it was never rewritten because it is clean
// (cowid == 0, so flushDirty skips it). Persist the in-memory body
// directly into the writable backend so it survives this rotation
// instead of later surfacing as an unresolvable SHAMapMissingNode.
auto const hash = node.getHash().asUInt256();
Serializer s;
node.serializeWithPrefix(s);
dbRotating_->store(NodeObjectType::AccountNode, std::move(s.modData()), hash, 0);
JLOG(journal_.warn()) << "copyNode: re-stored node missing from both backends, hash="
<< hash << " type=" << static_cast<int>(node.getType());
}
if ((++nodeCount % checkHealthInterval_) == 0u)
{
if (healthWait() == HealthResult::Stopping)
@@ -348,6 +366,23 @@ SHAMapStoreImp::run()
JLOG(journal_.debug())
<< "copied ledger " << validatedSeq << " nodecount " << nodeCount;
// Close the getKeys()->swap exposure window: from here until
// rotate() completes, an ordinary read served by the archive is
// copied forward into the writable backend, so a node fetched
// from the doomed archive cannot be left RAM-only when the
// archive is deleted. RAII so the early returns below (and any
// exception) also clear the flag.
struct RotationExposureGuard
{
NodeStore::DatabaseRotating& db;
~RotationExposureGuard()
{
db.setRotationInFlight(false);
}
};
RotationExposureGuard const rotationExposureGuard{*dbRotating_};
dbRotating_->setRotationInFlight(true);
JLOG(journal_.debug()) << "freshening caches";
freshenCaches();
if (healthWait() == HealthResult::Stopping)