mirror of
https://github.com/XRPLF/rippled.git
synced 2026-07-23 15:10:34 +00:00
Compare commits
95 Commits
ximinez/on
...
ximinez/as
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
959b1f4c1a | ||
|
|
42c0368f46 | ||
|
|
5e90caa4fa | ||
|
|
7abff407b6 | ||
|
|
65b3cab13c | ||
|
|
c3a17d40da | ||
|
|
8bbe74cfbe | ||
|
|
c2b6cbc2ba | ||
|
|
6c1ecb0bf1 | ||
|
|
6b77a258df | ||
|
|
677acd09ae | ||
|
|
31b206b802 | ||
|
|
43bbedae3b | ||
|
|
4f6831b11d | ||
|
|
da6e767865 | ||
|
|
d0e9e483f8 | ||
|
|
1f593f2efa | ||
|
|
655ac9b1f8 | ||
|
|
557697a492 | ||
|
|
09b13c0218 | ||
|
|
cf83782225 | ||
|
|
dffa0a0910 | ||
|
|
8c56202c90 | ||
|
|
ba7d28f444 | ||
|
|
f2591b1651 | ||
|
|
5bcaa0af12 | ||
|
|
60700c5cbb | ||
|
|
452f6ea046 | ||
|
|
8a0e45c055 | ||
|
|
2b7ea64167 | ||
|
|
8ea3db6299 | ||
|
|
5b7f5153e4 | ||
|
|
dc931df242 | ||
|
|
ec2a5ab620 | ||
|
|
199fdeb09d | ||
|
|
cbd2855305 | ||
|
|
36fe2ea590 | ||
|
|
07fef06a53 | ||
|
|
ec91ad1687 | ||
|
|
35744453d5 | ||
|
|
60252ab648 | ||
|
|
88cc3dd15a | ||
|
|
2a9d394b69 | ||
|
|
2fa8305e95 | ||
|
|
a711ebc358 | ||
|
|
3dd44ae18a | ||
|
|
9eaebf2eea | ||
|
|
664b9ba5fe | ||
|
|
189f2d60bd | ||
|
|
beb8a1872d | ||
|
|
24db40e56c | ||
|
|
da4513d096 | ||
|
|
2e2fddefe9 | ||
|
|
db997ecad9 | ||
|
|
b7d9f5a357 | ||
|
|
68e4fbdf2b | ||
|
|
6acf730b9b | ||
|
|
bb0a09ae21 | ||
|
|
d94232007f | ||
|
|
df8bfbe5af | ||
|
|
347d1a19ef | ||
|
|
d65fab27a1 | ||
|
|
b5d25c5ab1 | ||
|
|
7222150095 | ||
|
|
a6519011d2 | ||
|
|
a67da5c2ed | ||
|
|
5c1b94f1d3 | ||
|
|
d2f23b2f5b | ||
|
|
4067e5025f | ||
|
|
0017eb345a | ||
|
|
ed4330a7d6 | ||
|
|
c2ca6907d4 | ||
|
|
feba605998 | ||
|
|
b322097529 | ||
|
|
e159d27373 | ||
|
|
ba53026006 | ||
|
|
34773080df | ||
|
|
3a30099db6 | ||
|
|
3c3bd75991 | ||
|
|
6766a176a3 | ||
|
|
7459fe454d | ||
|
|
ecb2ba5e19 | ||
|
|
106bf48725 | ||
|
|
74c968d4e3 | ||
|
|
167147281c | ||
|
|
ba60306610 | ||
|
|
6674500896 | ||
|
|
c5d7ebe93d | ||
|
|
8b4404b8c6 | ||
|
|
bfc0acc734 | ||
|
|
9f7f43c16e | ||
|
|
8e98ba7564 | ||
|
|
d0b5ca9dab | ||
|
|
5e51893e9b | ||
|
|
3422c11d02 |
@@ -1059,11 +1059,10 @@
|
||||
# The online delete process checks periodically
|
||||
# that xrpld is still in sync with the network,
|
||||
# and that the validated ledger is less than
|
||||
# 'age_threshold_seconds' old, and that all
|
||||
# recent ledgers are available. If not, then continue
|
||||
# 'age_threshold_seconds' old. If not, then continue
|
||||
# sleeping for this number of seconds and
|
||||
# checking until healthy.
|
||||
# Default is 2.
|
||||
# Default is 5.
|
||||
#
|
||||
# Notes:
|
||||
# The 'node_db' entry configures the primary, persistent storage.
|
||||
|
||||
@@ -623,14 +623,10 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
|
||||
// Allocate the current size plus a little extra, in case the cache grows while
|
||||
// allocating. Each time another allocation is needed, the extra also gets bigger until
|
||||
// it ultimately doubles the size + 1.
|
||||
|
||||
// Temporary experimental value. Make the buffer as small as possible to see if any
|
||||
// reallocations are needed. Do not merge.
|
||||
constexpr std::size_t baseShift = 63;
|
||||
constexpr std::size_t baseShift = 5;
|
||||
auto const bufferOffset = std::min(allocationIterations, std::size_t{baseShift});
|
||||
auto const bufferShift = baseShift - bufferOffset;
|
||||
auto const buffer = (size >> bufferShift) + 1;
|
||||
size += buffer;
|
||||
size += (size >> bufferShift) + 1;
|
||||
v.reserve(size);
|
||||
++allocationIterations;
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
#include <xrpl/nodestore/Backend.h>
|
||||
#include <xrpl/nodestore/Database.h>
|
||||
#include <xrpl/nodestore/Scheduler.h>
|
||||
#include <xrpl/protocol/Protocol.h>
|
||||
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
@@ -46,15 +45,15 @@ public:
|
||||
/**
|
||||
* Marks an online-delete rotation as in progress (or completed).
|
||||
*
|
||||
* While in flight, a read for ledgers after the inFlight value served by the archive backend is
|
||||
* copied forward into the writable backend even for ordinary (duplicate == false) fetches: the
|
||||
* archive is about to be deleted, and a node body canonicalized into caches during the rotation
|
||||
* window would otherwise survive only in RAM once the archive is dropped.
|
||||
* While in flight, a read served by the archive backend is copied
|
||||
* forward into the writable backend even for ordinary
|
||||
* (duplicate == false) fetches: the archive is about to be deleted,
|
||||
* and a node body canonicalized into caches during the rotation
|
||||
* window would otherwise survive only in RAM once the archive is
|
||||
* dropped.
|
||||
*/
|
||||
virtual void
|
||||
setRotationInFlight(LedgerIndex inFlight) = 0;
|
||||
virtual LedgerIndex
|
||||
getRotationInFlight() const = 0;
|
||||
setRotationInFlight(bool inFlight) = 0;
|
||||
};
|
||||
|
||||
} // namespace xrpl::NodeStore
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
#include <xrpl/nodestore/DatabaseRotating.h>
|
||||
#include <xrpl/nodestore/NodeObject.h>
|
||||
#include <xrpl/nodestore/Scheduler.h>
|
||||
#include <xrpl/protocol/Protocol.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
@@ -72,27 +71,20 @@ public:
|
||||
sweep() override;
|
||||
|
||||
void
|
||||
setRotationInFlight(LedgerIndex inFlight) override;
|
||||
LedgerIndex
|
||||
getRotationInFlight() const override;
|
||||
setRotationInFlight(bool inFlight) override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<Backend> writableBackend_;
|
||||
std::shared_ptr<Backend> archiveBackend_;
|
||||
mutable std::mutex mutex_;
|
||||
|
||||
// Set to the index of the last rotated ledger between SHAMapStore
|
||||
// starting the cache-freshen phase and the completion of rotate().
|
||||
// While non-zero, archive hits on ordinary (duplicate == false)
|
||||
// fetches are copied forward into the writable backend if they are
|
||||
// for that ledger or later, since those are the ones we'll keep.
|
||||
// To be safe, copy forward if the provided ledger index is 0.
|
||||
// copyForwardCount_ tallies them per rotation for the
|
||||
// True between SHAMapStore starting the cache-freshen phase and the
|
||||
// completion of rotate(). While true, archive hits on ordinary
|
||||
// (duplicate == false) fetches are copied forward into the writable
|
||||
// backend; copyForwardCount_ tallies them per rotation for the
|
||||
// summary line logged at swap.
|
||||
// copyRejectCount_ tallies the ones that weren't copied.
|
||||
std::atomic<LedgerIndex> rotationInFlight_{0};
|
||||
std::atomic<bool> rotationInFlight_{false};
|
||||
std::atomic<std::uint64_t> copyForwardCount_{0};
|
||||
std::atomic<std::uint64_t> copyRejectCount_{0};
|
||||
|
||||
std::shared_ptr<NodeObject>
|
||||
fetchNodeObject(uint256 const& hash, std::uint32_t, FetchReport& fetchReport, bool duplicate)
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
#include <xrpl/nodestore/NodeObject.h>
|
||||
#include <xrpl/nodestore/Scheduler.h>
|
||||
#include <xrpl/nodestore/Types.h>
|
||||
#include <xrpl/protocol/Protocol.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
@@ -55,7 +54,6 @@ DatabaseRotatingImp::rotate(
|
||||
// deleted.
|
||||
std::shared_ptr<NodeStore::Backend> oldArchiveBackend;
|
||||
std::uint64_t copyForwards = 0;
|
||||
std::uint64_t copyRejects = 0;
|
||||
{
|
||||
std::scoped_lock const lock(mutex_);
|
||||
|
||||
@@ -68,31 +66,24 @@ DatabaseRotatingImp::rotate(
|
||||
writableBackend_ = std::move(newBackend);
|
||||
|
||||
copyForwards = copyForwardCount_.exchange(0, std::memory_order_relaxed);
|
||||
copyRejects = copyRejectCount_.exchange(0, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
if (copyForwards > 0)
|
||||
{
|
||||
JLOG(j_.warn()) << "Rotating: copied forward " << copyForwards
|
||||
<< " archive-served reads into the writable backend "
|
||||
"during the rotation window. Rejected "
|
||||
<< copyRejects;
|
||||
"during the rotation window";
|
||||
}
|
||||
|
||||
f(newWritableBackendName, newArchiveBackendName);
|
||||
}
|
||||
|
||||
void
|
||||
DatabaseRotatingImp::setRotationInFlight(LedgerIndex inFlight)
|
||||
DatabaseRotatingImp::setRotationInFlight(bool inFlight)
|
||||
{
|
||||
rotationInFlight_.store(inFlight, std::memory_order_release);
|
||||
JLOG(j_.debug()) << "Rotating: copy-forward on archive reads from " << inFlight << " forward";
|
||||
}
|
||||
|
||||
LedgerIndex
|
||||
DatabaseRotatingImp::getRotationInFlight() const
|
||||
{
|
||||
return rotationInFlight_.load(std::memory_order_acquire);
|
||||
JLOG(j_.debug()) << "Rotating: copy-forward on archive reads "
|
||||
<< (inFlight ? "enabled" : "disabled");
|
||||
}
|
||||
|
||||
std::string
|
||||
@@ -150,7 +141,7 @@ DatabaseRotatingImp::sweep()
|
||||
std::shared_ptr<NodeObject>
|
||||
DatabaseRotatingImp::fetchNodeObject(
|
||||
uint256 const& hash,
|
||||
std::uint32_t ledgerSeq,
|
||||
std::uint32_t,
|
||||
FetchReport& fetchReport,
|
||||
bool duplicate)
|
||||
{
|
||||
@@ -199,35 +190,24 @@ DatabaseRotatingImp::fetchNodeObject(
|
||||
nodeObject = fetch(archive);
|
||||
if (nodeObject)
|
||||
{
|
||||
{
|
||||
// Refresh the writable backend pointer
|
||||
std::scoped_lock const lock(mutex_);
|
||||
writable = writableBackend_;
|
||||
}
|
||||
|
||||
// Update writable backend with data from the archive backend.
|
||||
// While a rotation is in flight, ordinary (duplicate == false)
|
||||
// reads served by the archive are copied forward too: the
|
||||
// archive is about to be deleted, and a body canonicalized
|
||||
// into the cache after the freshen getKeys() snapshot would
|
||||
// otherwise survive only in RAM once the archive is dropped.
|
||||
auto const inFlight = getRotationInFlight();
|
||||
if (duplicate || (inFlight != 0 && (ledgerSeq == 0 || ledgerSeq >= inFlight)))
|
||||
if (duplicate || rotationInFlight_.load(std::memory_order_acquire))
|
||||
{
|
||||
{
|
||||
// Refresh the writable backend pointer
|
||||
std::scoped_lock const lock(mutex_);
|
||||
writable = writableBackend_;
|
||||
}
|
||||
|
||||
if (!duplicate)
|
||||
{
|
||||
JLOG(j_.warn()) << "Rotating: copy node for ledger " << ledgerSeq
|
||||
<< " from archive to writable backend: " << hash;
|
||||
copyForwardCount_.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
writable->store(nodeObject);
|
||||
}
|
||||
else if (inFlight != 0)
|
||||
{
|
||||
JLOG(j_.warn()) << "Rotating: DO NOT copy node for ledger " << ledgerSeq
|
||||
<< " from archive to writable backend: " << hash;
|
||||
copyRejectCount_.fetch_add(1, std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -899,6 +899,10 @@ STObject::add(Serializer& s, WhichFields whichFields) const
|
||||
XRPL_ASSERT(
|
||||
(sType != STI_OBJECT) || (field->getFName().fieldType == STI_OBJECT),
|
||||
"xrpl::STObject::add : valid field type");
|
||||
XRPL_ASSERT(
|
||||
getStyle(field->getFName()) != SoeDefault || !field->isDefault(),
|
||||
"xrpl::STObject::add : non-default value");
|
||||
|
||||
field->addFieldID(s);
|
||||
field->add(s);
|
||||
if (sType == STI_ARRAY || sType == STI_OBJECT)
|
||||
|
||||
@@ -5,21 +5,17 @@
|
||||
#include <test/jtx/noop.h>
|
||||
|
||||
#include <xrpld/app/ledger/LedgerMaster.h>
|
||||
#include <xrpld/app/misc/SHAMapStore.h>
|
||||
#include <xrpld/core/Config.h>
|
||||
|
||||
#include <xrpl/basics/ToString.h>
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/beast/unit_test/suite.h>
|
||||
#include <xrpl/protocol/Feature.h>
|
||||
#include <xrpl/protocol/Protocol.h>
|
||||
#include <xrpl/protocol/SField.h>
|
||||
#include <xrpl/protocol/STObject.h>
|
||||
#include <xrpl/protocol/STTx.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <sstream>
|
||||
#include <vector>
|
||||
|
||||
namespace xrpl::test {
|
||||
@@ -115,71 +111,6 @@ class LedgerMaster_test : public beast::unit_test::Suite
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
testCompleteLedgerRange(FeatureBitset features)
|
||||
{
|
||||
// Note that this test is intentionally very similar to
|
||||
// SHAMapStore_test::testLedgerGaps, but has a different
|
||||
// focus.
|
||||
|
||||
testcase("Complete Ledger operations");
|
||||
|
||||
using namespace test::jtx;
|
||||
|
||||
auto const deleteInterval = 8;
|
||||
|
||||
Env env{*this, envconfig(onlineDelete, deleteInterval)};
|
||||
|
||||
auto const alice = Account("alice");
|
||||
env.fund(XRP(1000), alice);
|
||||
env.close();
|
||||
|
||||
auto& lm = env.app().getLedgerMaster();
|
||||
LedgerIndex minSeq = 2;
|
||||
LedgerIndex maxSeq = env.closed()->header().seq;
|
||||
auto& store = env.app().getSHAMapStore();
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
LedgerIndex lastRotated = store.getLastRotated();
|
||||
BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
|
||||
BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
|
||||
BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
|
||||
BEAST_EXPECT(minSeq + 1 > maxSeq - 1);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
|
||||
|
||||
// Close enough ledgers to rotate a few times
|
||||
for (int i = 0; i < 24; ++i)
|
||||
{
|
||||
for (int t = 0; t < 3; ++t)
|
||||
{
|
||||
env(noop(alice));
|
||||
}
|
||||
env.close();
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
|
||||
++maxSeq;
|
||||
|
||||
if (maxSeq == lastRotated + deleteInterval)
|
||||
{
|
||||
minSeq = lastRotated;
|
||||
lastRotated = maxSeq;
|
||||
}
|
||||
BEAST_EXPECTS(
|
||||
env.closed()->header().seq == maxSeq, to_string(env.closed()->header().seq));
|
||||
BEAST_EXPECTS(store.getLastRotated() == lastRotated, to_string(store.getLastRotated()));
|
||||
std::stringstream expectedRange;
|
||||
expectedRange << minSeq << "-" << maxSeq;
|
||||
BEAST_EXPECTS(lm.getCompleteLedgers() == expectedRange.str(), lm.getCompleteLedgers());
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
void
|
||||
run() override
|
||||
@@ -193,7 +124,6 @@ public:
|
||||
testWithFeats(FeatureBitset features)
|
||||
{
|
||||
testTxnIdFromIndex(features);
|
||||
testCompleteLedgerRange(features);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
#include <test/jtx/Env.h>
|
||||
#include <test/jtx/amount.h>
|
||||
#include <test/jtx/envconfig.h>
|
||||
#include <test/jtx/noop.h>
|
||||
|
||||
#include <xrpld/app/ledger/LedgerMaster.h>
|
||||
#include <xrpld/app/main/Application.h>
|
||||
#include <xrpld/app/main/NodeStoreScheduler.h>
|
||||
#include <xrpld/app/misc/SHAMapStore.h>
|
||||
@@ -28,15 +26,12 @@
|
||||
#include <boost/filesystem/path.hpp>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
|
||||
namespace xrpl::test {
|
||||
@@ -48,7 +43,10 @@ class SHAMapStore_test : public beast::unit_test::Suite
|
||||
static auto
|
||||
onlineDelete(std::unique_ptr<Config> cfg)
|
||||
{
|
||||
return jtx::onlineDelete(std::move(cfg), kDeleteInterval);
|
||||
cfg->ledgerHistory = kDeleteInterval;
|
||||
auto& section = cfg->section(Sections::kNodeDatabase);
|
||||
section.set(Keys::kOnlineDelete, std::to_string(kDeleteInterval));
|
||||
return cfg;
|
||||
}
|
||||
|
||||
static auto
|
||||
@@ -146,11 +144,11 @@ class SHAMapStore_test : public beast::unit_test::Suite
|
||||
auto& store = env.app().getSHAMapStore();
|
||||
|
||||
int ledgerSeq = 3;
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
BEAST_EXPECT(!store.getLastRotated());
|
||||
|
||||
env.close();
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
auto ledger = env.rpc("ledger", "validated");
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++)));
|
||||
@@ -230,7 +228,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(kDeleteInterval + 4)));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
BEAST_EXPECT(store.getLastRotated() == kDeleteInterval + 3);
|
||||
lastRotated = store.getLastRotated();
|
||||
@@ -257,7 +255,7 @@ public:
|
||||
!getHash(ledgers[i]).empty());
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
BEAST_EXPECT(store.getLastRotated() == kDeleteInterval + lastRotated);
|
||||
|
||||
@@ -295,7 +293,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
// The database will always have back to ledger 2,
|
||||
// regardless of lastRotated.
|
||||
@@ -310,7 +308,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
|
||||
BEAST_EXPECT(lastRotated != store.getLastRotated());
|
||||
@@ -326,7 +324,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
ledgerCheck(env, kDeleteInterval + 1, lastRotated);
|
||||
BEAST_EXPECT(lastRotated != store.getLastRotated());
|
||||
@@ -365,7 +363,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
ledgerCheck(env, ledgerSeq - 2, 2);
|
||||
BEAST_EXPECT(lastRotated == store.getLastRotated());
|
||||
@@ -375,7 +373,7 @@ public:
|
||||
BEAST_EXPECT(!RPC::containsError(canDelete[jss::result]));
|
||||
BEAST_EXPECT(canDelete[jss::result][jss::can_delete] == ledgerSeq + (kDeleteInterval / 2));
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
ledgerCheck(env, ledgerSeq - 2, 2);
|
||||
BEAST_EXPECT(store.getLastRotated() == lastRotated);
|
||||
@@ -388,7 +386,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
|
||||
|
||||
@@ -404,7 +402,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
BEAST_EXPECT(store.getLastRotated() == lastRotated);
|
||||
|
||||
@@ -416,7 +414,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
ledgerCheck(env, ledgerSeq - firstBatch, firstBatch);
|
||||
|
||||
@@ -438,7 +436,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
BEAST_EXPECT(store.getLastRotated() == lastRotated);
|
||||
|
||||
@@ -450,7 +448,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
|
||||
|
||||
@@ -471,7 +469,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
BEAST_EXPECT(store.getLastRotated() == lastRotated);
|
||||
|
||||
@@ -483,7 +481,7 @@ public:
|
||||
BEAST_EXPECT(goodLedger(env, ledger, std::to_string(ledgerSeq++), true));
|
||||
}
|
||||
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
store.rendezvous();
|
||||
|
||||
ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
|
||||
|
||||
@@ -606,176 +604,6 @@ public:
|
||||
BEAST_EXPECT(dbr->getName() == "3");
|
||||
}
|
||||
|
||||
void
|
||||
testLedgerGaps()
|
||||
{
|
||||
// Note that this test is intentionally very similar to
|
||||
// LedgerMaster_test::testCompleteLedgerRange, but has a different
|
||||
// focus.
|
||||
|
||||
testcase("Wait for ledger gaps to fill in");
|
||||
|
||||
using namespace test::jtx;
|
||||
|
||||
Env env{*this, envconfig(onlineDelete)};
|
||||
|
||||
auto failureMessage = [&](char const* label, auto expected, auto actual) {
|
||||
std::stringstream ss;
|
||||
ss << label << ": Expected: " << expected << ", Got: " << actual;
|
||||
return ss.str();
|
||||
};
|
||||
|
||||
auto const alice = Account("alice");
|
||||
env.fund(XRP(1000), alice);
|
||||
env.close();
|
||||
|
||||
auto& lm = env.app().getLedgerMaster();
|
||||
LedgerIndex minSeq = 2;
|
||||
LedgerIndex maxSeq = env.closed()->header().seq;
|
||||
auto& store = env.app().getSHAMapStore();
|
||||
LedgerIndex lastRotated = store.getLastRotated();
|
||||
while (lastRotated != 3)
|
||||
{
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
lastRotated = store.getLastRotated();
|
||||
}
|
||||
BEAST_EXPECTS(maxSeq == 3, std::to_string(maxSeq));
|
||||
BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
|
||||
BEAST_EXPECT(minSeq + 1 > maxSeq - 1);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
|
||||
|
||||
// Close enough ledgers to rotate a few times
|
||||
while (maxSeq < 20)
|
||||
{
|
||||
for (int t = 0; t < 3; ++t)
|
||||
{
|
||||
env(noop(alice));
|
||||
}
|
||||
env.close();
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
|
||||
++maxSeq;
|
||||
|
||||
if (maxSeq + 1 == lastRotated + kDeleteInterval)
|
||||
{
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
// The next ledger will trigger a rotation. Delete the
|
||||
// current ledger from LedgerMaster.
|
||||
std::this_thread::sleep_for(100ms);
|
||||
LedgerIndex const deleteSeq = maxSeq;
|
||||
{
|
||||
std::size_t iterations = 30;
|
||||
while (!lm.haveLedger(deleteSeq) && --iterations > 0)
|
||||
{
|
||||
std::this_thread::sleep_for(100ms);
|
||||
}
|
||||
// Even the slowest machines should be able to finalize deleteSeq within 10
|
||||
// loops (400ms). If this test ever actually fails feel free to lower this
|
||||
// cutoff. The intent of this test is to flag if the loop takes a very long
|
||||
// time, but still allow the rest of this function to finish.
|
||||
BEAST_EXPECTS(iterations > 20, std::to_string(iterations));
|
||||
if (!BEAST_EXPECT(lm.haveLedger(deleteSeq)))
|
||||
return;
|
||||
}
|
||||
lm.clearLedger(deleteSeq);
|
||||
|
||||
auto expectedRange = [](auto minSeq, auto deleteSeq, auto maxSeq) {
|
||||
std::stringstream expectedRange;
|
||||
expectedRange << minSeq << "-" << (deleteSeq - 1);
|
||||
if (deleteSeq + 1 == maxSeq)
|
||||
{
|
||||
expectedRange << "," << maxSeq;
|
||||
}
|
||||
else if (deleteSeq < maxSeq)
|
||||
{
|
||||
expectedRange << "," << (deleteSeq + 1) << "-" << maxSeq;
|
||||
}
|
||||
return expectedRange.str();
|
||||
};
|
||||
BEAST_EXPECTS(
|
||||
lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
failureMessage(
|
||||
"Complete ledgers",
|
||||
expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
lm.getCompleteLedgers()));
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 1);
|
||||
|
||||
// Close another ledger, which will trigger a rotation, but the
|
||||
// rotation will be stuck until the missing ledger is filled in.
|
||||
env.close();
|
||||
// Do not call rendezvous() here without a timeout; it will block until the missing
|
||||
// ledger is backfilled. That will not happen automatically. It's a manual step that
|
||||
// is done later in this test.
|
||||
++maxSeq;
|
||||
|
||||
// Nothing has changed
|
||||
BEAST_EXPECTS(
|
||||
store.getLastRotated() == lastRotated,
|
||||
failureMessage("lastRotated", lastRotated, store.getLastRotated()));
|
||||
BEAST_EXPECTS(
|
||||
lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
failureMessage(
|
||||
"Complete ledgers",
|
||||
expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
lm.getCompleteLedgers()));
|
||||
|
||||
// Close 5 more ledgers, waiting a little bit in between to
|
||||
// simulate the ledger making progress while online delete waits
|
||||
// for the missing ledger to be filled in.
|
||||
// This ensures the healthWait check has time to run and
|
||||
// detect the gap.
|
||||
for (int l = 0; l < 5; ++l)
|
||||
{
|
||||
env.close();
|
||||
++maxSeq;
|
||||
// Nothing has changed
|
||||
BEAST_EXPECTS(
|
||||
store.getLastRotated() == lastRotated,
|
||||
failureMessage("lastRotated", lastRotated, store.getLastRotated()));
|
||||
BEAST_EXPECTS(
|
||||
lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
failureMessage(
|
||||
"Complete Ledgers",
|
||||
expectedRange(minSeq, deleteSeq, maxSeq),
|
||||
lm.getCompleteLedgers()));
|
||||
// The Store is "stuck" in healthWait() and won't finish the run() loop until
|
||||
// it's backfilled
|
||||
BEAST_EXPECT(!store.rendezvous(100ms));
|
||||
}
|
||||
|
||||
// Put the missing ledger back in LedgerMaster
|
||||
lm.setLedgerRangePresent(deleteSeq, deleteSeq);
|
||||
|
||||
// Wait for the rotation to finish
|
||||
BEAST_EXPECT(store.rendezvous());
|
||||
|
||||
minSeq = lastRotated;
|
||||
lastRotated = deleteSeq + 1;
|
||||
}
|
||||
BEAST_EXPECT(maxSeq != lastRotated + kDeleteInterval);
|
||||
BEAST_EXPECTS(
|
||||
env.closed()->header().seq == maxSeq,
|
||||
failureMessage("maxSeq", maxSeq, env.closed()->header().seq));
|
||||
BEAST_EXPECTS(
|
||||
store.getLastRotated() == lastRotated,
|
||||
failureMessage("lastRotated", lastRotated, store.getLastRotated()));
|
||||
std::stringstream expectedRange;
|
||||
expectedRange << minSeq << "-" << maxSeq;
|
||||
BEAST_EXPECTS(
|
||||
lm.getCompleteLedgers() == expectedRange.str(),
|
||||
failureMessage("CompleteLedgers", expectedRange.str(), lm.getCompleteLedgers()));
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
|
||||
BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
run() override
|
||||
{
|
||||
@@ -783,7 +611,6 @@ public:
|
||||
testAutomatic();
|
||||
testCanDelete();
|
||||
testRotate();
|
||||
testLedgerGaps();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
#include <xrpld/core/Config.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
@@ -63,19 +62,6 @@ envconfig(F&& modfunc, Args&&... args)
|
||||
return modfunc(envconfig(), std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief adjust config to enable online_delete
|
||||
*
|
||||
* @param cfg config instance to be modified
|
||||
*
|
||||
* @param deleteInterval how many new ledgers should be available before
|
||||
* rotating. Defaults to 8, because the standalone minimum is 8.
|
||||
*
|
||||
* @return unique_ptr to Config instance
|
||||
*/
|
||||
std::unique_ptr<Config>
|
||||
onlineDelete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval = 8);
|
||||
|
||||
/**
|
||||
* @brief adjust config so no admin ports are enabled
|
||||
*
|
||||
|
||||
@@ -7,10 +7,8 @@
|
||||
#include <xrpl/config/Constants.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace xrpl::test {
|
||||
@@ -62,15 +60,6 @@ setupConfigForUnitTests(Config& cfg)
|
||||
|
||||
namespace jtx {
|
||||
|
||||
std::unique_ptr<Config>
|
||||
onlineDelete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval)
|
||||
{
|
||||
cfg->ledgerHistory = deleteInterval;
|
||||
auto& section = cfg->section(Sections::kNodeDatabase);
|
||||
section.set(Keys::kOnlineDelete, std::to_string(deleteInterval));
|
||||
return cfg;
|
||||
}
|
||||
|
||||
std::unique_ptr<Config>
|
||||
noAdmin(std::unique_ptr<Config> cfg)
|
||||
{
|
||||
|
||||
@@ -123,10 +123,7 @@ public:
|
||||
failedSave(std::uint32_t seq, uint256 const& hash);
|
||||
|
||||
std::string
|
||||
getCompleteLedgers() const;
|
||||
|
||||
std::size_t
|
||||
missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const;
|
||||
getCompleteLedgers();
|
||||
|
||||
/**
|
||||
* Apply held transactions to the open ledger
|
||||
@@ -351,7 +348,7 @@ private:
|
||||
// A set of transactions to replay during the next close
|
||||
std::unique_ptr<LedgerReplay> replayData_;
|
||||
|
||||
std::recursive_mutex mutable completeLock_;
|
||||
std::recursive_mutex completeLock_;
|
||||
RangeSet<std::uint32_t> completeLedgers_;
|
||||
|
||||
// Publish thread is running.
|
||||
|
||||
@@ -57,7 +57,6 @@
|
||||
#include <xrpl/shamap/SHAMapMissingNode.h>
|
||||
#include <xrpl/shamap/SHAMapTreeNode.h>
|
||||
|
||||
#include <boost/icl/concept/interval_associator.hpp>
|
||||
#include <boost/icl/concept/interval_set.hpp>
|
||||
|
||||
#include <xrpl.pb.h>
|
||||
@@ -1577,33 +1576,12 @@ LedgerMaster::getPublishedLedger()
|
||||
}
|
||||
|
||||
std::string
|
||||
LedgerMaster::getCompleteLedgers() const
|
||||
LedgerMaster::getCompleteLedgers()
|
||||
{
|
||||
std::scoped_lock const sl(completeLock_);
|
||||
return to_string(completeLedgers_);
|
||||
}
|
||||
|
||||
std::size_t
|
||||
LedgerMaster::missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const
|
||||
{
|
||||
if (first > last)
|
||||
{
|
||||
// LCOV_EXCL_START
|
||||
UNREACHABLE("xrpl::LedgerMaster::missingFromCompleteLedgerRange : invalid parameters");
|
||||
return 0;
|
||||
// LCOV_EXCL_STOP
|
||||
}
|
||||
|
||||
RangeSet<LedgerIndex> const target{range(first, last)};
|
||||
|
||||
auto const missing = [&target, this] {
|
||||
std::scoped_lock const sl(completeLock_);
|
||||
return target - completeLedgers_;
|
||||
}();
|
||||
|
||||
return boost::icl::size(missing);
|
||||
}
|
||||
|
||||
std::optional<NetClock::time_point>
|
||||
LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
|
||||
{
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
#include <xrpl/nodestore/Scheduler.h>
|
||||
#include <xrpl/protocol/Protocol.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
@@ -35,8 +34,8 @@ public:
|
||||
virtual void
|
||||
start() = 0;
|
||||
|
||||
[[nodiscard]] virtual bool
|
||||
rendezvous(std::optional<std::chrono::milliseconds> const& timeout = {}) const = 0;
|
||||
virtual void
|
||||
rendezvous() const = 0;
|
||||
|
||||
virtual void
|
||||
stop() = 0;
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
#include <xrpl/basics/ByteUtilities.h>
|
||||
#include <xrpl/basics/Log.h>
|
||||
#include <xrpl/basics/contract.h>
|
||||
#include <xrpl/basics/scope.h>
|
||||
#include <xrpl/beast/core/CurrentThreadName.h>
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
#include <xrpl/beast/utility/instrumentation.h>
|
||||
@@ -33,8 +32,6 @@
|
||||
#include <boost/filesystem/path.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <limits>
|
||||
@@ -239,21 +236,14 @@ SHAMapStoreImp::onLedgerClosed(std::shared_ptr<Ledger const> const& ledger)
|
||||
cond_.notify_one();
|
||||
}
|
||||
|
||||
bool
|
||||
SHAMapStoreImp::rendezvous(std::optional<std::chrono::milliseconds> const& timeout) const
|
||||
void
|
||||
SHAMapStoreImp::rendezvous() const
|
||||
{
|
||||
if (!working_)
|
||||
return true;
|
||||
|
||||
auto notWorking = [&] { return !working_; };
|
||||
return;
|
||||
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
if (timeout)
|
||||
{
|
||||
return rendezvous_.wait_for(lock, *timeout, notWorking);
|
||||
}
|
||||
rendezvous_.wait(lock, notWorking);
|
||||
return true;
|
||||
rendezvous_.wait(lock, [&] { return !working_; });
|
||||
}
|
||||
|
||||
int
|
||||
@@ -262,45 +252,6 @@ SHAMapStoreImp::fdRequired() const
|
||||
return fdRequired_;
|
||||
}
|
||||
|
||||
void
|
||||
SHAMapStoreImp::rescueNode(SHAMapTreeNode const& node)
|
||||
{
|
||||
XRPL_ASSERT(node.cowid() == 0, "SHAMapStoreImp::copyNode : rescued node must be clean");
|
||||
// Reachable from the validated state map in memory, but present in
|
||||
// neither backend: its only on-disk copy lived in a backend removed by
|
||||
// an earlier rotation, and it was never rewritten because it is clean
|
||||
// (cowid == 0, so flushDirty skips it). Persist the in-memory body
|
||||
// directly into the writable backend so it survives this rotation
|
||||
// instead of later surfacing as an unresolvable SHAMapMissingNode.
|
||||
|
||||
auto const nodeType = node.getType();
|
||||
auto const objectType = std::invoke([nodeType] {
|
||||
switch (nodeType)
|
||||
{
|
||||
case SHAMapNodeType::TnAccountState:
|
||||
return NodeObjectType::AccountNode;
|
||||
case SHAMapNodeType::TnTransactionNm:
|
||||
return NodeObjectType::TransactionNode;
|
||||
default:
|
||||
return NodeObjectType::Unknown;
|
||||
}
|
||||
});
|
||||
|
||||
auto const hash = node.getHash().asUInt256();
|
||||
if (objectType == NodeObjectType::Unknown)
|
||||
{
|
||||
JLOG(journal_.warn()) << "copyNode: unable to re-store node with unknown type, hash="
|
||||
<< hash << " type=" << static_cast<int>(nodeType);
|
||||
return;
|
||||
}
|
||||
Serializer s;
|
||||
node.serializeWithPrefix(s);
|
||||
dbRotating_->store(objectType, std::move(s.modData()), hash, 0);
|
||||
|
||||
JLOG(journal_.warn()) << "copyNode: re-stored node missing from both backends, hash=" << hash
|
||||
<< " type=" << static_cast<int>(nodeType);
|
||||
}
|
||||
|
||||
bool
|
||||
SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node)
|
||||
{
|
||||
@@ -309,7 +260,19 @@ SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node)
|
||||
node.getHash().asUInt256(), 0, NodeStore::FetchType::Synchronous, true);
|
||||
if (!obj)
|
||||
{
|
||||
rescueNode(node);
|
||||
XRPL_ASSERT(node.cowid() == 0, "SHAMapStoreImp::copyNode : rescued node must be clean");
|
||||
// Reachable from the validated state map in memory, but present in
|
||||
// neither backend: its only on-disk copy lived in a backend removed by
|
||||
// an earlier rotation, and it was never rewritten because it is clean
|
||||
// (cowid == 0, so flushDirty skips it). Persist the in-memory body
|
||||
// directly into the writable backend so it survives this rotation
|
||||
// instead of later surfacing as an unresolvable SHAMapMissingNode.
|
||||
auto const hash = node.getHash().asUInt256();
|
||||
Serializer s;
|
||||
node.serializeWithPrefix(s);
|
||||
dbRotating_->store(NodeObjectType::AccountNode, std::move(s.modData()), hash, 0);
|
||||
JLOG(journal_.warn()) << "copyNode: re-stored node missing from both backends, hash="
|
||||
<< hash << " type=" << static_cast<int>(node.getType());
|
||||
}
|
||||
if ((++nodeCount % checkHealthInterval_) == 0u)
|
||||
{
|
||||
@@ -335,11 +298,6 @@ SHAMapStoreImp::run()
|
||||
|
||||
while (true)
|
||||
{
|
||||
XRPL_ASSERT(
|
||||
dbRotating_->getRotationInFlight() == 0,
|
||||
"SHAMapStoreImp::run : rotationInFlight_ must be zero "
|
||||
"outside rotation window");
|
||||
|
||||
healthy_ = true;
|
||||
std::shared_ptr<Ledger const> validatedLedger;
|
||||
|
||||
@@ -372,45 +330,14 @@ SHAMapStoreImp::run()
|
||||
bool const readyToRotate = validatedSeq >= lastRotated + deleteInterval_ &&
|
||||
canDelete_ >= lastRotated - 1 && healthWait() == HealthResult::KeepGoing;
|
||||
|
||||
{
|
||||
JLOG(journal_.trace()) << "run: Setting lastGoodValidatedLedger_ to " << validatedSeq;
|
||||
// Note that this is set after the healthWait() check, so that we
|
||||
// don't start the rotation until the validated ledger is fully
|
||||
// processed. It is not guaranteed to be done at this point. It also
|
||||
// allows the testLedgerGaps unit test to work.
|
||||
std::unique_lock<std::mutex> const lock(mutex_);
|
||||
lastGoodValidatedLedger_ = validatedSeq;
|
||||
}
|
||||
|
||||
// will delete up to (not including) lastRotated
|
||||
if (readyToRotate)
|
||||
{
|
||||
auto const diff = validatedSeq - lastRotated;
|
||||
JLOG(journal_.warn()) << "rotating validatedSeq " << validatedSeq << " lastRotated "
|
||||
<< lastRotated << " diff " << diff << " deleteInterval "
|
||||
<< deleteInterval_ << " canDelete_ " << canDelete_ << " state "
|
||||
<< lastRotated << " deleteInterval " << deleteInterval_
|
||||
<< " canDelete_ " << canDelete_ << " state "
|
||||
<< app_.getOPs().strOperatingMode(false) << " age "
|
||||
<< ledgerMaster_->getValidatedLedgerAge().count()
|
||||
<< "s. Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
|
||||
|
||||
// Close the getKeys()->swap exposure window: from here until
|
||||
// rotate() completes, an ordinary read for new ledgers served by the archive is
|
||||
// copied forward into the writable backend, so a node fetched
|
||||
// from the doomed archive cannot be left RAM-only when the
|
||||
// archive is deleted. RAII so the early returns below (and any
|
||||
// exception) also clear the flag.
|
||||
struct RotationExposureGuard
|
||||
{
|
||||
NodeStore::DatabaseRotating& db;
|
||||
~RotationExposureGuard()
|
||||
{
|
||||
db.setRotationInFlight(0);
|
||||
}
|
||||
};
|
||||
RotationExposureGuard const rotationExposureGuard{*dbRotating_};
|
||||
// Anything before lastRotated is going to get deleted soon, so we don't care about
|
||||
// moving it to the writable DB.
|
||||
dbRotating_->setRotationInFlight(lastRotated);
|
||||
<< ledgerMaster_->getValidatedLedgerAge().count() << 's';
|
||||
|
||||
clearPrior(lastRotated);
|
||||
if (healthWait() == HealthResult::Stopping)
|
||||
@@ -439,6 +366,23 @@ SHAMapStoreImp::run()
|
||||
JLOG(journal_.debug())
|
||||
<< "copied ledger " << validatedSeq << " nodecount " << nodeCount;
|
||||
|
||||
// Close the getKeys()->swap exposure window: from here until
|
||||
// rotate() completes, an ordinary read served by the archive is
|
||||
// copied forward into the writable backend, so a node fetched
|
||||
// from the doomed archive cannot be left RAM-only when the
|
||||
// archive is deleted. RAII so the early returns below (and any
|
||||
// exception) also clear the flag.
|
||||
struct RotationExposureGuard
|
||||
{
|
||||
NodeStore::DatabaseRotating& db;
|
||||
~RotationExposureGuard()
|
||||
{
|
||||
db.setRotationInFlight(false);
|
||||
}
|
||||
};
|
||||
RotationExposureGuard const rotationExposureGuard{*dbRotating_};
|
||||
dbRotating_->setRotationInFlight(true);
|
||||
|
||||
JLOG(journal_.debug()) << "freshening caches";
|
||||
freshenCaches();
|
||||
if (healthWait() == HealthResult::Stopping)
|
||||
@@ -468,14 +412,7 @@ SHAMapStoreImp::run()
|
||||
clearCaches(validatedSeq);
|
||||
});
|
||||
|
||||
auto const currentValidatedSeq = ledgerMaster_->getValidLedgerIndex();
|
||||
auto const processingDiff = currentValidatedSeq - validatedSeq;
|
||||
JLOG(journal_.warn())
|
||||
<< "finished rotation. validatedSeq: " << validatedSeq
|
||||
<< ", lastRotated: " << lastRotated << " diff " << diff
|
||||
<< ". Updated validated seq is " << currentValidatedSeq << ", " << processingDiff
|
||||
<< " ledgers were validated during the rotation processs. Complete ledgers: "
|
||||
<< ledgerMaster_->getCompleteLedgers();
|
||||
JLOG(journal_.warn()) << "finished rotation " << validatedSeq;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -670,7 +607,8 @@ SHAMapStoreImp::freshenCaches()
|
||||
{
|
||||
if (freshenCache(*treeNodeCache_))
|
||||
return;
|
||||
freshenCache(app_.getMasterTransaction().getCache());
|
||||
if (freshenCache(app_.getMasterTransaction().getCache()))
|
||||
return;
|
||||
}
|
||||
|
||||
void
|
||||
@@ -718,94 +656,20 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
|
||||
SHAMapStoreImp::HealthResult
|
||||
SHAMapStoreImp::healthWait()
|
||||
{
|
||||
// Gets the current status of the server from ledgerMaster_ and netOPs_. Must be called
|
||||
// while mutex_ is unlocked to avoid unlikely, but possible, deadlock with ledgerMaster_'s
|
||||
// completeLock_.
|
||||
// Releasing the lock may mean that status will be slightly out of date when the lock is
|
||||
// reacquired, but it's close enough. In a normal rotation, healthWait() is called frequently,
|
||||
// so a false positive will be detected on the next call, and a false negative will be detected
|
||||
// in the next loop iteration. Database rotation is important, but not timely, so an extra
|
||||
// delay is fine.
|
||||
auto readServerStatus = [this](
|
||||
LedgerIndex& index,
|
||||
std::chrono::seconds& age,
|
||||
OperatingMode& mode,
|
||||
std::size_t& numMissing,
|
||||
LedgerIndex const lowerBound,
|
||||
ScopeUnlock<decltype(mutex_)> const&) {
|
||||
index = ledgerMaster_->getValidLedgerIndex();
|
||||
auto age = ledgerMaster_->getValidatedLedgerAge();
|
||||
OperatingMode mode = netOPs_->getOperatingMode();
|
||||
std::unique_lock lock(mutex_);
|
||||
while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_))
|
||||
{
|
||||
lock.unlock();
|
||||
JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count()
|
||||
<< "s for node to stabilize. state: "
|
||||
<< app_.getOPs().strOperatingMode(mode, false) << ". age "
|
||||
<< age.count() << 's';
|
||||
std::this_thread::sleep_for(recoveryWaitTime_);
|
||||
age = ledgerMaster_->getValidatedLedgerAge();
|
||||
mode = netOPs_->getOperatingMode();
|
||||
|
||||
numMissing =
|
||||
lowerBound == 0 ? 0 : ledgerMaster_->missingFromCompleteLedgerRange(lowerBound, index);
|
||||
};
|
||||
// Tracked server status properties
|
||||
LedgerIndex index = 0;
|
||||
std::chrono::seconds age;
|
||||
OperatingMode mode = OperatingMode::DISCONNECTED;
|
||||
std::size_t numMissing = 0;
|
||||
LedgerIndex lastLedger = 0;
|
||||
|
||||
std::unique_lock lock(mutex_);
|
||||
|
||||
auto const waitTime = recoveryWaitTime_;
|
||||
auto const ageThreshold = ageThreshold_;
|
||||
{
|
||||
auto const lowerBound = lastGoodValidatedLedger_;
|
||||
|
||||
ScopeUnlock const unlock(lock);
|
||||
|
||||
readServerStatus(index, age, mode, numMissing, lowerBound, unlock);
|
||||
}
|
||||
|
||||
auto healthy = [&]() {
|
||||
// Special case: If the server is disconnected, it's not doing any ledger I/O, because
|
||||
// it's focused on trying to get peers. A disconnected state is should never be caused by
|
||||
// the activity of the server. It's usually limited to hardware or connectivity issues. Take
|
||||
// advantage of that to run as much rotation I/O as possible before it comes back online.
|
||||
if (mode == OperatingMode::DISCONNECTED)
|
||||
return true;
|
||||
if (age > ageThreshold)
|
||||
return false;
|
||||
if (numMissing > 0)
|
||||
return false;
|
||||
if (mode != OperatingMode::FULL)
|
||||
return false;
|
||||
return true;
|
||||
};
|
||||
|
||||
while (!stop_ && !healthy())
|
||||
{
|
||||
// this value shouldn't change, so grab it while we have the
|
||||
// lock
|
||||
auto const lowerBound = lastGoodValidatedLedger_;
|
||||
|
||||
ScopeUnlock const unlock(lock);
|
||||
|
||||
auto const [stream, waitMs] = std::invoke(
|
||||
[mode, age, ageThreshold, index, lastLedger, waitTime, this]()
|
||||
-> std::pair<beast::Journal::Stream, std::chrono::milliseconds> {
|
||||
if (mode != OperatingMode::FULL || age > ageThreshold)
|
||||
return {journal_.warn(), waitTime};
|
||||
if (index != lastLedger)
|
||||
{
|
||||
// We expect this ledger to be built soon, so log at a lower level, and don't
|
||||
// wait as long.
|
||||
return {
|
||||
journal_.trace(),
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(waitTime) / 4};
|
||||
}
|
||||
return {journal_.info(), waitTime};
|
||||
});
|
||||
JLOG(stream) << "Waiting " << waitMs.count() << "ms for node to stabilize. state: "
|
||||
<< app_.getOPs().strOperatingMode(mode, false) << ". age " << age.count()
|
||||
<< "s. Missing ledgers: " << numMissing << ". Expect: " << lowerBound << "-"
|
||||
<< index << ". Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
|
||||
std::this_thread::sleep_for(waitMs);
|
||||
|
||||
readServerStatus(index, age, mode, numMissing, lowerBound, unlock);
|
||||
lastLedger = index;
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
return stop_ ? HealthResult::Stopping : HealthResult::KeepGoing;
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <concepts>
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
@@ -89,11 +88,6 @@ private:
|
||||
std::thread thread_;
|
||||
bool stop_ = false;
|
||||
bool healthy_ = true;
|
||||
// Used to prevent ledger gaps from forming during online deletion. Keeps
|
||||
// track of the last validated ledger that was processed without gaps. There
|
||||
// are no guarantees about gaps while online delete is not running. For
|
||||
// that, use advisory_delete and check for gaps externally.
|
||||
LedgerIndex lastGoodValidatedLedger_ = 0;
|
||||
mutable std::condition_variable cond_;
|
||||
mutable std::condition_variable rendezvous_;
|
||||
mutable std::mutex mutex_;
|
||||
@@ -108,12 +102,12 @@ private:
|
||||
std::chrono::milliseconds backOff_{100};
|
||||
std::chrono::seconds ageThreshold_{60};
|
||||
/**
|
||||
* If the node is out of sync, or any recent ledgers are not
|
||||
* available during an online_delete healthWait() call, sleep
|
||||
* the thread for this time, and continue checking until recovery.
|
||||
* If the node is out of sync during an online_delete healthWait()
|
||||
* call, sleep the thread for this time, and continue checking until
|
||||
* recovery.
|
||||
* See also: "recovery_wait_seconds" in xrpld-example.cfg
|
||||
*/
|
||||
std::chrono::seconds recoveryWaitTime_{2};
|
||||
std::chrono::seconds recoveryWaitTime_{5};
|
||||
|
||||
// these do not exist upon SHAMapStore creation, but do exist
|
||||
// as of run() or before
|
||||
@@ -169,8 +163,8 @@ public:
|
||||
void
|
||||
onLedgerClosed(std::shared_ptr<Ledger const> const& ledger) override;
|
||||
|
||||
bool
|
||||
rendezvous(std::optional<std::chrono::milliseconds> const& timeout = {}) const override;
|
||||
void
|
||||
rendezvous() const override;
|
||||
int
|
||||
fdRequired() const override;
|
||||
|
||||
@@ -178,9 +172,6 @@ public:
|
||||
minimumOnline() const override;
|
||||
|
||||
private:
|
||||
// Force write a node to the writable backend during rotation so it doesn't get lost
|
||||
void
|
||||
rescueNode(SHAMapTreeNode const& node);
|
||||
// callback for visitNodes
|
||||
bool
|
||||
copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node);
|
||||
@@ -200,18 +191,7 @@ private:
|
||||
|
||||
for (auto const& key : cache.getKeys())
|
||||
{
|
||||
[[maybe_unused]]
|
||||
auto const obj =
|
||||
dbRotating_->fetchNodeObject(key, 0, NodeStore::FetchType::Synchronous, true);
|
||||
if constexpr (std::derived_from<typename CacheInstance::mapped_type, SHAMapTreeNode>)
|
||||
{
|
||||
if (!obj)
|
||||
{
|
||||
auto const node = cache.fetch(key);
|
||||
if (node)
|
||||
rescueNode(*node);
|
||||
}
|
||||
}
|
||||
dbRotating_->fetchNodeObject(key, 0, NodeStore::FetchType::Synchronous, true);
|
||||
if (!(++check % checkHealthInterval_) && healthWait() == HealthResult::Stopping)
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -268,9 +268,9 @@ saveValidatedLedger(
|
||||
app.getAcceptedLedgerCache().canonicalizeReplaceClient(ledger->header().hash, aLedger);
|
||||
}
|
||||
}
|
||||
catch (std::exception const& e)
|
||||
catch (std::exception const&)
|
||||
{
|
||||
JLOG(j.warn()) << "An accepted ledger was missing nodes " << e.what();
|
||||
JLOG(j.warn()) << "An accepted ledger was missing nodes";
|
||||
app.getLedgerMaster().failedSave(seq, ledger->header().hash);
|
||||
// Clients can now trust the database for information about this
|
||||
// ledger sequence.
|
||||
|
||||
Reference in New Issue
Block a user