mirror of
https://github.com/Xahau/xahaud.git
synced 2026-08-24 17:00:53 +00:00
Compare commits
28 Commits
sync-2.6.0
...
manifest-c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3f977df5de | ||
|
|
d80b52569e | ||
|
|
da85d20341 | ||
|
|
0a6ffc3a34 | ||
|
|
404124a6c7 | ||
|
|
8bd8d9a12d | ||
|
|
ff43bca41f | ||
|
|
9dc9978087 | ||
|
|
fd1b5c4b20 | ||
|
|
f96657c876 | ||
|
|
5bc9cc5fe3 | ||
|
|
2c57459e4d | ||
|
|
d5feb23d04 | ||
|
|
34042f998b | ||
|
|
79fa1ee4f5 | ||
|
|
b9892da0a7 | ||
|
|
d3264e16c5 | ||
|
|
6da9ab90ae | ||
|
|
84edc16a92 | ||
|
|
fd8fedbeca | ||
|
|
d31f9fe421 | ||
|
|
1ea905a0d3 | ||
|
|
a0ea133c5c | ||
|
|
9462bafa49 | ||
|
|
8696eeb454 | ||
|
|
9cc426b3fb | ||
|
|
6426a11058 | ||
|
|
ad84ef5a9a |
@@ -62,6 +62,24 @@
|
||||
|
||||
namespace ripple {
|
||||
|
||||
namespace base64 {
|
||||
|
||||
/** Returns the maximum number of characters needed to base64-encode bytes. */
|
||||
constexpr std::size_t
|
||||
encoded_size(std::size_t const numBytes)
|
||||
{
|
||||
return 4 * ((numBytes + 2) / 3);
|
||||
}
|
||||
|
||||
/** Returns an upper bound on bytes decoded from base64 characters. */
|
||||
constexpr std::size_t
|
||||
decoded_size(std::size_t const numChars)
|
||||
{
|
||||
return ((numChars / 4) * 3) + 2;
|
||||
}
|
||||
|
||||
} // namespace base64
|
||||
|
||||
std::string
|
||||
base64_encode(std::uint8_t const* data, std::size_t len);
|
||||
|
||||
|
||||
@@ -212,6 +212,7 @@ JSS(build_version); // out: NetworkOPs
|
||||
JSS(bytes_written);
|
||||
JSS(cancel_after); // out: AccountChannels
|
||||
JSS(can_delete); // out: CanDelete
|
||||
JSS(candidates); // in: ValidatorList
|
||||
JSS(mpt_amount); // out: mpt_holders
|
||||
JSS(mpt_issuance_id); // in: Payment, mpt_holders
|
||||
JSS(mptoken_index); // out: mpt_holders
|
||||
|
||||
@@ -111,18 +111,6 @@ get_inverse()
|
||||
return &tab[0];
|
||||
}
|
||||
|
||||
/// Returns max chars needed to encode a base64 string
|
||||
inline std::size_t constexpr encoded_size(std::size_t n)
|
||||
{
|
||||
return 4 * ((n + 2) / 3);
|
||||
}
|
||||
|
||||
/// Returns max bytes needed to decode a base64 string
|
||||
inline std::size_t constexpr decoded_size(std::size_t n)
|
||||
{
|
||||
return ((n / 4) * 3) + 2;
|
||||
}
|
||||
|
||||
/** Encode a series of octets as a padded, base64 string.
|
||||
|
||||
The resulting string will not be null terminated.
|
||||
|
||||
@@ -243,6 +243,20 @@ public:
|
||||
return m2;
|
||||
}
|
||||
|
||||
static std::optional<ManifestRetention>
|
||||
retentionOf(ManifestCache const& cache, PublicKey const& master)
|
||||
{
|
||||
std::optional<ManifestRetention> result;
|
||||
cache.for_each_manifest(
|
||||
[](std::size_t) {},
|
||||
[&result, &master](
|
||||
Manifest const& manifest, ManifestRetention retention) {
|
||||
if (manifest.masterKey == master)
|
||||
result = retention;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
void
|
||||
testLoadStore(ManifestCache& m)
|
||||
{
|
||||
@@ -287,25 +301,71 @@ public:
|
||||
env.journal);
|
||||
|
||||
{
|
||||
// save should not store untrusted master keys to db
|
||||
// except for revocations
|
||||
m.save(
|
||||
*dbCon,
|
||||
"ValidatorManifests",
|
||||
[&unl](PublicKey const& pubKey) {
|
||||
return unl->listed(pubKey);
|
||||
// Protection is current-source policy, not a second permanent
|
||||
// manifest authority. Source removal demotes revocations into
|
||||
// the ordinary bounded population. The existing wallet format
|
||||
// still saves retained revocations.
|
||||
ManifestCache persistence;
|
||||
auto const protectedSecret = randomSecretKey();
|
||||
auto const protectedSigning = randomKeyPair(KeyType::secp256k1);
|
||||
auto const protectedOld = makeManifest(
|
||||
protectedSecret,
|
||||
KeyType::ed25519,
|
||||
protectedSigning.second,
|
||||
KeyType::secp256k1,
|
||||
0);
|
||||
auto const protectedRevocation =
|
||||
makeRevocation(protectedSecret, KeyType::ed25519);
|
||||
auto const arbitrarySecret = randomSecretKey();
|
||||
auto const arbitraryRevocation =
|
||||
makeRevocation(arbitrarySecret, KeyType::ed25519);
|
||||
auto const protectedMaster = protectedRevocation.masterKey;
|
||||
auto const arbitraryMaster = arbitraryRevocation.masterKey;
|
||||
BEAST_EXPECT(
|
||||
persistence.applyManifest(
|
||||
clone(protectedRevocation),
|
||||
ManifestRetention::protected_) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(
|
||||
persistence.applyManifest(
|
||||
clone(arbitraryRevocation),
|
||||
ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
|
||||
persistence.reconcileRetention({});
|
||||
BEAST_EXPECT(
|
||||
retentionOf(persistence, protectedMaster) ==
|
||||
ManifestRetention::evictable);
|
||||
BEAST_EXPECT(
|
||||
retentionOf(persistence, arbitraryMaster) ==
|
||||
ManifestRetention::evictable);
|
||||
|
||||
persistence.save(
|
||||
*dbCon, "ValidatorManifests", [](PublicKey const&) {
|
||||
return false;
|
||||
});
|
||||
|
||||
ManifestCache loaded;
|
||||
|
||||
loaded.load(*dbCon, "ValidatorManifests");
|
||||
|
||||
// check that all loaded manifests are revocations
|
||||
std::vector<Manifest const*> const loadedManifests(
|
||||
sort(getPopulatedManifests(loaded)));
|
||||
|
||||
for (auto const& man : loadedManifests)
|
||||
BEAST_EXPECT(man->revoked());
|
||||
BEAST_EXPECT(loaded.revoked(protectedMaster));
|
||||
BEAST_EXPECT(loaded.revoked(arbitraryMaster));
|
||||
BEAST_EXPECT(
|
||||
retentionOf(loaded, protectedMaster) ==
|
||||
ManifestRetention::protected_);
|
||||
// Startup reconciliation is what returns database-only rows
|
||||
// to the bounded population once live sources are known.
|
||||
loaded.reconcileRetention({});
|
||||
BEAST_EXPECT(
|
||||
retentionOf(loaded, protectedMaster) ==
|
||||
ManifestRetention::evictable);
|
||||
BEAST_EXPECT(
|
||||
retentionOf(loaded, arbitraryMaster) ==
|
||||
ManifestRetention::evictable);
|
||||
BEAST_EXPECT(
|
||||
loaded.applyManifest(
|
||||
clone(protectedOld), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::stale);
|
||||
}
|
||||
{
|
||||
// save should store all trusted master keys to db
|
||||
@@ -415,12 +475,63 @@ public:
|
||||
cfgRevocation));
|
||||
|
||||
BEAST_EXPECT(loaded.revoked(pk));
|
||||
loaded.reconcileRetention({});
|
||||
BEAST_EXPECT(
|
||||
retentionOf(loaded, pk) == ManifestRetention::protected_);
|
||||
}
|
||||
}
|
||||
boost::filesystem::remove(
|
||||
getDatabasePath() / boost::filesystem::path(dbName));
|
||||
}
|
||||
|
||||
void
|
||||
testStartupRetentionReconciliation()
|
||||
{
|
||||
testcase("startup retention reconciliation");
|
||||
|
||||
jtx::Env env(*this);
|
||||
ManifestCache manifests;
|
||||
ManifestCache publisherManifests;
|
||||
auto const formerSecret = randomSecretKey();
|
||||
auto const formerSigning = randomKeyPair(KeyType::secp256k1);
|
||||
auto const former = makeManifest(
|
||||
formerSecret,
|
||||
KeyType::ed25519,
|
||||
formerSigning.second,
|
||||
KeyType::secp256k1,
|
||||
0);
|
||||
auto const revocation =
|
||||
makeRevocation(randomSecretKey(), KeyType::ed25519);
|
||||
auto const formerMaster = former.masterKey;
|
||||
auto const revokedMaster = revocation.masterKey;
|
||||
|
||||
// Database rows are provisionally loaded as protected before current
|
||||
// local and publisher sources are known.
|
||||
BEAST_EXPECT(
|
||||
manifests.applyManifest(
|
||||
clone(former), ManifestRetention::protected_) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(
|
||||
manifests.applyManifest(
|
||||
clone(revocation), ManifestRetention::protected_) ==
|
||||
ManifestDisposition::accepted);
|
||||
|
||||
ValidatorList validators(
|
||||
manifests,
|
||||
publisherManifests,
|
||||
env.timeKeeper(),
|
||||
env.app().config().legacy("database_path"),
|
||||
env.journal);
|
||||
BEAST_EXPECT(validators.load({}, {}, {}));
|
||||
|
||||
BEAST_EXPECT(
|
||||
retentionOf(manifests, formerMaster) ==
|
||||
ManifestRetention::evictable);
|
||||
BEAST_EXPECT(
|
||||
retentionOf(manifests, revokedMaster) ==
|
||||
ManifestRetention::evictable);
|
||||
}
|
||||
|
||||
void
|
||||
testGetSignature()
|
||||
{
|
||||
@@ -464,8 +575,10 @@ public:
|
||||
auto const kp0 = randomKeyPair(KeyType::secp256k1);
|
||||
BEAST_EXPECT(
|
||||
ManifestDisposition::accepted ==
|
||||
cache.applyManifest(makeManifest(
|
||||
sk, KeyType::ed25519, kp0.second, KeyType::secp256k1, 0)));
|
||||
cache.applyManifest(
|
||||
makeManifest(
|
||||
sk, KeyType::ed25519, kp0.second, KeyType::secp256k1, 0),
|
||||
ManifestRetention::evictable));
|
||||
BEAST_EXPECT(cache.getSigningKey(pk) == kp0.first);
|
||||
BEAST_EXPECT(cache.getMasterKey(kp0.first) == pk);
|
||||
|
||||
@@ -476,8 +589,10 @@ public:
|
||||
auto const kp1 = randomKeyPair(KeyType::secp256k1);
|
||||
BEAST_EXPECT(
|
||||
ManifestDisposition::accepted ==
|
||||
cache.applyManifest(makeManifest(
|
||||
sk, KeyType::ed25519, kp1.second, KeyType::secp256k1, 1)));
|
||||
cache.applyManifest(
|
||||
makeManifest(
|
||||
sk, KeyType::ed25519, kp1.second, KeyType::secp256k1, 1),
|
||||
ManifestRetention::evictable));
|
||||
BEAST_EXPECT(cache.getSigningKey(pk) == kp1.first);
|
||||
BEAST_EXPECT(cache.getMasterKey(kp1.first) == pk);
|
||||
BEAST_EXPECT(cache.getMasterKey(kp0.first) == kp0.first);
|
||||
@@ -486,8 +601,10 @@ public:
|
||||
// applied with the same signing key but a higher sequence
|
||||
BEAST_EXPECT(
|
||||
ManifestDisposition::badEphemeralKey ==
|
||||
cache.applyManifest(makeManifest(
|
||||
sk, KeyType::ed25519, kp1.second, KeyType::secp256k1, 2)));
|
||||
cache.applyManifest(
|
||||
makeManifest(
|
||||
sk, KeyType::ed25519, kp1.second, KeyType::secp256k1, 2),
|
||||
ManifestRetention::evictable));
|
||||
BEAST_EXPECT(cache.getSigningKey(pk) == kp1.first);
|
||||
BEAST_EXPECT(cache.getMasterKey(kp1.first) == pk);
|
||||
BEAST_EXPECT(cache.getMasterKey(kp0.first) == kp0.first);
|
||||
@@ -497,7 +614,9 @@ public:
|
||||
// key from a revoked master public key
|
||||
BEAST_EXPECT(
|
||||
ManifestDisposition::accepted ==
|
||||
cache.applyManifest(makeRevocation(sk, KeyType::ed25519)));
|
||||
cache.applyManifest(
|
||||
makeRevocation(sk, KeyType::ed25519),
|
||||
ManifestRetention::evictable));
|
||||
BEAST_EXPECT(cache.revoked(pk));
|
||||
BEAST_EXPECT(cache.getSigningKey(pk) == pk);
|
||||
BEAST_EXPECT(cache.getMasterKey(kp0.first) == kp0.first);
|
||||
@@ -975,6 +1094,319 @@ public:
|
||||
".example.com"));
|
||||
}
|
||||
|
||||
void
|
||||
testUntrustedEviction()
|
||||
{
|
||||
testcase("untrusted eviction");
|
||||
|
||||
auto now = std::chrono::steady_clock::time_point{};
|
||||
ManifestCache cache(
|
||||
beast::Journal(beast::Journal::getNullSink()),
|
||||
[&now] { return now; });
|
||||
|
||||
auto const trustedMasterSecret = randomSecretKey();
|
||||
auto const trustedMaster =
|
||||
derivePublicKey(KeyType::ed25519, trustedMasterSecret);
|
||||
auto const trustedSigning = randomKeyPair(KeyType::secp256k1);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(
|
||||
makeManifest(
|
||||
trustedMasterSecret,
|
||||
KeyType::ed25519,
|
||||
trustedSigning.second,
|
||||
KeyType::secp256k1,
|
||||
0),
|
||||
ManifestRetention::protected_) ==
|
||||
ManifestDisposition::accepted);
|
||||
|
||||
std::vector<PublicKey> untrustedMasters;
|
||||
std::vector<PublicKey> untrustedSigningKeys;
|
||||
untrustedMasters.reserve(1000);
|
||||
untrustedSigningKeys.reserve(1000);
|
||||
for (std::size_t i = 0; i < 1000; ++i)
|
||||
{
|
||||
auto const masterSecret = randomSecretKey();
|
||||
auto const master = derivePublicKey(KeyType::ed25519, masterSecret);
|
||||
auto const signing = randomKeyPair(KeyType::secp256k1);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(
|
||||
makeManifest(
|
||||
masterSecret,
|
||||
KeyType::ed25519,
|
||||
signing.second,
|
||||
KeyType::secp256k1,
|
||||
0),
|
||||
ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
untrustedMasters.push_back(master);
|
||||
untrustedSigningKeys.push_back(signing.first);
|
||||
}
|
||||
|
||||
auto retainedMasters = [&cache]() {
|
||||
hash_set<PublicKey> result;
|
||||
cache.for_each_manifest([&result](Manifest const& manifest) {
|
||||
result.insert(manifest.masterKey);
|
||||
});
|
||||
return result;
|
||||
};
|
||||
|
||||
auto const beforeInvalid = retainedMasters();
|
||||
auto const invalidMasterSecret = randomSecretKey();
|
||||
auto const invalidSigning = randomKeyPair(KeyType::secp256k1);
|
||||
bool sampledCurrentValidations = false;
|
||||
BEAST_EXPECT(
|
||||
cache
|
||||
.applyManifestWithEviction(
|
||||
makeManifest(
|
||||
invalidMasterSecret,
|
||||
KeyType::ed25519,
|
||||
invalidSigning.second,
|
||||
KeyType::secp256k1,
|
||||
0,
|
||||
true),
|
||||
[&sampledCurrentValidations] {
|
||||
sampledCurrentValidations = true;
|
||||
return hash_set<PublicKey>{};
|
||||
})
|
||||
.disposition == ManifestDisposition::invalid);
|
||||
BEAST_EXPECT(!sampledCurrentValidations);
|
||||
BEAST_EXPECT(retainedMasters() == beforeInvalid);
|
||||
|
||||
auto const incomingMasterSecret = randomSecretKey();
|
||||
auto const incomingMaster =
|
||||
derivePublicKey(KeyType::ed25519, incomingMasterSecret);
|
||||
auto const incomingSigning = randomKeyPair(KeyType::secp256k1);
|
||||
auto const incoming = makeManifest(
|
||||
incomingMasterSecret,
|
||||
KeyType::ed25519,
|
||||
incomingSigning.second,
|
||||
KeyType::secp256k1,
|
||||
0);
|
||||
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(
|
||||
clone(incoming), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::untrustedCapacity);
|
||||
|
||||
hash_set<PublicKey> const currentValidationKeys = {
|
||||
untrustedSigningKeys.front()};
|
||||
auto const incomingAdmission = cache.applyManifestWithEviction(
|
||||
clone(incoming),
|
||||
[¤tValidationKeys] { return currentValidationKeys; });
|
||||
BEAST_EXPECT(
|
||||
incomingAdmission.disposition == ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(!incomingAdmission.acceptedUpdate);
|
||||
|
||||
auto const afterEviction = retainedMasters();
|
||||
BEAST_EXPECT(afterEviction.size() == 1001);
|
||||
BEAST_EXPECT(afterEviction.contains(trustedMaster));
|
||||
BEAST_EXPECT(afterEviction.contains(untrustedMasters.front()));
|
||||
BEAST_EXPECT(afterEviction.contains(incomingMaster));
|
||||
|
||||
std::optional<std::size_t> evicted;
|
||||
for (std::size_t i = 0; i < untrustedMasters.size(); ++i)
|
||||
{
|
||||
if (!afterEviction.contains(untrustedMasters[i]))
|
||||
{
|
||||
BEAST_EXPECT(!evicted);
|
||||
evicted = i;
|
||||
}
|
||||
}
|
||||
BEAST_EXPECT(evicted);
|
||||
if (evicted)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
cache.getMasterKey(untrustedSigningKeys[*evicted]) ==
|
||||
untrustedSigningKeys[*evicted]);
|
||||
}
|
||||
|
||||
auto const replacementSigning = randomKeyPair(KeyType::secp256k1);
|
||||
auto const updateAdmission =
|
||||
cache.applyManifestWithEviction(makeManifest(
|
||||
incomingMasterSecret,
|
||||
KeyType::ed25519,
|
||||
replacementSigning.second,
|
||||
KeyType::secp256k1,
|
||||
1));
|
||||
BEAST_EXPECT(
|
||||
updateAdmission.disposition == ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(updateAdmission.acceptedUpdate);
|
||||
BEAST_EXPECT(retainedMasters() == afterEviction);
|
||||
|
||||
hash_set<PublicKey> allCurrentSigningKeys;
|
||||
cache.for_each_manifest(
|
||||
[&allCurrentSigningKeys](Manifest const& manifest) {
|
||||
if (manifest.signingKey)
|
||||
allCurrentSigningKeys.insert(*manifest.signingKey);
|
||||
});
|
||||
allCurrentSigningKeys.erase(replacementSigning.first);
|
||||
|
||||
auto const secondIncomingMasterSecret = randomSecretKey();
|
||||
auto const secondIncomingMaster =
|
||||
derivePublicKey(KeyType::ed25519, secondIncomingMasterSecret);
|
||||
auto const secondIncomingSigning = randomKeyPair(KeyType::secp256k1);
|
||||
BEAST_EXPECT(
|
||||
cache
|
||||
.applyManifestWithEviction(
|
||||
makeManifest(
|
||||
secondIncomingMasterSecret,
|
||||
KeyType::ed25519,
|
||||
secondIncomingSigning.second,
|
||||
KeyType::secp256k1,
|
||||
0),
|
||||
[&allCurrentSigningKeys] { return allCurrentSigningKeys; })
|
||||
.disposition == ManifestDisposition::accepted);
|
||||
auto const afterSoleDormantEviction = retainedMasters();
|
||||
BEAST_EXPECT(afterSoleDormantEviction.size() == 1001);
|
||||
BEAST_EXPECT(afterSoleDormantEviction.contains(trustedMaster));
|
||||
BEAST_EXPECT(afterSoleDormantEviction.contains(secondIncomingMaster));
|
||||
BEAST_EXPECT(!afterSoleDormantEviction.contains(incomingMaster));
|
||||
|
||||
// With no dormant victim, eviction falls back to the full evictable
|
||||
// population.
|
||||
allCurrentSigningKeys.clear();
|
||||
cache.for_each_manifest(
|
||||
[&allCurrentSigningKeys](Manifest const& manifest) {
|
||||
if (manifest.signingKey)
|
||||
allCurrentSigningKeys.insert(*manifest.signingKey);
|
||||
});
|
||||
auto const thirdIncomingMasterSecret = randomSecretKey();
|
||||
auto const thirdIncomingMaster =
|
||||
derivePublicKey(KeyType::ed25519, thirdIncomingMasterSecret);
|
||||
auto const thirdIncomingSigning = randomKeyPair(KeyType::secp256k1);
|
||||
BEAST_EXPECT(
|
||||
cache
|
||||
.applyManifestWithEviction(
|
||||
makeManifest(
|
||||
thirdIncomingMasterSecret,
|
||||
KeyType::ed25519,
|
||||
thirdIncomingSigning.second,
|
||||
KeyType::secp256k1,
|
||||
0),
|
||||
[&allCurrentSigningKeys] { return allCurrentSigningKeys; })
|
||||
.disposition == ManifestDisposition::accepted);
|
||||
auto const afterFallbackEviction = retainedMasters();
|
||||
BEAST_EXPECT(afterFallbackEviction.size() == 1001);
|
||||
BEAST_EXPECT(afterFallbackEviction.contains(trustedMaster));
|
||||
BEAST_EXPECT(afterFallbackEviction.contains(thirdIncomingMaster));
|
||||
|
||||
// Three eviction permits were consumed above. Time is held fixed while
|
||||
// the remainder of the burst is consumed.
|
||||
std::vector<Manifest> evictionBurst;
|
||||
evictionBurst.reserve(8);
|
||||
for (std::size_t i = 0; i < 8; ++i)
|
||||
{
|
||||
auto const masterSecret = randomSecretKey();
|
||||
auto const signing = randomKeyPair(KeyType::secp256k1);
|
||||
evictionBurst.push_back(makeManifest(
|
||||
masterSecret,
|
||||
KeyType::ed25519,
|
||||
signing.second,
|
||||
KeyType::secp256k1,
|
||||
0));
|
||||
}
|
||||
for (std::size_t i = 0; i < 7; ++i)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifestWithEviction(std::move(evictionBurst[i]))
|
||||
.disposition == ManifestDisposition::accepted);
|
||||
}
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifestWithEviction(std::move(evictionBurst.back()))
|
||||
.disposition == ManifestDisposition::untrustedCapacity);
|
||||
BEAST_EXPECT(retainedMasters().size() == 1001);
|
||||
|
||||
now += std::chrono::seconds{1};
|
||||
auto const refilledMasterSecret = randomSecretKey();
|
||||
auto const refilledSigning = randomKeyPair(KeyType::secp256k1);
|
||||
BEAST_EXPECT(
|
||||
cache
|
||||
.applyManifestWithEviction(makeManifest(
|
||||
refilledMasterSecret,
|
||||
KeyType::ed25519,
|
||||
refilledSigning.second,
|
||||
KeyType::secp256k1,
|
||||
0))
|
||||
.disposition == ManifestDisposition::accepted);
|
||||
|
||||
// Exhausting the eviction budget must not block an existing entry.
|
||||
auto const finalReplacementSigning = randomKeyPair(KeyType::secp256k1);
|
||||
BEAST_EXPECT(
|
||||
cache
|
||||
.applyManifestWithEviction(makeManifest(
|
||||
refilledMasterSecret,
|
||||
KeyType::ed25519,
|
||||
finalReplacementSigning.second,
|
||||
KeyType::secp256k1,
|
||||
1))
|
||||
.disposition == ManifestDisposition::accepted);
|
||||
}
|
||||
|
||||
void
|
||||
testRetentionInvalidatesSnapshot()
|
||||
{
|
||||
testcase("retention changes invalidate snapshot");
|
||||
|
||||
ManifestCache cache;
|
||||
auto const masterSecret = randomSecretKey();
|
||||
auto const master = derivePublicKey(KeyType::ed25519, masterSecret);
|
||||
auto const signing = randomKeyPair(KeyType::secp256k1);
|
||||
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(
|
||||
makeManifest(
|
||||
masterSecret,
|
||||
KeyType::ed25519,
|
||||
signing.second,
|
||||
KeyType::secp256k1,
|
||||
0),
|
||||
ManifestRetention::evictable) == ManifestDisposition::accepted);
|
||||
|
||||
auto const admitted = cache.sequence();
|
||||
cache.setRetention(randomMasterKey(), ManifestRetention::protected_);
|
||||
BEAST_EXPECT(cache.sequence() == admitted);
|
||||
|
||||
cache.setRetention(master, ManifestRetention::protected_);
|
||||
BEAST_EXPECT(cache.sequence() == admitted + 1);
|
||||
|
||||
cache.reconcileRetention({master});
|
||||
BEAST_EXPECT(cache.sequence() == admitted + 1);
|
||||
|
||||
cache.reconcileRetention({});
|
||||
BEAST_EXPECT(cache.sequence() == admitted + 2);
|
||||
BEAST_EXPECT(cache.getSequence(master) == 0);
|
||||
|
||||
cache.reconcileRetention({master});
|
||||
BEAST_EXPECT(cache.sequence() == admitted + 3);
|
||||
|
||||
cache.setRetention(master, ManifestRetention::protected_);
|
||||
BEAST_EXPECT(cache.sequence() == admitted + 3);
|
||||
|
||||
ManifestCache retryCache;
|
||||
auto const retryMasterSecret = randomSecretKey();
|
||||
auto const retrySigning = randomKeyPair(KeyType::secp256k1);
|
||||
BEAST_EXPECT(
|
||||
retryCache.applyManifest(
|
||||
makeManifest(
|
||||
retryMasterSecret,
|
||||
KeyType::ed25519,
|
||||
retrySigning.second,
|
||||
KeyType::secp256k1,
|
||||
0),
|
||||
ManifestRetention::evictable) == ManifestDisposition::accepted);
|
||||
auto const retryAdmitted = retryCache.sequence();
|
||||
BEAST_EXPECT(
|
||||
retryCache.applyManifest(
|
||||
makeManifest(
|
||||
retryMasterSecret,
|
||||
KeyType::ed25519,
|
||||
retrySigning.second,
|
||||
KeyType::secp256k1,
|
||||
0),
|
||||
ManifestRetention::protected_) == ManifestDisposition::stale);
|
||||
BEAST_EXPECT(retryCache.sequence() == retryAdmitted + 1);
|
||||
}
|
||||
|
||||
void
|
||||
run() override
|
||||
{
|
||||
@@ -1015,21 +1447,30 @@ public:
|
||||
// applyManifest should accept new manifests with
|
||||
// higher sequence numbers
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_a0)) ==
|
||||
cache.applyManifest(
|
||||
clone(s_a0), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_a0)) == ManifestDisposition::stale);
|
||||
cache.applyManifest(
|
||||
clone(s_a0), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::stale);
|
||||
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_a1)) ==
|
||||
cache.applyManifest(
|
||||
clone(s_a1), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_a1)) == ManifestDisposition::stale);
|
||||
cache.applyManifest(
|
||||
clone(s_a1), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::stale);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_a0)) == ManifestDisposition::stale);
|
||||
cache.applyManifest(
|
||||
clone(s_a0), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::stale);
|
||||
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_a2)) ==
|
||||
cache.applyManifest(
|
||||
clone(s_a2), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::badEphemeralKey);
|
||||
|
||||
// applyManifest should accept manifests with max sequence numbers
|
||||
@@ -1037,29 +1478,40 @@ public:
|
||||
BEAST_EXPECT(!cache.revoked(pk_a));
|
||||
BEAST_EXPECT(s_aMax.revoked());
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_aMax)) ==
|
||||
cache.applyManifest(
|
||||
clone(s_aMax), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_aMax)) ==
|
||||
cache.applyManifest(
|
||||
clone(s_aMax), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::stale);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_a1)) == ManifestDisposition::stale);
|
||||
cache.applyManifest(
|
||||
clone(s_a1), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::stale);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_a0)) == ManifestDisposition::stale);
|
||||
cache.applyManifest(
|
||||
clone(s_a0), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::stale);
|
||||
BEAST_EXPECT(cache.revoked(pk_a));
|
||||
|
||||
// applyManifest should reject manifests with invalid signatures
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_b0)) ==
|
||||
cache.applyManifest(
|
||||
clone(s_b0), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_b0)) == ManifestDisposition::stale);
|
||||
cache.applyManifest(
|
||||
clone(s_b0), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::stale);
|
||||
BEAST_EXPECT(!deserializeManifest(fake));
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_b1)) ==
|
||||
cache.applyManifest(
|
||||
clone(s_b1), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::invalid);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_b2)) ==
|
||||
cache.applyManifest(
|
||||
clone(s_b2), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
|
||||
auto const s_c0 = makeManifest(
|
||||
@@ -1069,17 +1521,21 @@ public:
|
||||
KeyType::ed25519,
|
||||
47);
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(clone(s_c0)) ==
|
||||
cache.applyManifest(
|
||||
clone(s_c0), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::badMasterKey);
|
||||
}
|
||||
|
||||
testLoadStore(cache);
|
||||
testStartupRetentionReconciliation();
|
||||
testGetSignature();
|
||||
testGetKeys();
|
||||
testValidatorToken();
|
||||
testManifestDeserialization();
|
||||
testManifestDomainNames();
|
||||
testManifestVersioning();
|
||||
testUntrustedEviction();
|
||||
testRetentionInvalidatesSnapshot();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -131,7 +131,8 @@ private:
|
||||
std::vector<Validator> const& validators,
|
||||
std::size_t sequence,
|
||||
std::size_t validUntil,
|
||||
std::optional<std::size_t> validFrom = {})
|
||||
std::optional<std::size_t> validFrom = {},
|
||||
std::vector<Validator> const& candidates = {})
|
||||
{
|
||||
std::string data = "{\"sequence\":" + std::to_string(sequence) +
|
||||
",\"expiration\":" + std::to_string(validUntil);
|
||||
@@ -142,11 +143,31 @@ private:
|
||||
for (auto const& val : validators)
|
||||
{
|
||||
data += "{\"validation_public_key\":\"" + strHex(val.masterPublic) +
|
||||
"\",\"manifest\":\"" + val.manifest + "\"},";
|
||||
"\"";
|
||||
if (!val.manifest.empty())
|
||||
data += ",\"manifest\":\"" + val.manifest + "\"";
|
||||
data += "},";
|
||||
}
|
||||
|
||||
data.pop_back();
|
||||
data += "]}";
|
||||
data += "]";
|
||||
|
||||
if (!candidates.empty())
|
||||
{
|
||||
data += ",\"candidates\":[";
|
||||
for (auto const& val : candidates)
|
||||
{
|
||||
data += "{\"validation_public_key\":\"" +
|
||||
strHex(val.masterPublic) + "\"";
|
||||
if (!val.manifest.empty())
|
||||
data += ",\"manifest\":\"" + val.manifest + "\"";
|
||||
data += "},";
|
||||
}
|
||||
data.pop_back();
|
||||
data += "]";
|
||||
}
|
||||
|
||||
data += "}";
|
||||
return base64_encode(data);
|
||||
}
|
||||
|
||||
@@ -285,7 +306,9 @@ private:
|
||||
localSigningPublicOuter, emptyCfgKeys, emptyCfgPublishers));
|
||||
BEAST_EXPECT(trustedKeys->listed(localSigningPublicOuter));
|
||||
|
||||
manifests.applyManifest(*deserializeManifest(cfgManifest));
|
||||
manifests.applyManifest(
|
||||
*deserializeManifest(cfgManifest),
|
||||
ManifestRetention::evictable);
|
||||
BEAST_EXPECT(trustedKeys->load(
|
||||
localSigningPublicOuter, emptyCfgKeys, emptyCfgPublishers));
|
||||
|
||||
@@ -382,7 +405,9 @@ private:
|
||||
app.config().legacy("database_path"),
|
||||
env.journal);
|
||||
|
||||
manifests.applyManifest(*deserializeManifest(cfgManifest));
|
||||
manifests.applyManifest(
|
||||
*deserializeManifest(cfgManifest),
|
||||
ManifestRetention::evictable);
|
||||
|
||||
BEAST_EXPECT(trustedKeys->load(
|
||||
localSigningPublicOuter, cfgKeys, emptyCfgPublishers));
|
||||
@@ -472,12 +497,14 @@ private:
|
||||
auto const pubRevokedSigning = randomKeyPair(KeyType::secp256k1);
|
||||
// make this manifest revoked (seq num = max)
|
||||
// -- thus should not be loaded
|
||||
pubManifests.applyManifest(*deserializeManifest(makeManifestString(
|
||||
pubRevokedPublic,
|
||||
pubRevokedSecret,
|
||||
pubRevokedSigning.first,
|
||||
pubRevokedSigning.second,
|
||||
std::numeric_limits<std::uint32_t>::max())));
|
||||
pubManifests.applyManifest(
|
||||
*deserializeManifest(makeManifestString(
|
||||
pubRevokedPublic,
|
||||
pubRevokedSecret,
|
||||
pubRevokedSigning.first,
|
||||
pubRevokedSigning.second,
|
||||
std::numeric_limits<std::uint32_t>::max())),
|
||||
ManifestRetention::evictable);
|
||||
|
||||
// these two are not revoked (and not in the manifest cache at all.)
|
||||
auto legitKey1 = randomMasterKey();
|
||||
@@ -511,12 +538,14 @@ private:
|
||||
auto const pubRevokedSigning = randomKeyPair(KeyType::secp256k1);
|
||||
// make this manifest revoked (seq num = max)
|
||||
// -- thus should not be loaded
|
||||
pubManifests.applyManifest(*deserializeManifest(makeManifestString(
|
||||
pubRevokedPublic,
|
||||
pubRevokedSecret,
|
||||
pubRevokedSigning.first,
|
||||
pubRevokedSigning.second,
|
||||
std::numeric_limits<std::uint32_t>::max())));
|
||||
pubManifests.applyManifest(
|
||||
*deserializeManifest(makeManifestString(
|
||||
pubRevokedPublic,
|
||||
pubRevokedSecret,
|
||||
pubRevokedSigning.first,
|
||||
pubRevokedSigning.second,
|
||||
std::numeric_limits<std::uint32_t>::max())),
|
||||
ManifestRetention::evictable);
|
||||
|
||||
// this one is not revoked (and not in the manifest cache at all.)
|
||||
auto legitKey = randomMasterKey();
|
||||
@@ -667,12 +696,17 @@ private:
|
||||
|
||||
checkResult(
|
||||
trustedKeys->applyLists(
|
||||
manifest1,
|
||||
version,
|
||||
{{expiredblob, expiredSig, {}}, {blob2, sig2, {}}},
|
||||
siteUri),
|
||||
manifest1, version, {{expiredblob, expiredSig, {}}}, siteUri),
|
||||
publisherPublic,
|
||||
ListDisposition::expired,
|
||||
ListDisposition::expired);
|
||||
expectUntrusted(lists.at(1));
|
||||
|
||||
checkResult(
|
||||
trustedKeys->applyLists(
|
||||
manifest1, version, {{blob2, sig2, {}}}, siteUri),
|
||||
publisherPublic,
|
||||
ListDisposition::accepted,
|
||||
ListDisposition::accepted);
|
||||
|
||||
expectTrusted(lists.at(2));
|
||||
@@ -982,6 +1016,439 @@ private:
|
||||
checkAvailable(trustedKeys, hexPublic, manifest2, 0, {});
|
||||
}
|
||||
|
||||
void
|
||||
testCandidates()
|
||||
{
|
||||
testcase("Publisher candidate manifests");
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
ManifestCache validatorManifests;
|
||||
ManifestCache publisherManifests;
|
||||
jtx::Env env(*this);
|
||||
auto& app = env.app();
|
||||
auto trustedKeys = std::make_unique<ValidatorList>(
|
||||
validatorManifests,
|
||||
publisherManifests,
|
||||
env.timeKeeper(),
|
||||
app.config().legacy("database_path"),
|
||||
env.journal);
|
||||
|
||||
auto const publisherSecret = randomSecretKey();
|
||||
auto const publisherPublic =
|
||||
derivePublicKey(KeyType::ed25519, publisherSecret);
|
||||
auto const publisherSigning = randomKeyPair(KeyType::secp256k1);
|
||||
auto const publisherManifest = base64_encode(makeManifestString(
|
||||
publisherPublic,
|
||||
publisherSecret,
|
||||
publisherSigning.first,
|
||||
publisherSigning.second,
|
||||
1));
|
||||
BEAST_EXPECT(trustedKeys->load(
|
||||
{}, {}, std::vector<std::string>{strHex(publisherPublic)}));
|
||||
|
||||
// Fill the bounded population. A publisher candidate is still admitted
|
||||
// because its tier changes retention policy, not manifest authority.
|
||||
for (std::size_t i = 0; i < ValidatorList::maxPublisherCandidates; ++i)
|
||||
{
|
||||
auto const filler = randomValidator();
|
||||
auto manifest = deserializeManifest(base64_decode(filler.manifest));
|
||||
BEAST_EXPECT(manifest);
|
||||
if (manifest)
|
||||
BEAST_EXPECT(
|
||||
validatorManifests.applyManifest(
|
||||
std::move(*manifest), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
}
|
||||
|
||||
auto const listed = randomValidator();
|
||||
auto const candidate = randomValidator();
|
||||
auto const validUntil = env.timeKeeper().now() + 1h;
|
||||
auto const blob = makeList(
|
||||
{listed},
|
||||
1,
|
||||
validUntil.time_since_epoch().count(),
|
||||
{},
|
||||
{candidate});
|
||||
auto const signature = signList(blob, publisherSigning);
|
||||
BEAST_EXPECT(
|
||||
trustedKeys
|
||||
->applyLists(
|
||||
publisherManifest,
|
||||
1,
|
||||
{{blob, signature, {}}},
|
||||
"testCandidates.test")
|
||||
.bestDisposition() == ListDisposition::accepted);
|
||||
|
||||
BEAST_EXPECT(trustedKeys->listed(listed.masterPublic));
|
||||
BEAST_EXPECT(!trustedKeys->listed(candidate.masterPublic));
|
||||
BEAST_EXPECT(!trustedKeys->trusted(candidate.masterPublic));
|
||||
auto const policy = trustedKeys->manifestPolicy(candidate.masterPublic);
|
||||
BEAST_EXPECT(!policy.consensusListed);
|
||||
BEAST_EXPECT(policy.publisherCandidate);
|
||||
BEAST_EXPECT(policy.relayEligible());
|
||||
|
||||
// All provenance classes share ManifestCache's one high-water and
|
||||
// signer-to-master index. Candidate status only protects retention.
|
||||
BEAST_EXPECT(
|
||||
validatorManifests.getMasterKey(candidate.signingPublic) ==
|
||||
candidate.masterPublic);
|
||||
BEAST_EXPECT(
|
||||
validatorManifests.getSigningKey(candidate.masterPublic) ==
|
||||
candidate.signingPublic);
|
||||
BEAST_EXPECT(
|
||||
validatorManifests.getSequence(candidate.masterPublic) == 1);
|
||||
|
||||
auto const candidateMasterSecret = randomSecretKey();
|
||||
auto const candidateMaster =
|
||||
derivePublicKey(KeyType::ed25519, candidateMasterSecret);
|
||||
auto const candidateSigning1 = randomKeyPair(KeyType::secp256k1);
|
||||
auto const candidateSigning2 = randomKeyPair(KeyType::secp256k1);
|
||||
Validator const rotatingCandidate{
|
||||
candidateMaster,
|
||||
candidateSigning1.first,
|
||||
base64_encode(makeManifestString(
|
||||
candidateMaster,
|
||||
candidateMasterSecret,
|
||||
candidateSigning1.first,
|
||||
candidateSigning1.second,
|
||||
1))};
|
||||
auto const blob2 = makeList(
|
||||
{listed},
|
||||
2,
|
||||
validUntil.time_since_epoch().count(),
|
||||
{},
|
||||
{candidate, rotatingCandidate});
|
||||
auto const signature2 = signList(blob2, publisherSigning);
|
||||
BEAST_EXPECT(
|
||||
trustedKeys
|
||||
->applyLists(
|
||||
publisherManifest,
|
||||
1,
|
||||
{{blob2, signature2, {}}},
|
||||
"testCandidates.test")
|
||||
.bestDisposition() == ListDisposition::accepted);
|
||||
|
||||
auto rotation = deserializeManifest(makeManifestString(
|
||||
candidateMaster,
|
||||
candidateMasterSecret,
|
||||
candidateSigning2.first,
|
||||
candidateSigning2.second,
|
||||
2));
|
||||
BEAST_EXPECT(rotation);
|
||||
if (rotation)
|
||||
BEAST_EXPECT(
|
||||
validatorManifests.applyManifest(
|
||||
std::move(*rotation), ManifestRetention::protected_) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(
|
||||
validatorManifests.getSigningKey(candidateMaster) ==
|
||||
candidateSigning2.first);
|
||||
|
||||
// Removing the candidate removes only its protection. Because the
|
||||
// bounded population is already full, its transient manifest is
|
||||
// discarded rather than creating a second retention pool.
|
||||
auto const blob3 =
|
||||
makeList({listed}, 3, validUntil.time_since_epoch().count());
|
||||
auto const signature3 = signList(blob3, publisherSigning);
|
||||
BEAST_EXPECT(
|
||||
trustedKeys
|
||||
->applyLists(
|
||||
publisherManifest,
|
||||
1,
|
||||
{{blob3, signature3, {}}},
|
||||
"testCandidates.test")
|
||||
.bestDisposition() == ListDisposition::accepted);
|
||||
BEAST_EXPECT(
|
||||
!trustedKeys->manifestPolicy(candidateMaster).publisherCandidate);
|
||||
BEAST_EXPECT(!validatorManifests.getSequence(candidateMaster));
|
||||
|
||||
// Protection follows the current union of configured/listed and
|
||||
// candidate masters. Removing a tier-1 entry demotes it through the
|
||||
// same reconciliation path; it does not remain protected forever.
|
||||
auto const replacementListed = randomValidator();
|
||||
auto const blob4 = makeList(
|
||||
{replacementListed}, 4, validUntil.time_since_epoch().count());
|
||||
auto const signature4 = signList(blob4, publisherSigning);
|
||||
BEAST_EXPECT(
|
||||
trustedKeys
|
||||
->applyLists(
|
||||
publisherManifest,
|
||||
1,
|
||||
{{blob4, signature4, {}}},
|
||||
"testCandidates.test")
|
||||
.bestDisposition() == ListDisposition::accepted);
|
||||
BEAST_EXPECT(!trustedKeys->listed(listed.masterPublic));
|
||||
BEAST_EXPECT(!validatorManifests.getSequence(listed.masterPublic));
|
||||
BEAST_EXPECT(trustedKeys->listed(replacementListed.masterPublic));
|
||||
}
|
||||
|
||||
void
|
||||
testCandidateAggregateCap()
|
||||
{
|
||||
testcase("Publisher candidate aggregate cap");
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
struct Publisher
|
||||
{
|
||||
PublicKey master;
|
||||
std::pair<PublicKey, SecretKey> signing;
|
||||
std::string manifest;
|
||||
};
|
||||
|
||||
auto makePublisher = []() {
|
||||
auto const masterSecret = randomSecretKey();
|
||||
auto const master = derivePublicKey(KeyType::ed25519, masterSecret);
|
||||
auto signing = randomKeyPair(KeyType::secp256k1);
|
||||
auto manifest = base64_encode(makeManifestString(
|
||||
master, masterSecret, signing.first, signing.second, 1));
|
||||
return Publisher{master, std::move(signing), std::move(manifest)};
|
||||
};
|
||||
|
||||
std::array<Publisher, 2> const publishers{
|
||||
makePublisher(), makePublisher()};
|
||||
std::array<std::vector<Validator>, 2> candidates;
|
||||
std::vector<PublicKey> candidateMasters;
|
||||
candidateMasters.reserve(1200);
|
||||
for (auto& publisherCandidates : candidates)
|
||||
{
|
||||
publisherCandidates.reserve(600);
|
||||
for (std::size_t i = 0; i < 600; ++i)
|
||||
{
|
||||
auto candidate = randomValidator();
|
||||
candidateMasters.push_back(candidate.masterPublic);
|
||||
publisherCandidates.push_back(std::move(candidate));
|
||||
}
|
||||
}
|
||||
std::sort(candidateMasters.begin(), candidateMasters.end());
|
||||
|
||||
jtx::Env env(*this);
|
||||
auto const validUntil = env.timeKeeper().now() + 1h;
|
||||
auto const listed = randomValidator();
|
||||
std::array<std::string, 2> blobs;
|
||||
std::array<std::string, 2> signatures;
|
||||
for (std::size_t i = 0; i < publishers.size(); ++i)
|
||||
{
|
||||
blobs[i] = makeList(
|
||||
{listed},
|
||||
1,
|
||||
validUntil.time_since_epoch().count(),
|
||||
{},
|
||||
candidates[i]);
|
||||
signatures[i] = signList(blobs[i], publishers[i].signing);
|
||||
}
|
||||
|
||||
auto checkOrder = [&](std::array<std::size_t, 2> const& order) {
|
||||
ManifestCache validatorManifests;
|
||||
ManifestCache publisherManifests;
|
||||
auto trustedKeys = std::make_unique<ValidatorList>(
|
||||
validatorManifests,
|
||||
publisherManifests,
|
||||
env.timeKeeper(),
|
||||
env.app().config().legacy("database_path"),
|
||||
env.journal);
|
||||
BEAST_EXPECT(trustedKeys->load(
|
||||
{},
|
||||
{},
|
||||
std::vector<std::string>{
|
||||
strHex(publishers[0].master),
|
||||
strHex(publishers[1].master)}));
|
||||
|
||||
for (auto const i : order)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
trustedKeys
|
||||
->applyLists(
|
||||
publishers[i].manifest,
|
||||
1,
|
||||
{{blobs[i], signatures[i], {}}},
|
||||
"testCandidateAggregateCap.test")
|
||||
.bestDisposition() == ListDisposition::accepted);
|
||||
}
|
||||
|
||||
hash_set<PublicKey> protectedMasters;
|
||||
validatorManifests.for_each_manifest(
|
||||
[&protectedMasters](std::size_t size) {
|
||||
protectedMasters.reserve(size);
|
||||
},
|
||||
[&protectedMasters](
|
||||
Manifest const& manifest, ManifestRetention retention) {
|
||||
if (retention == ManifestRetention::protected_)
|
||||
protectedMasters.insert(manifest.masterKey);
|
||||
});
|
||||
|
||||
for (std::size_t i = 0; i < candidateMasters.size(); ++i)
|
||||
{
|
||||
auto const& master = candidateMasters[i];
|
||||
auto const policy = trustedKeys->manifestPolicy(master);
|
||||
bool const retained = i < ValidatorList::maxPublisherCandidates;
|
||||
BEAST_EXPECT(policy.publisherCandidate == retained);
|
||||
BEAST_EXPECT(policy.relayEligible() == retained);
|
||||
BEAST_EXPECT(protectedMasters.contains(master) == retained);
|
||||
}
|
||||
};
|
||||
|
||||
checkOrder({0, 1});
|
||||
checkOrder({1, 0});
|
||||
}
|
||||
|
||||
void
|
||||
testExpiredPendingAccounting()
|
||||
{
|
||||
testcase("Expired pending list preserves other publisher counts");
|
||||
using namespace std::chrono_literals;
|
||||
|
||||
struct Publisher
|
||||
{
|
||||
PublicKey master;
|
||||
std::pair<PublicKey, SecretKey> signing;
|
||||
std::string manifest;
|
||||
};
|
||||
|
||||
auto makePublisher = []() {
|
||||
auto const masterSecret = randomSecretKey();
|
||||
auto const master = derivePublicKey(KeyType::ed25519, masterSecret);
|
||||
auto signing = randomKeyPair(KeyType::secp256k1);
|
||||
auto manifest = base64_encode(makeManifestString(
|
||||
master, masterSecret, signing.first, signing.second, 1));
|
||||
return Publisher{master, std::move(signing), std::move(manifest)};
|
||||
};
|
||||
|
||||
ManifestCache validatorManifests;
|
||||
ManifestCache publisherManifests;
|
||||
jtx::Env env(*this);
|
||||
auto& app = env.app();
|
||||
auto validators = std::make_unique<ValidatorList>(
|
||||
validatorManifests,
|
||||
publisherManifests,
|
||||
env.timeKeeper(),
|
||||
app.config().legacy("database_path"),
|
||||
env.journal);
|
||||
|
||||
auto const publisherA = makePublisher();
|
||||
auto const publisherB = makePublisher();
|
||||
BEAST_EXPECT(validators->load(
|
||||
{}, {}, {strHex(publisherA.master), strHex(publisherB.master)}));
|
||||
|
||||
auto const shared = randomValidator();
|
||||
auto const replacement = randomValidator();
|
||||
auto const now = env.timeKeeper().now();
|
||||
std::string const siteUri = "expired-pending-accounting.test";
|
||||
|
||||
auto const currentBlob =
|
||||
makeList({shared}, 1, (now + 1h).time_since_epoch().count());
|
||||
auto const currentSig = signList(currentBlob, publisherB.signing);
|
||||
checkResult(
|
||||
validators->applyLists(
|
||||
publisherB.manifest,
|
||||
2,
|
||||
{{currentBlob, currentSig, {}}},
|
||||
siteUri),
|
||||
publisherB.master,
|
||||
ListDisposition::accepted,
|
||||
ListDisposition::accepted);
|
||||
BEAST_EXPECT(validators->listed(shared.masterPublic));
|
||||
|
||||
auto const pendingBlob = makeList(
|
||||
{shared},
|
||||
1,
|
||||
(now + 20s).time_since_epoch().count(),
|
||||
(now + 10s).time_since_epoch().count());
|
||||
auto const pendingSig = signList(pendingBlob, publisherA.signing);
|
||||
checkResult(
|
||||
validators->applyLists(
|
||||
publisherA.manifest,
|
||||
2,
|
||||
{{pendingBlob, pendingSig, {}}},
|
||||
siteUri),
|
||||
publisherA.master,
|
||||
ListDisposition::pending,
|
||||
ListDisposition::pending);
|
||||
|
||||
env.timeKeeper().set(now + 21s);
|
||||
validators->updateTrusted(
|
||||
{},
|
||||
env.timeKeeper().now(),
|
||||
app.getOPs(),
|
||||
app.overlay(),
|
||||
app.getHashRouter());
|
||||
BEAST_EXPECT(app.getOPs().isUNLBlocked());
|
||||
BEAST_EXPECT(validators->listed(shared.masterPublic));
|
||||
|
||||
auto const refreshBlob =
|
||||
makeList({replacement}, 2, (now + 1h).time_since_epoch().count());
|
||||
auto const refreshSig = signList(refreshBlob, publisherA.signing);
|
||||
checkResult(
|
||||
validators->applyLists(
|
||||
publisherA.manifest,
|
||||
2,
|
||||
{{refreshBlob, refreshSig, {}}},
|
||||
siteUri),
|
||||
publisherA.master,
|
||||
ListDisposition::accepted,
|
||||
ListDisposition::accepted);
|
||||
|
||||
// Publisher A never contributed the shared key, so refreshing A must
|
||||
// not remove publisher B's contribution.
|
||||
BEAST_EXPECT(validators->listed(shared.masterPublic));
|
||||
BEAST_EXPECT(validators->listed(replacement.masterPublic));
|
||||
|
||||
// Reapplying a pending list after its expiry exercises applyList's
|
||||
// remaining-to-current recovery path rather than updateTrusted.
|
||||
app.getOPs().clearUNLBlocked();
|
||||
auto const expiredOnly = randomValidator();
|
||||
auto const pendingAgain = makeList(
|
||||
{expiredOnly},
|
||||
3,
|
||||
(env.timeKeeper().now() + 20s).time_since_epoch().count(),
|
||||
(env.timeKeeper().now() + 10s).time_since_epoch().count());
|
||||
auto const pendingAgainSig = signList(pendingAgain, publisherA.signing);
|
||||
checkResult(
|
||||
validators->applyLists(
|
||||
publisherA.manifest,
|
||||
2,
|
||||
{{pendingAgain, pendingAgainSig, {}}},
|
||||
siteUri),
|
||||
publisherA.master,
|
||||
ListDisposition::pending,
|
||||
ListDisposition::pending);
|
||||
|
||||
env.timeKeeper().set(env.timeKeeper().now() + 21s);
|
||||
checkResult(
|
||||
validators->applyListsAndBroadcast(
|
||||
publisherA.manifest,
|
||||
2,
|
||||
{{pendingAgain, pendingAgainSig, {}}},
|
||||
siteUri,
|
||||
uint256{},
|
||||
app.overlay(),
|
||||
app.getHashRouter(),
|
||||
app.getOPs()),
|
||||
publisherA.master,
|
||||
ListDisposition::expired,
|
||||
ListDisposition::expired);
|
||||
BEAST_EXPECT(app.getOPs().isUNLBlocked());
|
||||
BEAST_EXPECT(!validators->listed(replacement.masterPublic));
|
||||
BEAST_EXPECT(!validators->listed(expiredOnly.masterPublic));
|
||||
BEAST_EXPECT(validators->listed(shared.masterPublic));
|
||||
|
||||
// Publisher A never contributed expiredOnly. When publisher B moves
|
||||
// away from shared, neither key may survive through phantom counts.
|
||||
auto const replacementB = randomValidator();
|
||||
auto const refreshB = makeList(
|
||||
{replacementB},
|
||||
2,
|
||||
(env.timeKeeper().now() + 1h).time_since_epoch().count());
|
||||
auto const refreshBSig = signList(refreshB, publisherB.signing);
|
||||
checkResult(
|
||||
validators->applyLists(
|
||||
publisherB.manifest, 2, {{refreshB, refreshBSig, {}}}, siteUri),
|
||||
publisherB.master,
|
||||
ListDisposition::accepted,
|
||||
ListDisposition::accepted);
|
||||
BEAST_EXPECT(!validators->listed(shared.masterPublic));
|
||||
BEAST_EXPECT(validators->listed(replacementB.masterPublic));
|
||||
}
|
||||
|
||||
void
|
||||
testGetAvailable()
|
||||
{
|
||||
@@ -1237,7 +1704,8 @@ private:
|
||||
1));
|
||||
|
||||
BEAST_EXPECT(
|
||||
manifestsOuter.applyManifest(std::move(*m1)) ==
|
||||
manifestsOuter.applyManifest(
|
||||
std::move(*m1), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(trustedKeysOuter->listed(masterPublic));
|
||||
BEAST_EXPECT(trustedKeysOuter->trusted(masterPublic));
|
||||
@@ -1255,7 +1723,8 @@ private:
|
||||
signingKeys2.second,
|
||||
2));
|
||||
BEAST_EXPECT(
|
||||
manifestsOuter.applyManifest(std::move(*m2)) ==
|
||||
manifestsOuter.applyManifest(
|
||||
std::move(*m2), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(trustedKeysOuter->listed(masterPublic));
|
||||
BEAST_EXPECT(trustedKeysOuter->trusted(masterPublic));
|
||||
@@ -1273,7 +1742,8 @@ private:
|
||||
|
||||
BEAST_EXPECT(mMax->revoked());
|
||||
BEAST_EXPECT(
|
||||
manifestsOuter.applyManifest(std::move(*mMax)) ==
|
||||
manifestsOuter.applyManifest(
|
||||
std::move(*mMax), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
BEAST_EXPECT(
|
||||
manifestsOuter.getSigningKey(masterPublic) == masterPublic);
|
||||
@@ -2772,7 +3242,8 @@ private:
|
||||
if (self)
|
||||
{
|
||||
valManifests.applyManifest(
|
||||
*deserializeManifest(base64_decode(self->manifest)));
|
||||
*deserializeManifest(base64_decode(self->manifest)),
|
||||
ManifestRetention::evictable);
|
||||
BEAST_EXPECT(result->load(
|
||||
self->signingPublic,
|
||||
emptyCfgKeys,
|
||||
@@ -4134,6 +4605,9 @@ public:
|
||||
testGenesisQuorum();
|
||||
testConfigLoad();
|
||||
testApplyLists();
|
||||
testCandidates();
|
||||
testCandidateAggregateCap();
|
||||
testExpiredPendingAccounting();
|
||||
testGetAvailable();
|
||||
testUpdateTrusted();
|
||||
testExpires();
|
||||
|
||||
@@ -135,7 +135,7 @@ private:
|
||||
std::string msg;
|
||||
bool ssl;
|
||||
bool failFetch = false;
|
||||
bool failApply = false;
|
||||
bool expectUnlisted = false;
|
||||
int serverVersion = 1;
|
||||
std::chrono::seconds expiresFromNow = detail::default_expires;
|
||||
std::chrono::seconds effectiveOverlap =
|
||||
@@ -252,9 +252,11 @@ private:
|
||||
for (auto const& val : u.list)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
trustedKeys.listed(val.masterPublic) != u.cfg.failApply);
|
||||
trustedKeys.listed(val.masterPublic) !=
|
||||
u.cfg.expectUnlisted);
|
||||
BEAST_EXPECT(
|
||||
trustedKeys.listed(val.signingPublic) != u.cfg.failApply);
|
||||
trustedKeys.listed(val.signingPublic) !=
|
||||
u.cfg.expectUnlisted);
|
||||
}
|
||||
|
||||
Json::Value myStatus;
|
||||
@@ -570,7 +572,7 @@ public:
|
||||
"Applied 1 expired validator list(s)",
|
||||
ssl,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
1,
|
||||
0s}});
|
||||
testFetchList(
|
||||
@@ -579,7 +581,7 @@ public:
|
||||
"Applied 1 expired validator list(s)",
|
||||
ssl,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
1,
|
||||
0s,
|
||||
-1s}});
|
||||
|
||||
@@ -564,6 +564,10 @@ class Validations_test : public beast::unit_test::suite
|
||||
{
|
||||
hash_set<PeerID> const expectedKeys = {a.nodeID(), b.nodeID()};
|
||||
BEAST_EXPECT(harness.vals().getCurrentNodeIDs() == expectedKeys);
|
||||
hash_set<PeerKey> const expectedSigningKeys = {
|
||||
a.currKey(), b.currKey()};
|
||||
BEAST_EXPECT(
|
||||
harness.vals().getCurrentNodeKeys() == expectedSigningKeys);
|
||||
}
|
||||
|
||||
harness.clock().advance(3s);
|
||||
@@ -579,11 +583,16 @@ class Validations_test : public beast::unit_test::suite
|
||||
{
|
||||
hash_set<PeerID> const expectedKeys = {a.nodeID(), b.nodeID()};
|
||||
BEAST_EXPECT(harness.vals().getCurrentNodeIDs() == expectedKeys);
|
||||
hash_set<PeerKey> const expectedSigningKeys = {
|
||||
a.currKey(), b.currKey()};
|
||||
BEAST_EXPECT(
|
||||
harness.vals().getCurrentNodeKeys() == expectedSigningKeys);
|
||||
}
|
||||
|
||||
// Pass enough time for them to go stale
|
||||
harness.clock().advance(harness.parms().validationCURRENT_LOCAL);
|
||||
BEAST_EXPECT(harness.vals().getCurrentNodeIDs().empty());
|
||||
BEAST_EXPECT(harness.vals().getCurrentNodeKeys().empty());
|
||||
}
|
||||
|
||||
void
|
||||
|
||||
@@ -45,6 +45,7 @@
|
||||
#include <boost/beast/core/multi_buffer.hpp>
|
||||
#include <boost/endian/conversion.hpp>
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
|
||||
namespace ripple {
|
||||
|
||||
@@ -74,6 +75,95 @@ class compression_test : public beast::unit_test::suite
|
||||
using Compressed = compression::Compressed;
|
||||
using Algorithm = compression::Algorithm;
|
||||
|
||||
class ProtocolHandler
|
||||
{
|
||||
public:
|
||||
bool invoked = false;
|
||||
|
||||
bool
|
||||
compressionEnabled() const
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
onMessageUnknown(std::uint16_t)
|
||||
{
|
||||
invoked = true;
|
||||
}
|
||||
|
||||
void
|
||||
onMessageBegin(
|
||||
std::uint16_t,
|
||||
std::shared_ptr<::google::protobuf::Message> const&,
|
||||
std::size_t,
|
||||
std::size_t,
|
||||
bool)
|
||||
{
|
||||
invoked = true;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void
|
||||
onMessage(std::shared_ptr<T> const&)
|
||||
{
|
||||
invoked = true;
|
||||
}
|
||||
|
||||
void
|
||||
onMessageEnd(
|
||||
std::uint16_t,
|
||||
std::shared_ptr<::google::protobuf::Message> const&)
|
||||
{
|
||||
invoked = true;
|
||||
}
|
||||
};
|
||||
|
||||
static void
|
||||
put16(
|
||||
std::vector<std::uint8_t>& frame,
|
||||
std::size_t const offset,
|
||||
std::uint16_t const value)
|
||||
{
|
||||
frame[offset] = static_cast<std::uint8_t>(value >> 8);
|
||||
frame[offset + 1] = static_cast<std::uint8_t>(value);
|
||||
}
|
||||
|
||||
static void
|
||||
put32(
|
||||
std::vector<std::uint8_t>& frame,
|
||||
std::size_t const offset,
|
||||
std::uint32_t const value)
|
||||
{
|
||||
frame[offset] = static_cast<std::uint8_t>(value >> 24);
|
||||
frame[offset + 1] = static_cast<std::uint8_t>(value >> 16);
|
||||
frame[offset + 2] = static_cast<std::uint8_t>(value >> 8);
|
||||
frame[offset + 3] = static_cast<std::uint8_t>(value);
|
||||
}
|
||||
|
||||
static std::vector<std::uint8_t>
|
||||
uncompressedManifestFrame(std::size_t const payloadSize)
|
||||
{
|
||||
std::vector<std::uint8_t> frame(compression::headerBytes + payloadSize);
|
||||
put32(frame, 0, static_cast<std::uint32_t>(payloadSize));
|
||||
put16(frame, 4, protocol::mtMANIFESTS);
|
||||
return frame;
|
||||
}
|
||||
|
||||
static std::vector<std::uint8_t>
|
||||
compressedManifestFrame(
|
||||
std::size_t const payloadSize,
|
||||
std::size_t const uncompressedSize)
|
||||
{
|
||||
std::vector<std::uint8_t> frame(
|
||||
compression::headerBytesCompressed + payloadSize);
|
||||
put32(frame, 0, static_cast<std::uint32_t>(payloadSize));
|
||||
frame[0] |= static_cast<std::uint8_t>(Algorithm::LZ4);
|
||||
put16(frame, 4, protocol::mtMANIFESTS);
|
||||
put32(frame, 6, static_cast<std::uint32_t>(uncompressedSize));
|
||||
return frame;
|
||||
}
|
||||
|
||||
public:
|
||||
compression_test()
|
||||
{
|
||||
@@ -529,11 +619,55 @@ public:
|
||||
handshake(0, 0);
|
||||
}
|
||||
|
||||
void
|
||||
testManifestFrameLimit()
|
||||
{
|
||||
testcase("TMManifests frame limit");
|
||||
|
||||
auto invokeFrame = [](std::vector<std::uint8_t> const& frame,
|
||||
ProtocolHandler& handler) {
|
||||
std::array<boost::asio::const_buffer, 1> const buffers{
|
||||
boost::asio::buffer(frame)};
|
||||
std::size_t hint = 0;
|
||||
return invokeProtocolMessage(buffers, handler, hint);
|
||||
};
|
||||
|
||||
{
|
||||
auto const frame =
|
||||
uncompressedManifestFrame(maximumManifestsMessageSize + 1);
|
||||
ProtocolHandler handler;
|
||||
auto const [consumed, ec] = invokeFrame(frame, handler);
|
||||
BEAST_EXPECT(consumed == frame.size());
|
||||
BEAST_EXPECT(!ec);
|
||||
BEAST_EXPECT(!handler.invoked);
|
||||
}
|
||||
{
|
||||
auto const frame =
|
||||
compressedManifestFrame(1, maximumManifestsMessageSize + 1);
|
||||
ProtocolHandler handler;
|
||||
auto const [consumed, ec] = invokeFrame(frame, handler);
|
||||
BEAST_EXPECT(consumed == frame.size());
|
||||
BEAST_EXPECT(!ec);
|
||||
BEAST_EXPECT(!handler.invoked);
|
||||
}
|
||||
{
|
||||
auto const frame =
|
||||
uncompressedManifestFrame(maximumManifestsMessageSize);
|
||||
ProtocolHandler handler;
|
||||
auto const [consumed, ec] = invokeFrame(frame, handler);
|
||||
BEAST_EXPECT(consumed == frame.size());
|
||||
BEAST_EXPECT(
|
||||
ec == make_error_code(boost::system::errc::bad_message));
|
||||
BEAST_EXPECT(!handler.invoked);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
run() override
|
||||
{
|
||||
testProtocol();
|
||||
testHandshake();
|
||||
testManifestFrameLimit();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
785
src/test/overlay/manifest_relay_test.cpp
Normal file
785
src/test/overlay/manifest_relay_test.cpp
Normal file
@@ -0,0 +1,785 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of rippled: https://github.com/ripple/rippled
|
||||
Copyright 2026 Xahau Ledger Foundation.
|
||||
|
||||
Permission to use, copy, modify, and/or distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include <test/jtx.h>
|
||||
#include <test/jtx/Env.h>
|
||||
#include <xrpld/app/misc/ValidatorList.h>
|
||||
#include <xrpld/overlay/detail/OverlayImpl.h>
|
||||
#include <xrpld/overlay/detail/PeerImp.h>
|
||||
#include <xrpld/peerfinder/detail/SlotImp.h>
|
||||
#include <xrpl/basics/make_SSLContext.h>
|
||||
#include <xrpl/beast/unit_test.h>
|
||||
#include <xrpl/protocol/STExchange.h>
|
||||
#include <xrpl/protocol/Sign.h>
|
||||
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
|
||||
namespace ripple {
|
||||
namespace test {
|
||||
|
||||
class manifest_relay_test : public beast::unit_test::suite
|
||||
{
|
||||
using socket_type = boost::asio::ip::tcp::socket;
|
||||
using middle_type = boost::beast::tcp_stream;
|
||||
using stream_type = boost::beast::ssl_stream<middle_type>;
|
||||
|
||||
class PeerTest : public PeerImp
|
||||
{
|
||||
public:
|
||||
PeerTest(
|
||||
Application& app,
|
||||
std::shared_ptr<PeerFinder::Slot> const& slot,
|
||||
http_request_type&& request,
|
||||
PublicKey const& publicKey,
|
||||
ProtocolVersion protocol,
|
||||
Resource::Consumer consumer,
|
||||
std::unique_ptr<manifest_relay_test::stream_type>&& stream,
|
||||
OverlayImpl& overlay)
|
||||
: PeerImp(
|
||||
app,
|
||||
nextId_++,
|
||||
slot,
|
||||
std::move(request),
|
||||
publicKey,
|
||||
protocol,
|
||||
consumer,
|
||||
std::move(stream),
|
||||
overlay)
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
run() override
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
send(std::shared_ptr<Message> const& message) override
|
||||
{
|
||||
sent_.push_back(message);
|
||||
}
|
||||
|
||||
std::vector<std::shared_ptr<Message>> sent_;
|
||||
inline static Peer::id_t nextId_ = 1;
|
||||
};
|
||||
|
||||
std::shared_ptr<boost::asio::ssl::context> context_{make_SSLContext("")};
|
||||
std::uint16_t endpoint_{1};
|
||||
|
||||
struct ManifestData
|
||||
{
|
||||
std::string serialized;
|
||||
PublicKey masterKey;
|
||||
std::optional<PublicKey> signingKey;
|
||||
};
|
||||
|
||||
ManifestData
|
||||
makeManifest(
|
||||
SecretKey const& masterSecret,
|
||||
std::pair<PublicKey, SecretKey> const& signing,
|
||||
std::uint32_t const sequence,
|
||||
bool const large = false)
|
||||
{
|
||||
auto const masterPublic =
|
||||
derivePublicKey(KeyType::ed25519, masterSecret);
|
||||
|
||||
STObject st{sfGeneric};
|
||||
st[sfSequence] = sequence;
|
||||
st[sfPublicKey] = masterPublic;
|
||||
st[sfSigningPubKey] = signing.first;
|
||||
if (large)
|
||||
{
|
||||
st[sfDomain] =
|
||||
makeSlice(std::string(63, 'a') + "." + std::string(63, 'b'));
|
||||
}
|
||||
sign(st, HashPrefix::manifest, KeyType::secp256k1, signing.second);
|
||||
sign(
|
||||
st,
|
||||
HashPrefix::manifest,
|
||||
KeyType::ed25519,
|
||||
masterSecret,
|
||||
sfMasterSignature);
|
||||
|
||||
Serializer serialized;
|
||||
st.add(serialized);
|
||||
std::string const bytes{
|
||||
static_cast<char const*>(serialized.data()), serialized.size()};
|
||||
auto manifest = deserializeManifest(bytes);
|
||||
BEAST_EXPECT(manifest.has_value());
|
||||
return {bytes, masterPublic, signing.first};
|
||||
}
|
||||
|
||||
ManifestData
|
||||
makeManifest(
|
||||
SecretKey const& masterSecret,
|
||||
std::uint32_t const sequence,
|
||||
bool const large = false)
|
||||
{
|
||||
return makeManifest(
|
||||
masterSecret, randomKeyPair(KeyType::secp256k1), sequence, large);
|
||||
}
|
||||
|
||||
ManifestData
|
||||
makeRevocation(SecretKey const& masterSecret)
|
||||
{
|
||||
auto const masterPublic =
|
||||
derivePublicKey(KeyType::ed25519, masterSecret);
|
||||
|
||||
STObject st{sfGeneric};
|
||||
st[sfSequence] = std::numeric_limits<std::uint32_t>::max();
|
||||
st[sfPublicKey] = masterPublic;
|
||||
sign(
|
||||
st,
|
||||
HashPrefix::manifest,
|
||||
KeyType::ed25519,
|
||||
masterSecret,
|
||||
sfMasterSignature);
|
||||
|
||||
Serializer serialized;
|
||||
st.add(serialized);
|
||||
std::string const bytes{
|
||||
static_cast<char const*>(serialized.data()), serialized.size()};
|
||||
auto manifest = deserializeManifest(bytes);
|
||||
BEAST_EXPECT(manifest.has_value());
|
||||
BEAST_EXPECT(manifest && manifest->revoked());
|
||||
return {bytes, masterPublic, std::nullopt};
|
||||
}
|
||||
|
||||
ManifestData
|
||||
makeManifest(bool const large = false)
|
||||
{
|
||||
return makeManifest(randomSecretKey(), 0, large);
|
||||
}
|
||||
|
||||
std::shared_ptr<PeerTest>
|
||||
addPeer(jtx::Env& env)
|
||||
{
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
boost::beast::http::request<boost::beast::http::dynamic_body> request;
|
||||
auto stream = std::make_unique<stream_type>(
|
||||
socket_type(env.app().getIOService()), *context_);
|
||||
|
||||
auto const octet = endpoint_++;
|
||||
beast::IP::Endpoint const local{beast::IP::Address::from_string(
|
||||
"172.1.1." + std::to_string(octet))};
|
||||
beast::IP::Endpoint const remote{beast::IP::Address::from_string(
|
||||
"172.1.2." + std::to_string(octet))};
|
||||
auto const peerPublic = std::get<0>(randomKeyPair(KeyType::ed25519));
|
||||
auto consumer = overlay.resourceManager().newInboundEndpoint(remote);
|
||||
auto slot = overlay.peerFinder().new_inbound_slot(local, remote);
|
||||
auto peer = std::make_shared<PeerTest>(
|
||||
env.app(),
|
||||
slot,
|
||||
std::move(request),
|
||||
peerPublic,
|
||||
ProtocolVersion{1, 7},
|
||||
consumer,
|
||||
std::move(stream),
|
||||
overlay);
|
||||
overlay.add_active(peer);
|
||||
return peer;
|
||||
}
|
||||
|
||||
static std::optional<protocol::TMManifests>
|
||||
unpack(std::shared_ptr<Message> const& message)
|
||||
{
|
||||
if (!message)
|
||||
return std::nullopt;
|
||||
|
||||
auto const& buffer = message->getBuffer(compression::Compressed::Off);
|
||||
if (buffer.size() <= compression::headerBytes)
|
||||
return std::nullopt;
|
||||
|
||||
protocol::TMManifests manifests;
|
||||
if (!manifests.ParseFromArray(
|
||||
buffer.data() + compression::headerBytes,
|
||||
static_cast<int>(buffer.size() - compression::headerBytes)))
|
||||
return std::nullopt;
|
||||
return manifests;
|
||||
}
|
||||
|
||||
static std::optional<protocol::TMManifests>
|
||||
unpackFirst(std::vector<std::shared_ptr<Message>> const& messages)
|
||||
{
|
||||
if (messages.empty())
|
||||
return std::nullopt;
|
||||
return unpack(messages.front());
|
||||
}
|
||||
|
||||
void
|
||||
expectBundle(std::shared_ptr<PeerTest> const& peer, std::size_t const count)
|
||||
{
|
||||
BEAST_EXPECT(peer->sent_.size() == 1);
|
||||
if (peer->sent_.size() != 1)
|
||||
return;
|
||||
|
||||
auto const relayed = unpack(peer->sent_.front());
|
||||
BEAST_EXPECT(relayed.has_value());
|
||||
if (relayed)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
static_cast<std::size_t>(relayed->list_size()) == count);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
installCandidate(jtx::Env& env, ManifestData const& candidate)
|
||||
{
|
||||
auto const publisherMasterSecret = randomSecretKey();
|
||||
auto const publisherMaster =
|
||||
derivePublicKey(KeyType::ed25519, publisherMasterSecret);
|
||||
auto const publisherSigning = randomKeyPair(KeyType::secp256k1);
|
||||
auto const publisherManifest =
|
||||
makeManifest(publisherMasterSecret, publisherSigning, 1);
|
||||
auto const listed = makeManifest();
|
||||
|
||||
BEAST_EXPECT(env.app().validators().load(
|
||||
{}, {}, std::vector<std::string>{strHex(publisherMaster)}));
|
||||
|
||||
auto const expiration = (env.timeKeeper().now() + std::chrono::hours{1})
|
||||
.time_since_epoch()
|
||||
.count();
|
||||
std::string const payload =
|
||||
"{\"sequence\":1,\"expiration\":" + std::to_string(expiration) +
|
||||
",\"validators\":[{\"validation_public_key\":\"" +
|
||||
strHex(listed.masterKey) +
|
||||
"\"}],\"candidates\":[{\"validation_public_key\":\"" +
|
||||
strHex(candidate.masterKey) + "\",\"manifest\":\"" +
|
||||
base64_encode(candidate.serialized) + "\"}]}";
|
||||
auto const blob = base64_encode(payload);
|
||||
auto const signature = strHex(sign(
|
||||
publisherSigning.first,
|
||||
publisherSigning.second,
|
||||
makeSlice(payload)));
|
||||
|
||||
BEAST_EXPECT(
|
||||
env.app()
|
||||
.validators()
|
||||
.applyLists(
|
||||
base64_encode(publisherManifest.serialized),
|
||||
1,
|
||||
{{blob, signature, {}}},
|
||||
"manifest_relay_test")
|
||||
.bestDisposition() == ListDisposition::accepted);
|
||||
auto const policy =
|
||||
env.app().validators().manifestPolicy(candidate.masterKey);
|
||||
BEAST_EXPECT(!policy.consensusListed);
|
||||
BEAST_EXPECT(policy.publisherCandidate);
|
||||
BEAST_EXPECT(policy.relayEligible());
|
||||
}
|
||||
|
||||
void
|
||||
testCandidateUpdateRelays()
|
||||
{
|
||||
testcase("publisher candidate rotations and revocations relay");
|
||||
|
||||
jtx::Env env{*this};
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
auto const source = addPeer(env);
|
||||
auto const destination = addPeer(env);
|
||||
auto const masterSecret = randomSecretKey();
|
||||
auto const published = makeManifest(masterSecret, 1);
|
||||
auto const rotation = makeManifest(masterSecret, 2);
|
||||
auto const revocation = makeRevocation(masterSecret);
|
||||
|
||||
installCandidate(env, published);
|
||||
BEAST_EXPECT(!env.app().validators().listed(published.masterKey));
|
||||
BEAST_EXPECT(!env.app().validators().trusted(published.masterKey));
|
||||
BEAST_EXPECT(
|
||||
env.app().validatorManifests().getSequence(published.masterKey) ==
|
||||
1);
|
||||
|
||||
auto send = [&](ManifestData const& data) {
|
||||
auto incoming = std::make_shared<protocol::TMManifests>();
|
||||
incoming->add_list()->set_stobject(data.serialized);
|
||||
overlay.onManifests(incoming, source);
|
||||
};
|
||||
|
||||
send(rotation);
|
||||
expectBundle(destination, 1);
|
||||
BEAST_EXPECT(
|
||||
env.app().validatorManifests().getSequence(published.masterKey) ==
|
||||
2);
|
||||
BEAST_EXPECT(
|
||||
env.app().validatorManifests().getSigningKey(published.masterKey) ==
|
||||
rotation.signingKey);
|
||||
BEAST_EXPECT(!env.app().validators().listed(published.masterKey));
|
||||
BEAST_EXPECT(!env.app().validators().trusted(published.masterKey));
|
||||
|
||||
auto snapshot = unpackFirst(overlay.getManifestsMessages());
|
||||
BEAST_EXPECT(snapshot.has_value());
|
||||
if (snapshot)
|
||||
{
|
||||
BEAST_EXPECT(std::any_of(
|
||||
snapshot->list().begin(),
|
||||
snapshot->list().end(),
|
||||
[&](auto const& entry) {
|
||||
return entry.stobject() == rotation.serialized;
|
||||
}));
|
||||
}
|
||||
|
||||
destination->sent_.clear();
|
||||
send(revocation);
|
||||
expectBundle(destination, 1);
|
||||
BEAST_EXPECT(
|
||||
!env.app().validatorManifests().getSequence(published.masterKey));
|
||||
BEAST_EXPECT(
|
||||
env.app().validatorManifests().revoked(published.masterKey));
|
||||
BEAST_EXPECT(!env.app().validators().listed(published.masterKey));
|
||||
BEAST_EXPECT(!env.app().validators().trusted(published.masterKey));
|
||||
}
|
||||
|
||||
void
|
||||
testCandidateCannotRollbackOrdinaryHighWater()
|
||||
{
|
||||
testcase("candidate cannot roll back ordinary manifest high-water");
|
||||
|
||||
jtx::Env env{*this};
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
auto const source = addPeer(env);
|
||||
auto const destination = addPeer(env);
|
||||
auto const masterSecret = randomSecretKey();
|
||||
auto const published = makeManifest(masterSecret, 1);
|
||||
auto const staleRotation = makeManifest(masterSecret, 2);
|
||||
auto const revocation = makeRevocation(masterSecret);
|
||||
|
||||
auto ordinary = deserializeManifest(revocation.serialized);
|
||||
BEAST_EXPECT(ordinary.has_value());
|
||||
if (ordinary)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
env.app().validatorManifests().applyManifest(
|
||||
std::move(*ordinary), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
}
|
||||
installCandidate(env, published);
|
||||
|
||||
auto incoming = std::make_shared<protocol::TMManifests>();
|
||||
incoming->add_list()->set_stobject(staleRotation.serialized);
|
||||
overlay.onManifests(incoming, source);
|
||||
|
||||
BEAST_EXPECT(destination->sent_.empty());
|
||||
BEAST_EXPECT(
|
||||
env.app().validatorManifests().revoked(published.masterKey));
|
||||
|
||||
auto const snapshot = unpackFirst(overlay.getManifestsMessages());
|
||||
BEAST_EXPECT(snapshot.has_value());
|
||||
if (snapshot)
|
||||
{
|
||||
BEAST_EXPECT(std::any_of(
|
||||
snapshot->list().begin(),
|
||||
snapshot->list().end(),
|
||||
[&](auto const& entry) {
|
||||
return entry.stobject() == revocation.serialized;
|
||||
}));
|
||||
BEAST_EXPECT(std::none_of(
|
||||
snapshot->list().begin(),
|
||||
snapshot->list().end(),
|
||||
[&](auto const& entry) {
|
||||
return entry.stobject() == staleRotation.serialized;
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
testBoundedReceive()
|
||||
{
|
||||
testcase("bounded receive includes trusted entries after the cap");
|
||||
|
||||
auto const trusted = makeManifest();
|
||||
jtx::Env env{
|
||||
*this, jtx::envconfig([&trusted](std::unique_ptr<Config> config) {
|
||||
config->section("validators")
|
||||
.append(toBase58(TokenType::NodePublic, trusted.masterKey));
|
||||
return config;
|
||||
})};
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
auto const source = addPeer(env);
|
||||
auto const destination = addPeer(env);
|
||||
|
||||
auto incoming = std::make_shared<protocol::TMManifests>();
|
||||
for (std::size_t i = 0; i < kMaxManifestsPerMessage + 1; ++i)
|
||||
incoming->add_list()->set_stobject(makeManifest().serialized);
|
||||
incoming->add_list()->set_stobject(trusted.serialized);
|
||||
|
||||
auto const before = env.app().validatorManifests().sequence();
|
||||
overlay.onManifests(incoming, source);
|
||||
|
||||
BEAST_EXPECT(
|
||||
env.app().validatorManifests().sequence() ==
|
||||
before + kMaxManifestsPerMessage + 1);
|
||||
BEAST_EXPECT(source->sent_.empty());
|
||||
expectBundle(destination, 1);
|
||||
|
||||
auto const snapshot = unpackFirst(overlay.getManifestsMessages());
|
||||
BEAST_EXPECT(snapshot.has_value());
|
||||
if (snapshot)
|
||||
{
|
||||
BEAST_EXPECT(snapshot->list_size() == kMaxManifestsPerMessage + 1);
|
||||
BEAST_EXPECT(
|
||||
Message::messageSize(*snapshot) <= maximumManifestsMessageSize);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
testTotalEntryLimit()
|
||||
{
|
||||
testcase("total entry limit bounds malformed work");
|
||||
|
||||
jtx::Env env{*this};
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
auto const source = addPeer(env);
|
||||
auto const destination = addPeer(env);
|
||||
|
||||
auto incoming = std::make_shared<protocol::TMManifests>();
|
||||
for (std::size_t i = 0; i < kMaxManifestEntriesPerMessage + 1; ++i)
|
||||
incoming->add_list()->set_stobject("");
|
||||
|
||||
BEAST_EXPECT(incoming->IsInitialized());
|
||||
auto const before = env.app().validatorManifests().sequence();
|
||||
overlay.onManifests(incoming, source);
|
||||
|
||||
BEAST_EXPECT(env.app().validatorManifests().sequence() == before);
|
||||
BEAST_EXPECT(source->sent_.empty());
|
||||
BEAST_EXPECT(destination->sent_.empty());
|
||||
}
|
||||
|
||||
void
|
||||
testKnownUpdateRelays()
|
||||
{
|
||||
testcase("known unlisted rotations relay");
|
||||
|
||||
jtx::Env env{*this};
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
auto const source = addPeer(env);
|
||||
auto const destination = addPeer(env);
|
||||
auto const masterSecret = randomSecretKey();
|
||||
auto const first = makeManifest(masterSecret, 0);
|
||||
auto const second = makeManifest(masterSecret, 1);
|
||||
|
||||
auto incoming = std::make_shared<protocol::TMManifests>();
|
||||
incoming->add_list()->set_stobject(first.serialized);
|
||||
overlay.onManifests(incoming, source);
|
||||
|
||||
BEAST_EXPECT(source->sent_.empty());
|
||||
BEAST_EXPECT(destination->sent_.empty());
|
||||
|
||||
incoming = std::make_shared<protocol::TMManifests>();
|
||||
incoming->add_list()->set_stobject(second.serialized);
|
||||
overlay.onManifests(incoming, source);
|
||||
|
||||
BEAST_EXPECT(source->sent_.empty());
|
||||
expectBundle(destination, 1);
|
||||
}
|
||||
|
||||
void
|
||||
testRevocationRelay()
|
||||
{
|
||||
testcase("revocation relay follows retained and listed policy");
|
||||
|
||||
auto const retainedSecret = randomSecretKey();
|
||||
auto const retained = makeManifest(retainedSecret, 0);
|
||||
auto const retainedRevocation = makeRevocation(retainedSecret);
|
||||
auto const firstSeenRevocation = makeRevocation(randomSecretKey());
|
||||
auto const listedRevocation = makeRevocation(randomSecretKey());
|
||||
|
||||
jtx::Env env{
|
||||
*this,
|
||||
jtx::envconfig([&listedRevocation](std::unique_ptr<Config> config) {
|
||||
config->section("validators")
|
||||
.append(toBase58(
|
||||
TokenType::NodePublic, listedRevocation.masterKey));
|
||||
return config;
|
||||
})};
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
auto const source = addPeer(env);
|
||||
auto const destination = addPeer(env);
|
||||
|
||||
auto send = [&](ManifestData const& data) {
|
||||
auto incoming = std::make_shared<protocol::TMManifests>();
|
||||
incoming->add_list()->set_stobject(data.serialized);
|
||||
overlay.onManifests(incoming, source);
|
||||
};
|
||||
|
||||
send(retained);
|
||||
BEAST_EXPECT(destination->sent_.empty());
|
||||
|
||||
send(retainedRevocation);
|
||||
expectBundle(destination, 1);
|
||||
BEAST_EXPECT(
|
||||
env.app().validatorManifests().revoked(retained.masterKey));
|
||||
destination->sent_.clear();
|
||||
|
||||
send(firstSeenRevocation);
|
||||
BEAST_EXPECT(destination->sent_.empty());
|
||||
BEAST_EXPECT(env.app().validatorManifests().revoked(
|
||||
firstSeenRevocation.masterKey));
|
||||
|
||||
send(listedRevocation);
|
||||
expectBundle(destination, 1);
|
||||
BEAST_EXPECT(
|
||||
env.app().validatorManifests().revoked(listedRevocation.masterKey));
|
||||
BEAST_EXPECT(source->sent_.empty());
|
||||
}
|
||||
|
||||
void
|
||||
testEvictedManifestStopsLocally()
|
||||
{
|
||||
testcase("reaccepted evicted manifest does not live relay");
|
||||
|
||||
jtx::Env env{*this};
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
auto const source = addPeer(env);
|
||||
auto const destination = addPeer(env);
|
||||
auto const target = makeManifest();
|
||||
|
||||
auto incoming = std::make_shared<protocol::TMManifests>();
|
||||
incoming->add_list()->set_stobject(target.serialized);
|
||||
overlay.onManifests(incoming, source);
|
||||
BEAST_EXPECT(source->sent_.empty());
|
||||
BEAST_EXPECT(destination->sent_.empty());
|
||||
|
||||
constexpr std::size_t untrustedCacheLimit = 1000;
|
||||
std::vector<ManifestData> retained;
|
||||
retained.reserve(untrustedCacheLimit);
|
||||
retained.push_back(target);
|
||||
auto& cache = env.app().validatorManifests();
|
||||
for (std::size_t i = 1; i < untrustedCacheLimit; ++i)
|
||||
{
|
||||
auto const filler = makeManifest();
|
||||
auto manifest = deserializeManifest(filler.serialized);
|
||||
BEAST_EXPECT(manifest.has_value());
|
||||
if (manifest)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(
|
||||
std::move(*manifest), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
}
|
||||
retained.push_back(filler);
|
||||
}
|
||||
|
||||
auto const replacement = makeManifest();
|
||||
auto replacementManifest = deserializeManifest(replacement.serialized);
|
||||
BEAST_EXPECT(replacementManifest.has_value());
|
||||
if (replacementManifest)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifestWithEviction(std::move(*replacementManifest))
|
||||
.disposition == ManifestDisposition::accepted);
|
||||
}
|
||||
|
||||
auto const evicted = std::find_if(
|
||||
retained.begin(), retained.end(), [&cache](ManifestData const& m) {
|
||||
return !cache.getSequence(m.masterKey);
|
||||
});
|
||||
BEAST_EXPECT(evicted != retained.end());
|
||||
if (evicted == retained.end())
|
||||
return;
|
||||
|
||||
incoming = std::make_shared<protocol::TMManifests>();
|
||||
incoming->add_list()->set_stobject(evicted->serialized);
|
||||
overlay.onManifests(incoming, source);
|
||||
|
||||
BEAST_EXPECT(cache.getSequence(evicted->masterKey) == 0);
|
||||
BEAST_EXPECT(source->sent_.empty());
|
||||
BEAST_EXPECT(destination->sent_.empty());
|
||||
}
|
||||
|
||||
void
|
||||
testSnapshotByteBudget()
|
||||
{
|
||||
testcase("snapshot chunks every protected entry within byte budget");
|
||||
|
||||
constexpr std::size_t trustedCount = 300;
|
||||
constexpr std::size_t untrustedCount = 100;
|
||||
std::vector<ManifestData> trusted;
|
||||
std::vector<ManifestData> untrusted;
|
||||
trusted.reserve(trustedCount);
|
||||
untrusted.reserve(untrustedCount);
|
||||
for (std::size_t i = 0; i < trustedCount; ++i)
|
||||
trusted.push_back(makeManifest(true));
|
||||
for (std::size_t i = 0; i < untrustedCount; ++i)
|
||||
untrusted.push_back(makeManifest());
|
||||
|
||||
jtx::Env env{
|
||||
*this, jtx::envconfig([&trusted](std::unique_ptr<Config> config) {
|
||||
auto& validators = config->section("validators");
|
||||
for (auto const& manifest : trusted)
|
||||
{
|
||||
validators.append(
|
||||
toBase58(TokenType::NodePublic, manifest.masterKey));
|
||||
}
|
||||
return config;
|
||||
})};
|
||||
|
||||
hash_set<PublicKey> trustedMasters;
|
||||
trustedMasters.reserve(trusted.size());
|
||||
auto& cache = env.app().validatorManifests();
|
||||
for (auto const& data : trusted)
|
||||
{
|
||||
BEAST_EXPECT(env.app().validators().listed(data.masterKey));
|
||||
trustedMasters.insert(data.masterKey);
|
||||
auto manifest = deserializeManifest(data.serialized);
|
||||
BEAST_EXPECT(manifest.has_value());
|
||||
if (manifest)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(
|
||||
std::move(*manifest), ManifestRetention::protected_) ==
|
||||
ManifestDisposition::accepted);
|
||||
}
|
||||
}
|
||||
for (auto const& data : untrusted)
|
||||
{
|
||||
BEAST_EXPECT(!env.app().validators().listed(data.masterKey));
|
||||
auto manifest = deserializeManifest(data.serialized);
|
||||
BEAST_EXPECT(manifest.has_value());
|
||||
if (manifest)
|
||||
{
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(
|
||||
std::move(*manifest), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
}
|
||||
}
|
||||
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
auto const messages = overlay.getManifestsMessages();
|
||||
BEAST_EXPECT(messages.size() > 1);
|
||||
|
||||
hash_set<PublicKey> seenTrusted;
|
||||
std::size_t seenUntrusted = 0;
|
||||
for (auto const& message : messages)
|
||||
{
|
||||
auto const snapshot = unpack(message);
|
||||
BEAST_EXPECT(snapshot.has_value());
|
||||
if (!snapshot)
|
||||
continue;
|
||||
BEAST_EXPECT(
|
||||
Message::messageSize(*snapshot) <= maximumManifestsMessageSize);
|
||||
BEAST_EXPECT(
|
||||
static_cast<std::size_t>(snapshot->list_size()) <=
|
||||
kMaxManifestEntriesPerMessage);
|
||||
for (auto const& entry : snapshot->list())
|
||||
{
|
||||
auto manifest = deserializeManifest(entry.stobject());
|
||||
BEAST_EXPECT(manifest.has_value());
|
||||
if (!manifest)
|
||||
continue;
|
||||
if (trustedMasters.contains(manifest->masterKey))
|
||||
seenTrusted.insert(manifest->masterKey);
|
||||
else
|
||||
++seenUntrusted;
|
||||
}
|
||||
}
|
||||
BEAST_EXPECT(seenTrusted.size() == trustedCount);
|
||||
BEAST_EXPECT(seenUntrusted <= untrustedCount);
|
||||
}
|
||||
|
||||
void
|
||||
testListingChangeInvalidatesSnapshot()
|
||||
{
|
||||
testcase("retention change invalidates cached snapshot");
|
||||
|
||||
jtx::Env env{*this};
|
||||
auto const target = makeManifest();
|
||||
auto manifest = deserializeManifest(target.serialized);
|
||||
BEAST_EXPECT(manifest.has_value());
|
||||
if (!manifest)
|
||||
return;
|
||||
|
||||
auto& cache = env.app().validatorManifests();
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(
|
||||
std::move(*manifest), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
|
||||
auto& overlay = dynamic_cast<OverlayImpl&>(env.app().overlay());
|
||||
auto const first = overlay.getManifestsMessages();
|
||||
BEAST_EXPECT(first.size() == 1);
|
||||
auto const manifestSeq = cache.sequence();
|
||||
|
||||
BEAST_EXPECT(env.app().validators().load(
|
||||
std::nullopt,
|
||||
{toBase58(TokenType::NodePublic, target.masterKey)},
|
||||
{},
|
||||
std::nullopt));
|
||||
BEAST_EXPECT(env.app().validators().listed(target.masterKey));
|
||||
BEAST_EXPECT(cache.sequence() == manifestSeq + 1);
|
||||
|
||||
auto const second = overlay.getManifestsMessages();
|
||||
BEAST_EXPECT(second.size() == 1);
|
||||
if (first.size() == 1 && second.size() == 1)
|
||||
BEAST_EXPECT(second.front() != first.front());
|
||||
}
|
||||
|
||||
void
|
||||
testConfigListingPromotesRetention()
|
||||
{
|
||||
testcase("config listing promotes retained manifest");
|
||||
|
||||
jtx::Env env{*this};
|
||||
auto const target = makeManifest();
|
||||
auto manifest = deserializeManifest(target.serialized);
|
||||
BEAST_EXPECT(manifest.has_value());
|
||||
if (!manifest)
|
||||
return;
|
||||
|
||||
auto& cache = env.app().validatorManifests();
|
||||
BEAST_EXPECT(
|
||||
cache.applyManifest(
|
||||
std::move(*manifest), ManifestRetention::evictable) ==
|
||||
ManifestDisposition::accepted);
|
||||
auto const before = cache.sequence();
|
||||
|
||||
BEAST_EXPECT(env.app().validators().load(
|
||||
std::nullopt,
|
||||
{toBase58(TokenType::NodePublic, target.masterKey)},
|
||||
{},
|
||||
std::nullopt));
|
||||
BEAST_EXPECT(env.app().validators().listed(target.masterKey));
|
||||
BEAST_EXPECT(cache.sequence() == before + 1);
|
||||
|
||||
cache.setRetention(target.masterKey, ManifestRetention::protected_);
|
||||
BEAST_EXPECT(cache.sequence() == before + 1);
|
||||
}
|
||||
|
||||
public:
|
||||
void
|
||||
run() override
|
||||
{
|
||||
testBoundedReceive();
|
||||
testTotalEntryLimit();
|
||||
testKnownUpdateRelays();
|
||||
testCandidateUpdateRelays();
|
||||
testCandidateCannotRollbackOrdinaryHighWater();
|
||||
testRevocationRelay();
|
||||
testEvictedManifestStopsLocally();
|
||||
testSnapshotByteBudget();
|
||||
testListingChangeInvalidatesSnapshot();
|
||||
testConfigListingPromotesRetention();
|
||||
}
|
||||
};
|
||||
|
||||
BEAST_DEFINE_TESTSUITE(manifest_relay, overlay, ripple);
|
||||
|
||||
} // namespace test
|
||||
} // namespace ripple
|
||||
@@ -21,13 +21,22 @@
|
||||
#define RIPPLE_APP_MISC_MANIFEST_H_INCLUDED
|
||||
|
||||
#include <xrpl/basics/UnorderedContainers.h>
|
||||
#include <xrpl/basics/base64.h>
|
||||
#include <xrpl/beast/utility/Journal.h>
|
||||
#include <xrpl/protocol/PublicKey.h>
|
||||
#include <xrpl/protocol/SecretKey.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <shared_mutex>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
namespace ripple {
|
||||
|
||||
@@ -54,12 +63,19 @@ namespace ripple {
|
||||
dynamically generates the signatureless form when it needs to verify
|
||||
the signature.
|
||||
|
||||
An instance of ManifestCache stores, for each trusted validator, (a) its
|
||||
An instance of ManifestCache stores, for each known validator, (a) its
|
||||
master public key, and (b) the most senior of all valid manifests it has
|
||||
seen for that validator, if any. On startup, the [validator_token] config
|
||||
entry (which contains the manifest for this validator) is decoded and
|
||||
added to the manifest cache. Other manifests are added as "gossip"
|
||||
received from rippled peers.
|
||||
received from rippled peers, including ones for validators this node does
|
||||
not trust. Manifests for untrusted validators are capped
|
||||
(kMaxUntrustedCount) so peer gossip cannot grow the cache without bound.
|
||||
Entries admitted as protected, or later reclassified as protected, are not
|
||||
capped or evicted. Entries return to the evictable population when their
|
||||
protected source disappears. At capacity, an evictable entry is evicted to
|
||||
admit a new valid manifest, preferring one with no current validation when
|
||||
activity information is available.
|
||||
|
||||
When an ephemeral key is compromised, a new signing key pair is created,
|
||||
along with a new manifest vouching for it (with a higher sequence number),
|
||||
@@ -151,6 +167,23 @@ struct Manifest
|
||||
std::string
|
||||
to_string(Manifest const& m);
|
||||
|
||||
/** Largest a valid manifest can be, in decoded bytes. */
|
||||
constexpr std::size_t kMaxManifestBytes = 358;
|
||||
|
||||
/** Largest a valid manifest can be, in base64 characters. */
|
||||
constexpr std::size_t kMaxManifestBase64 =
|
||||
base64::encoded_size(kMaxManifestBytes);
|
||||
|
||||
/** Maximum number of untrusted manifests processed from one message. */
|
||||
constexpr std::size_t kMaxManifestsPerMessage = 200;
|
||||
|
||||
/** Maximum total manifest entries processed from one message.
|
||||
|
||||
This is separate from the wire-byte limit and the valid-untrusted limit:
|
||||
malformed entries can be only a few bytes each and never reach the latter.
|
||||
*/
|
||||
constexpr std::size_t kMaxManifestEntriesPerMessage = 1000;
|
||||
|
||||
/** Constructs Manifest from serialized string
|
||||
|
||||
@param s Serialized manifest string
|
||||
@@ -226,7 +259,22 @@ enum class ManifestDisposition {
|
||||
badEphemeralKey,
|
||||
|
||||
/// Timely, but invalid signature
|
||||
invalid
|
||||
invalid,
|
||||
|
||||
/// Unlisted and limit reached
|
||||
untrustedCapacity
|
||||
};
|
||||
|
||||
/** Result of bounded untrusted-manifest admission.
|
||||
|
||||
`acceptedUpdate` is meaningful only when `disposition` is `accepted`. It is
|
||||
computed under the cache write lock and distinguishes an update of a
|
||||
retained identity from admission as a new identity.
|
||||
*/
|
||||
struct ManifestApplyResult
|
||||
{
|
||||
ManifestDisposition disposition;
|
||||
bool acceptedUpdate;
|
||||
};
|
||||
|
||||
inline std::string
|
||||
@@ -244,18 +292,72 @@ to_string(ManifestDisposition m)
|
||||
return "badEphemeralKey";
|
||||
case ManifestDisposition::invalid:
|
||||
return "invalid";
|
||||
case ManifestDisposition::untrustedCapacity:
|
||||
return "untrustedCapacity";
|
||||
default:
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
/** Retention policy for an accepted manifest.
|
||||
|
||||
This is deliberately independent of validator-list membership and
|
||||
consensus trust. Protected entries are not part of the bounded eviction
|
||||
population; evictable entries are.
|
||||
*/
|
||||
enum class ManifestRetention : std::uint8_t { evictable, protected_ };
|
||||
|
||||
class DatabaseCon;
|
||||
|
||||
/** Remembers manifests with the highest sequence number. */
|
||||
/** Remembers manifests with the highest sequence number.
|
||||
|
||||
This is protocol state, not merely a payload cache. While an entry remains
|
||||
resident, it supplies the sequence high-water mark, revocation state, and
|
||||
master/signing-key collision checks for that validator. Evicting an
|
||||
untrusted entry necessarily forgets those facts and can make an old
|
||||
manifest cache-new again.
|
||||
|
||||
Entries admitted with protected retention, or later reclassified as
|
||||
protected, are outside the eviction population. For other validators,
|
||||
recent validation activity is an eviction preference, not a retention
|
||||
guarantee or source of consensus trust. Tier-2 publisher provenance is the
|
||||
explicit way to protect a monitored candidate. This preference makes an
|
||||
actively validating untrusted identity harder to displace than a quiet one;
|
||||
if every evictable identity appears active, selection is uniformly random.
|
||||
|
||||
This is not complete adversarial containment. Once full, the cache gives
|
||||
valid novel identities a small eviction budget, bounding admitted identity
|
||||
churn. Capacity rejection normally avoids verification when no permit is
|
||||
available, although concurrent callers may race on observed availability.
|
||||
Updates to an already retained identity do not consume the budget.
|
||||
First-seen unlisted identities are not relayed live. A bounded selected
|
||||
subset propagates through the cached connection snapshot.
|
||||
Later rotations and revocations relay while the identity remains resident.
|
||||
If eviction forgets one, its reappearance is first-seen again and therefore
|
||||
does not immediately relay. A sustained sender can monopolize the eviction
|
||||
budget and delay a legitimate novel untrusted validator; protected
|
||||
validators are unaffected.
|
||||
*/
|
||||
class ManifestCache
|
||||
{
|
||||
private:
|
||||
using TimePoint = std::chrono::steady_clock::time_point;
|
||||
using Now = std::function<TimePoint()>;
|
||||
|
||||
beast::Journal j_;
|
||||
Now now_;
|
||||
|
||||
/** Serialize post-verification admission of evictable manifests.
|
||||
|
||||
Signature verification remains concurrent. Once verified, evictable
|
||||
admissions pass through this mutex so only the caller that can consume
|
||||
an eviction permit samples activity and selects a victim. When both
|
||||
mutexes are needed, this mutex is acquired before `mutex_`; caller code
|
||||
is never invoked while `mutex_` is held. Manifest jobs use the shared
|
||||
JobQueue worker pool, so work in this section must remain small.
|
||||
*/
|
||||
std::mutex evictionAdmissionMutex_;
|
||||
|
||||
std::shared_mutex mutable mutex_;
|
||||
|
||||
/** Active manifests stored by master public key. */
|
||||
@@ -266,10 +368,40 @@ private:
|
||||
|
||||
std::atomic<std::uint32_t> seq_{0};
|
||||
|
||||
/** Master keys currently counted against the bounded eviction cap. */
|
||||
hash_set<PublicKey> evictableKeys_;
|
||||
|
||||
/** Master keys explicitly protected by local validator configuration. */
|
||||
hash_set<PublicKey> configuredKeys_;
|
||||
|
||||
/** Maximum number of untrusted master keys retained in memory. */
|
||||
static constexpr std::size_t kMaxUntrustedCount = 1000;
|
||||
|
||||
/** Burst and refill rate for evictions after the untrusted cache fills. */
|
||||
static constexpr std::size_t kMaxEvictionPermits = 10;
|
||||
static constexpr std::chrono::seconds kEvictionPermitInterval{1};
|
||||
std::size_t evictionPermits_ = kMaxEvictionPermits;
|
||||
TimePoint evictionBudgetUpdated_;
|
||||
|
||||
/** Number of manifests rejected because the untrusted cache was full. */
|
||||
std::atomic<std::uint64_t> untrustedRejectCount_{0};
|
||||
|
||||
/** Number of capacity rejections between warning summaries. */
|
||||
static constexpr std::uint64_t kUntrustedRejectCount = 10000;
|
||||
|
||||
ManifestDisposition
|
||||
applyManifestImpl(
|
||||
Manifest m,
|
||||
ManifestRetention retention,
|
||||
bool mayEvict,
|
||||
std::function<hash_set<PublicKey>()> const& currentValidationKeys,
|
||||
bool* acceptedUpdate);
|
||||
|
||||
public:
|
||||
explicit ManifestCache(
|
||||
beast::Journal j = beast::Journal(beast::Journal::getNullSink()))
|
||||
: j_(j)
|
||||
beast::Journal j = beast::Journal(beast::Journal::getNullSink()),
|
||||
Now now = [] { return std::chrono::steady_clock::now(); })
|
||||
: j_(j), now_(std::move(now)), evictionBudgetUpdated_(now_())
|
||||
{
|
||||
}
|
||||
|
||||
@@ -345,6 +477,8 @@ public:
|
||||
|
||||
@param m Manifest to add
|
||||
|
||||
@param retention Whether the entry is protected or evictable
|
||||
|
||||
@return `ManifestDisposition::accepted` if successful, or
|
||||
`stale` or `invalid` otherwise
|
||||
|
||||
@@ -353,7 +487,54 @@ public:
|
||||
May be called concurrently
|
||||
*/
|
||||
ManifestDisposition
|
||||
applyManifest(Manifest m);
|
||||
applyManifest(Manifest m, ManifestRetention retention);
|
||||
|
||||
/** Add an untrusted manifest, evicting another at capacity.
|
||||
|
||||
A dormant evictable entry is chosen at random when possible. If every
|
||||
retained evictable signing key has a current validation, any evictable
|
||||
entry may be chosen. The candidate is fully verified before current
|
||||
validations are sampled or an entry is evicted.
|
||||
|
||||
Activity is matched to the manifest's current signing key. Immediately
|
||||
after rotation, an evictable identity appears dormant until a
|
||||
validation from its new key arrives.
|
||||
|
||||
@param m Manifest to add
|
||||
@param currentValidationKeys Supplies signing keys with current
|
||||
validations. It is called only when a verified novel manifest
|
||||
needs and is permitted to evict an entry after serialized
|
||||
admission rechecks. If omitted, victim selection is uniformly
|
||||
random. The callback must not re-enter this method.
|
||||
|
||||
@return disposition and an atomic indication that an accepted manifest
|
||||
updated an identity retained at admission time
|
||||
*/
|
||||
ManifestApplyResult
|
||||
applyManifestWithEviction(
|
||||
Manifest m,
|
||||
std::function<hash_set<PublicKey>()> const& currentValidationKeys = {});
|
||||
|
||||
/** Change the retention class of an already cached master.
|
||||
|
||||
Consensus trust is not changed. If an entry becomes evictable while
|
||||
the bounded population is full, it is discarded and can later be
|
||||
reacquired from its signed source.
|
||||
*/
|
||||
void
|
||||
setRetention(PublicKey const& pk, ManifestRetention retention);
|
||||
|
||||
/** Reconcile cached retention against the current protected-master set.
|
||||
|
||||
This changes retention only; it never grants validator-list membership
|
||||
or consensus weight. Cached entries outside `protectedMasters` and
|
||||
local validator configuration become evictable, subject to the
|
||||
ordinary bounded population. Terminal revocations are treated like
|
||||
other current high-water state: once every protected source
|
||||
disappears, they become evictable.
|
||||
*/
|
||||
void
|
||||
reconcileRetention(hash_set<PublicKey> const& protectedMasters);
|
||||
|
||||
/** Populate manifest cache with manifests in database and config.
|
||||
|
||||
@@ -442,7 +623,7 @@ public:
|
||||
@param pf Pre-function called with the maximum number of times f will be
|
||||
called (useful for memory allocations)
|
||||
|
||||
@param f Function called for each manifest
|
||||
@param f Function called for each manifest and its retention class
|
||||
|
||||
@par Thread Safety
|
||||
|
||||
@@ -454,10 +635,11 @@ public:
|
||||
{
|
||||
std::shared_lock lock{mutex_};
|
||||
pf(map_.size());
|
||||
for (auto const& [_, manifest] : map_)
|
||||
for (auto const& [master, manifest] : map_)
|
||||
{
|
||||
(void)_;
|
||||
f(manifest);
|
||||
f(manifest,
|
||||
evictableKeys_.contains(master) ? ManifestRetention::evictable
|
||||
: ManifestRetention::protected_);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -31,9 +31,12 @@
|
||||
#include <boost/iterator/counting_iterator.hpp>
|
||||
#include <boost/range/adaptors.hpp>
|
||||
#include <boost/thread/shared_mutex.hpp>
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <numeric>
|
||||
#include <shared_mutex>
|
||||
#include <utility>
|
||||
|
||||
namespace protocol {
|
||||
class TMValidatorList;
|
||||
@@ -124,6 +127,23 @@ struct ValidatorBlobInfo
|
||||
std::optional<std::string> manifest;
|
||||
};
|
||||
|
||||
/** Manifest transport/admission policy for a master key.
|
||||
|
||||
Publisher candidates receive protected manifest retention and relay, but
|
||||
are not listed or trusted for consensus.
|
||||
*/
|
||||
struct ValidatorManifestPolicy
|
||||
{
|
||||
bool consensusListed = false;
|
||||
bool publisherCandidate = false;
|
||||
|
||||
bool
|
||||
relayEligible() const
|
||||
{
|
||||
return consensusListed || publisherCandidate;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
Trusted Validators List
|
||||
-----------------------
|
||||
@@ -171,12 +191,26 @@ struct ValidatorBlobInfo
|
||||
*/
|
||||
class ValidatorList
|
||||
{
|
||||
struct PublisherValidator
|
||||
{
|
||||
PublicKey master;
|
||||
std::shared_ptr<Manifest const> manifest;
|
||||
};
|
||||
|
||||
struct PublisherCandidate
|
||||
{
|
||||
PublicKey master;
|
||||
std::shared_ptr<Manifest const> manifest;
|
||||
};
|
||||
|
||||
struct PublisherList
|
||||
{
|
||||
explicit PublisherList() = default;
|
||||
|
||||
std::vector<PublicKey> list;
|
||||
std::vector<std::string> manifests;
|
||||
std::vector<PublisherValidator> validators;
|
||||
// Validators published for monitoring only. They never contribute to
|
||||
// keyListings_, the trusted UNL, or quorum.
|
||||
std::vector<PublisherCandidate> candidates;
|
||||
std::size_t sequence;
|
||||
TimeKeeper::time_point validFrom;
|
||||
TimeKeeper::time_point validUntil;
|
||||
@@ -236,6 +270,11 @@ class ValidatorList
|
||||
// Published lists stored by publisher master public key
|
||||
hash_map<PublicKey, PublisherListCollection> publisherLists_;
|
||||
|
||||
// Current publisher-selected candidate masters. Candidate manifests use
|
||||
// the ordinary ManifestCache: tier membership changes retention and relay
|
||||
// policy, not identity-resolution semantics.
|
||||
hash_set<PublicKey> publisherCandidateMasters_;
|
||||
|
||||
// Listed master public keys with the number of lists they appear on
|
||||
hash_map<PublicKey, std::size_t> keyListings_;
|
||||
|
||||
@@ -274,6 +313,9 @@ class ValidatorList
|
||||
static const std::string filePrefix_;
|
||||
|
||||
public:
|
||||
/** Per-generation and aggregate bound for monitoring candidates. */
|
||||
static constexpr std::size_t maxPublisherCandidates = 1000;
|
||||
|
||||
ValidatorList(
|
||||
ManifestCache& validatorManifests,
|
||||
ManifestCache& publisherManifests,
|
||||
@@ -556,6 +598,15 @@ public:
|
||||
std::optional<PublicKey>
|
||||
getListedKey(PublicKey const& identity) const;
|
||||
|
||||
/** Return manifest admission/relay policy for an asserted master key.
|
||||
|
||||
This is deliberately distinct from consensus listing and trust.
|
||||
Current publisher candidates may relay valid self-signed manifest
|
||||
updates, but never gain validation weight through this API.
|
||||
*/
|
||||
ValidatorManifestPolicy
|
||||
manifestPolicy(PublicKey const& master) const;
|
||||
|
||||
/** Returns `true` if public key is a trusted publisher
|
||||
|
||||
@param identity Publisher public key
|
||||
@@ -793,10 +844,22 @@ private:
|
||||
void
|
||||
updatePublisherList(
|
||||
PublicKey const& pubKey,
|
||||
PublisherList const& current,
|
||||
std::vector<PublicKey> const& current,
|
||||
std::vector<PublicKey> const& oldList,
|
||||
lock_guard const&);
|
||||
|
||||
void
|
||||
ingestPublisherManifests(
|
||||
PublicKey const& pubKey,
|
||||
PublisherList const& current,
|
||||
lock_guard const&);
|
||||
|
||||
void
|
||||
rebuildPublisherCandidates(lock_guard const&);
|
||||
|
||||
static std::vector<PublicKey>
|
||||
validatorMasters(PublisherList const& list);
|
||||
|
||||
static void
|
||||
buildBlobInfos(
|
||||
std::map<std::size_t, ValidatorBlobInfo>& blobInfos,
|
||||
|
||||
@@ -23,14 +23,17 @@
|
||||
#include <xrpl/basics/Log.h>
|
||||
#include <xrpl/basics/StringUtilities.h>
|
||||
#include <xrpl/basics/base64.h>
|
||||
#include <xrpl/basics/random.h>
|
||||
#include <xrpl/json/json_reader.h>
|
||||
#include <xrpl/protocol/PublicKey.h>
|
||||
#include <xrpl/protocol/Sign.h>
|
||||
|
||||
#include <boost/algorithm/string/trim.hpp>
|
||||
|
||||
#include <iterator>
|
||||
#include <numeric>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
|
||||
namespace ripple {
|
||||
|
||||
@@ -55,6 +58,9 @@ deserializeManifest(Slice s, beast::Journal journal)
|
||||
if (s.empty())
|
||||
return std::nullopt;
|
||||
|
||||
if (s.size() > kMaxManifestBytes)
|
||||
return std::nullopt;
|
||||
|
||||
static SOTemplate const manifestFormat{
|
||||
// A manifest must include:
|
||||
// - the master public key
|
||||
@@ -367,8 +373,40 @@ ManifestCache::revoked(PublicKey const& pk) const
|
||||
}
|
||||
|
||||
ManifestDisposition
|
||||
ManifestCache::applyManifest(Manifest m)
|
||||
ManifestCache::applyManifest(Manifest m, ManifestRetention const retention)
|
||||
{
|
||||
return applyManifestImpl(std::move(m), retention, false, {}, nullptr);
|
||||
}
|
||||
|
||||
ManifestApplyResult
|
||||
ManifestCache::applyManifestWithEviction(
|
||||
Manifest m,
|
||||
std::function<hash_set<PublicKey>()> const& currentValidationKeys)
|
||||
{
|
||||
bool acceptedUpdate = false;
|
||||
auto const disposition = applyManifestImpl(
|
||||
std::move(m),
|
||||
ManifestRetention::evictable,
|
||||
true,
|
||||
currentValidationKeys,
|
||||
&acceptedUpdate);
|
||||
return {disposition, acceptedUpdate};
|
||||
}
|
||||
|
||||
ManifestDisposition
|
||||
ManifestCache::applyManifestImpl(
|
||||
Manifest m,
|
||||
ManifestRetention const retention,
|
||||
bool const mayEvict,
|
||||
std::function<hash_set<PublicKey>()> const& currentValidationKeys,
|
||||
bool* const acceptedUpdate)
|
||||
{
|
||||
if (acceptedUpdate)
|
||||
*acceptedUpdate = false;
|
||||
|
||||
bool const protectedRetention = retention == ManifestRetention::protected_;
|
||||
bool checkSignature = true;
|
||||
|
||||
// Check the manifest against the conditions that do not require a
|
||||
// `unique_lock` (write lock) on the `mutex_`. Since the signature can be
|
||||
// relatively expensive, the `checkSignature` parameter determines if the
|
||||
@@ -376,8 +414,9 @@ ManifestCache::applyManifest(Manifest m)
|
||||
// comment below), `checkSignature` only needs to be set to true on the
|
||||
// first run.
|
||||
auto prewriteCheck =
|
||||
[this, &m](auto const& iter, bool checkSignature, auto const& lock)
|
||||
-> std::optional<ManifestDisposition> {
|
||||
[this, &m, &checkSignature](
|
||||
auto const& iter,
|
||||
auto const& lock) -> std::optional<ManifestDisposition> {
|
||||
XRPL_ASSERT(
|
||||
lock.owns_lock(),
|
||||
"ripple::ManifestCache::applyManifest::prewriteCheck : locked");
|
||||
@@ -399,11 +438,16 @@ ManifestCache::applyManifest(Manifest m)
|
||||
return ManifestDisposition::stale;
|
||||
}
|
||||
|
||||
if (checkSignature && !m.verify())
|
||||
if (checkSignature)
|
||||
{
|
||||
if (auto stream = j_.warn())
|
||||
LOG_MANIFEST_ACTION(stream, "Invalid", m.masterKey, m.sequence);
|
||||
return ManifestDisposition::invalid;
|
||||
checkSignature = false;
|
||||
if (!m.verify())
|
||||
{
|
||||
if (auto stream = j_.warn())
|
||||
LOG_MANIFEST_ACTION(
|
||||
stream, "Invalid", m.masterKey, m.sequence);
|
||||
return ManifestDisposition::invalid;
|
||||
}
|
||||
}
|
||||
|
||||
// If the master key associated with a manifest is or might be
|
||||
@@ -467,15 +511,100 @@ ManifestCache::applyManifest(Manifest m)
|
||||
return std::nullopt;
|
||||
};
|
||||
|
||||
auto atEvictableCap = [this, protectedRetention](
|
||||
auto const& iter, auto const& lock) {
|
||||
XRPL_ASSERT(
|
||||
lock.owns_lock(),
|
||||
"ripple::ManifestCache::applyManifest::atEvictableCap : locked");
|
||||
(void)lock;
|
||||
return iter == map_.end() && !protectedRetention &&
|
||||
evictableKeys_.size() >= kMaxUntrustedCount;
|
||||
};
|
||||
|
||||
auto rejectAtUntrustedCap = [this, &m]() {
|
||||
if (auto stream = j_.debug())
|
||||
LOG_MANIFEST_ACTION(
|
||||
stream, "UntrustedCapacity", m.masterKey, m.sequence);
|
||||
if (auto const n = untrustedRejectCount_.fetch_add(1) + 1;
|
||||
n % kUntrustedRejectCount == 0)
|
||||
{
|
||||
JLOG(j_.warn()) << "Untrusted manifest cap reached; " << n
|
||||
<< " manifests rejected so far";
|
||||
}
|
||||
return ManifestDisposition::untrustedCapacity;
|
||||
};
|
||||
|
||||
auto evictionPermitAvailable = [this](auto const& lock) {
|
||||
XRPL_ASSERT(
|
||||
lock.owns_lock(),
|
||||
"ripple::ManifestCache::applyManifestImpl : eviction budget "
|
||||
"locked");
|
||||
(void)lock;
|
||||
return evictionPermits_ > 0 ||
|
||||
now_() - evictionBudgetUpdated_ >= kEvictionPermitInterval;
|
||||
};
|
||||
|
||||
{
|
||||
std::shared_lock sl{mutex_};
|
||||
if (auto d =
|
||||
prewriteCheck(map_.find(m.masterKey), /*checkSig*/ true, sl))
|
||||
return *d;
|
||||
auto const iter = map_.find(m.masterKey);
|
||||
if (atEvictableCap(iter, sl))
|
||||
{
|
||||
if (!mayEvict || !evictionPermitAvailable(sl))
|
||||
return rejectAtUntrustedCap();
|
||||
}
|
||||
if (auto d = prewriteCheck(iter, sl); d.has_value())
|
||||
{
|
||||
// A protected application also carries retention policy. Defer a
|
||||
// stale result to the write lock so an existing entry can be
|
||||
// promoted atomically instead of racing eviction.
|
||||
if (!protectedRetention || *d != ManifestDisposition::stale ||
|
||||
!evictableKeys_.contains(m.masterKey))
|
||||
return *d;
|
||||
}
|
||||
}
|
||||
|
||||
// Signature verification above remains concurrent. Serialize the short
|
||||
// post-verification admission path so racing callers cannot all sample
|
||||
// activity after observing the same eviction permit. This lock is always
|
||||
// acquired before mutex_, and caller code runs with mutex_ released.
|
||||
std::unique_lock<std::mutex> evictionAdmissionLock{
|
||||
evictionAdmissionMutex_, std::defer_lock};
|
||||
if (mayEvict)
|
||||
evictionAdmissionLock.lock();
|
||||
|
||||
bool needsCurrentValidationKeys = false;
|
||||
if (mayEvict)
|
||||
{
|
||||
std::shared_lock sl{mutex_};
|
||||
auto const iter = map_.find(m.masterKey);
|
||||
if (atEvictableCap(iter, sl))
|
||||
{
|
||||
if (!evictionPermitAvailable(sl))
|
||||
return rejectAtUntrustedCap();
|
||||
needsCurrentValidationKeys =
|
||||
static_cast<bool>(currentValidationKeys);
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<hash_set<PublicKey>> activeSigningKeys;
|
||||
if (needsCurrentValidationKeys)
|
||||
activeSigningKeys = currentValidationKeys();
|
||||
|
||||
std::unique_lock sl{mutex_};
|
||||
auto const iter = map_.find(m.masterKey);
|
||||
|
||||
bool const needsEviction = atEvictableCap(iter, sl);
|
||||
if (needsEviction && !mayEvict)
|
||||
return rejectAtUntrustedCap();
|
||||
|
||||
if (protectedRetention && iter != map_.end() &&
|
||||
m.sequence <= iter->second.sequence)
|
||||
{
|
||||
if (evictableKeys_.erase(m.masterKey) != 0)
|
||||
++seq_;
|
||||
return ManifestDisposition::stale;
|
||||
}
|
||||
|
||||
// Since we released the previously held read lock, it's possible that the
|
||||
// collections have been written to. This means we need to run
|
||||
// `prewriteCheck` again. This re-does work, but `prewriteCheck` is
|
||||
@@ -485,12 +614,79 @@ ManifestCache::applyManifest(Manifest m)
|
||||
// doesn't need to happen again (signature checks are somewhat expensive).
|
||||
// Note: It's a mistake to use an upgradable lock. This is a recipe for
|
||||
// deadlock.
|
||||
if (auto d = prewriteCheck(iter, /*checkSig*/ false, sl))
|
||||
if (auto d = prewriteCheck(iter, sl); d.has_value())
|
||||
return *d;
|
||||
|
||||
if (needsEviction)
|
||||
{
|
||||
XRPL_ASSERT(
|
||||
mayEvict && !evictableKeys_.empty(),
|
||||
"ripple::ManifestCache::applyManifestImpl : eviction inputs");
|
||||
|
||||
auto const now = now_();
|
||||
auto const elapsed = now - evictionBudgetUpdated_;
|
||||
auto const refillIntervals = elapsed / kEvictionPermitInterval;
|
||||
if (refillIntervals > 0)
|
||||
{
|
||||
auto const refill = refillIntervals >= kMaxEvictionPermits
|
||||
? kMaxEvictionPermits
|
||||
: static_cast<std::size_t>(refillIntervals);
|
||||
evictionPermits_ =
|
||||
std::min(kMaxEvictionPermits, evictionPermits_ + refill);
|
||||
evictionBudgetUpdated_ += refillIntervals * kEvictionPermitInterval;
|
||||
}
|
||||
if (evictionPermits_ == 0)
|
||||
return rejectAtUntrustedCap();
|
||||
--evictionPermits_;
|
||||
|
||||
std::optional<PublicKey> dormantVictim;
|
||||
std::size_t dormantSeen = 0;
|
||||
if (activeSigningKeys)
|
||||
{
|
||||
for (auto const& master : evictableKeys_)
|
||||
{
|
||||
auto const candidate = map_.find(master);
|
||||
XRPL_ASSERT(
|
||||
candidate != map_.end(),
|
||||
"ripple::ManifestCache::applyManifestImpl : evictable "
|
||||
"key retained");
|
||||
if (candidate == map_.end())
|
||||
continue;
|
||||
if (!candidate->second.signingKey ||
|
||||
!activeSigningKeys->contains(*candidate->second.signingKey))
|
||||
{
|
||||
++dormantSeen;
|
||||
if (dormantSeen == 1 || rand_int(dormantSeen - 1) == 0)
|
||||
dormantVictim = master;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PublicKey const victimMaster = [&]() {
|
||||
if (dormantVictim)
|
||||
return *dormantVictim;
|
||||
auto victim = evictableKeys_.begin();
|
||||
if (evictableKeys_.size() > 1)
|
||||
std::advance(victim, rand_int(evictableKeys_.size() - 1));
|
||||
return *victim;
|
||||
}();
|
||||
|
||||
auto const victim = map_.find(victimMaster);
|
||||
XRPL_ASSERT(
|
||||
victim != map_.end(),
|
||||
"ripple::ManifestCache::applyManifestImpl : victim retained");
|
||||
if (victim == map_.end())
|
||||
return rejectAtUntrustedCap();
|
||||
|
||||
if (victim->second.signingKey)
|
||||
signingToMasterKeys_.erase(*victim->second.signingKey);
|
||||
map_.erase(victim);
|
||||
evictableKeys_.erase(victimMaster);
|
||||
}
|
||||
|
||||
bool const revoked = m.revoked();
|
||||
// This is the first manifest we are seeing for a master key. This should
|
||||
// only ever happen once per validator run.
|
||||
// This master key is not currently retained. An untrusted key may reach
|
||||
// this path again after eviction.
|
||||
if (iter == map_.end())
|
||||
{
|
||||
if (auto stream = j_.info())
|
||||
@@ -500,6 +696,8 @@ ManifestCache::applyManifest(Manifest m)
|
||||
signingToMasterKeys_.emplace(*m.signingKey, m.masterKey);
|
||||
|
||||
auto masterKey = m.masterKey;
|
||||
if (!protectedRetention)
|
||||
evictableKeys_.insert(masterKey);
|
||||
map_.emplace(std::move(masterKey), std::move(m));
|
||||
|
||||
// Increment sequence to invalidate cached manifest messages
|
||||
@@ -518,6 +716,9 @@ ManifestCache::applyManifest(Manifest m)
|
||||
m.sequence,
|
||||
iter->second.sequence);
|
||||
|
||||
if (protectedRetention)
|
||||
evictableKeys_.erase(m.masterKey);
|
||||
|
||||
signingToMasterKeys_.erase(*iter->second.signingKey);
|
||||
|
||||
if (!revoked)
|
||||
@@ -528,9 +729,96 @@ ManifestCache::applyManifest(Manifest m)
|
||||
// Something has changed. Keep track of it.
|
||||
seq_++;
|
||||
|
||||
if (acceptedUpdate)
|
||||
*acceptedUpdate = true;
|
||||
|
||||
return ManifestDisposition::accepted;
|
||||
}
|
||||
|
||||
void
|
||||
ManifestCache::setRetention(
|
||||
PublicKey const& pk,
|
||||
ManifestRetention const retention)
|
||||
{
|
||||
std::unique_lock sl{mutex_};
|
||||
auto const iter = map_.find(pk);
|
||||
if (iter == map_.end())
|
||||
return;
|
||||
|
||||
if (retention == ManifestRetention::protected_)
|
||||
{
|
||||
if (evictableKeys_.erase(pk) != 0)
|
||||
{
|
||||
// Retention affects peer-snapshot priority even though the
|
||||
// retained manifest is unchanged.
|
||||
++seq_;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (evictableKeys_.contains(pk))
|
||||
return;
|
||||
|
||||
if (configuredKeys_.contains(pk))
|
||||
return;
|
||||
|
||||
if (evictableKeys_.size() < kMaxUntrustedCount)
|
||||
{
|
||||
evictableKeys_.insert(pk);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (iter->second.signingKey)
|
||||
signingToMasterKeys_.erase(*iter->second.signingKey);
|
||||
map_.erase(iter);
|
||||
}
|
||||
|
||||
// Retention changes peer-snapshot priority; removal also changes manifest
|
||||
// resolution.
|
||||
++seq_;
|
||||
}
|
||||
|
||||
void
|
||||
ManifestCache::reconcileRetention(hash_set<PublicKey> const& protectedMasters)
|
||||
{
|
||||
std::unique_lock sl{mutex_};
|
||||
bool changed = false;
|
||||
|
||||
for (auto iter = map_.begin(); iter != map_.end();)
|
||||
{
|
||||
auto const master = iter->first;
|
||||
if (configuredKeys_.contains(master) ||
|
||||
protectedMasters.contains(master))
|
||||
{
|
||||
changed = evictableKeys_.erase(master) != 0 || changed;
|
||||
++iter;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (evictableKeys_.contains(master))
|
||||
{
|
||||
++iter;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (evictableKeys_.size() < kMaxUntrustedCount)
|
||||
{
|
||||
evictableKeys_.insert(master);
|
||||
changed = true;
|
||||
++iter;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (iter->second.signingKey)
|
||||
signingToMasterKeys_.erase(*iter->second.signingKey);
|
||||
iter = map_.erase(iter);
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if (changed)
|
||||
++seq_;
|
||||
}
|
||||
|
||||
void
|
||||
ManifestCache::load(DatabaseCon& dbCon, std::string const& dbTable)
|
||||
{
|
||||
@@ -561,11 +849,16 @@ ManifestCache::load(
|
||||
JLOG(j_.warn()) << "Configured manifest revokes public key";
|
||||
}
|
||||
|
||||
if (applyManifest(std::move(*mo)) == ManifestDisposition::invalid)
|
||||
auto const masterKey = mo->masterKey;
|
||||
if (applyManifest(std::move(*mo), ManifestRetention::protected_) ==
|
||||
ManifestDisposition::invalid)
|
||||
{
|
||||
JLOG(j_.error()) << "Manifest in config was rejected";
|
||||
return false;
|
||||
}
|
||||
|
||||
std::unique_lock lock{mutex_};
|
||||
configuredKeys_.insert(masterKey);
|
||||
}
|
||||
|
||||
if (!configRevocation.empty())
|
||||
@@ -584,12 +877,22 @@ ManifestCache::load(
|
||||
|
||||
auto mo = deserializeManifest(base64_decode(revocationStr));
|
||||
|
||||
if (!mo || !mo->revoked() ||
|
||||
applyManifest(std::move(*mo)) == ManifestDisposition::invalid)
|
||||
if (!mo || !mo->revoked())
|
||||
{
|
||||
JLOG(j_.error()) << "Invalid validator key revocation in config";
|
||||
return false;
|
||||
}
|
||||
|
||||
auto const masterKey = mo->masterKey;
|
||||
if (applyManifest(std::move(*mo), ManifestRetention::protected_) ==
|
||||
ManifestDisposition::invalid)
|
||||
{
|
||||
JLOG(j_.error()) << "Invalid validator key revocation in config";
|
||||
return false;
|
||||
}
|
||||
|
||||
std::unique_lock lock{mutex_};
|
||||
configuredKeys_.insert(masterKey);
|
||||
}
|
||||
|
||||
return true;
|
||||
@@ -601,10 +904,25 @@ ManifestCache::save(
|
||||
std::string const& dbTable,
|
||||
std::function<bool(PublicKey const&)> const& isTrusted)
|
||||
{
|
||||
std::shared_lock lock{mutex_};
|
||||
hash_map<PublicKey, Manifest> manifests;
|
||||
{
|
||||
std::shared_lock lock{mutex_};
|
||||
manifests.reserve(map_.size());
|
||||
for (auto const& [master, m] : map_)
|
||||
{
|
||||
manifests.emplace(
|
||||
master,
|
||||
Manifest{
|
||||
m.serialized,
|
||||
m.masterKey,
|
||||
m.signingKey,
|
||||
m.sequence,
|
||||
m.domain});
|
||||
}
|
||||
}
|
||||
auto db = dbCon.checkoutDb();
|
||||
|
||||
saveManifests(*db, dbTable, isTrusted, map_, j_);
|
||||
saveManifests(*db, dbTable, isTrusted, manifests, j_);
|
||||
}
|
||||
|
||||
// Clean up macros to avoid namespace pollution
|
||||
|
||||
@@ -226,6 +226,8 @@ ValidatorList::load(
|
||||
keyListings_.insert({*localPubKey_, listThreshold_});
|
||||
if (inserted)
|
||||
{
|
||||
validatorManifests_.setRetention(
|
||||
*localPubKey_, ManifestRetention::protected_);
|
||||
JLOG(j_.debug()) << "Added own master key "
|
||||
<< toBase58(TokenType::NodePublic, *localPubKey_);
|
||||
}
|
||||
@@ -265,7 +267,8 @@ ValidatorList::load(
|
||||
JLOG(j_.warn()) << "Duplicate node identity: " << match[1];
|
||||
continue;
|
||||
}
|
||||
localPublisherList.list.emplace_back(*id);
|
||||
validatorManifests_.setRetention(*id, ManifestRetention::protected_);
|
||||
localPublisherList.validators.push_back(PublisherValidator{*id, {}});
|
||||
++count;
|
||||
}
|
||||
|
||||
@@ -277,6 +280,12 @@ ValidatorList::load(
|
||||
|
||||
JLOG(j_.debug()) << "Loaded " << count << " entries";
|
||||
|
||||
// Wallet rows are loaded before local and publisher configuration is
|
||||
// known. Reconcile that provisional startup retention now that the current
|
||||
// local source set is complete. Available publisher lists, if any, are
|
||||
// folded in by the same path.
|
||||
rebuildPublisherCandidates(lock);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -968,6 +977,10 @@ ValidatorList::applyListsAndBroadcast(
|
||||
networkOPs.clearUNLBlocked();
|
||||
}
|
||||
}
|
||||
else if (disposition == ListDisposition::expired)
|
||||
{
|
||||
networkOPs.setUNLBlocked();
|
||||
}
|
||||
bool broadcast = disposition <= ListDisposition::known_sequence;
|
||||
|
||||
// this function is only called for PublicKeys which are not specified
|
||||
@@ -1068,16 +1081,24 @@ ValidatorList::applyLists(
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<PublicKey>
|
||||
ValidatorList::validatorMasters(PublisherList const& list)
|
||||
{
|
||||
std::vector<PublicKey> result;
|
||||
result.reserve(list.validators.size());
|
||||
for (auto const& entry : list.validators)
|
||||
result.push_back(entry.master);
|
||||
return result;
|
||||
}
|
||||
|
||||
void
|
||||
ValidatorList::updatePublisherList(
|
||||
PublicKey const& pubKey,
|
||||
PublisherList const& current,
|
||||
std::vector<PublicKey> const& publisherList,
|
||||
std::vector<PublicKey> const& oldList,
|
||||
ValidatorList::lock_guard const&)
|
||||
{
|
||||
// Update keyListings_ for added and removed keys
|
||||
std::vector<PublicKey> const& publisherList = current.list;
|
||||
std::vector<std::string> const& manifests = current.manifests;
|
||||
auto iNew = publisherList.begin();
|
||||
auto iOld = oldList.begin();
|
||||
while (iNew != publisherList.end() || iOld != oldList.end())
|
||||
@@ -1087,6 +1108,8 @@ ValidatorList::updatePublisherList(
|
||||
{
|
||||
// Increment list count for added keys
|
||||
++keyListings_[*iNew];
|
||||
validatorManifests_.setRetention(
|
||||
*iNew, ManifestRetention::protected_);
|
||||
++iNew;
|
||||
}
|
||||
else if (
|
||||
@@ -1111,19 +1134,34 @@ ValidatorList::updatePublisherList(
|
||||
{
|
||||
JLOG(j_.warn()) << "No validator keys included in valid list";
|
||||
}
|
||||
}
|
||||
|
||||
for (auto const& valManifest : manifests)
|
||||
void
|
||||
ValidatorList::ingestPublisherManifests(
|
||||
PublicKey const& pubKey,
|
||||
PublisherList const& current,
|
||||
ValidatorList::lock_guard const&)
|
||||
{
|
||||
for (auto const& entry : current.validators)
|
||||
{
|
||||
auto m = deserializeManifest(base64_decode(valManifest));
|
||||
|
||||
if (!m || !keyListings_.count(m->masterKey))
|
||||
if (!entry.manifest)
|
||||
continue;
|
||||
if (entry.manifest->masterKey != entry.master)
|
||||
{
|
||||
JLOG(j_.warn()) << "List for " << strHex(pubKey)
|
||||
<< " contained untrusted validator manifest";
|
||||
<< " contained mismatched validator manifest";
|
||||
continue;
|
||||
}
|
||||
|
||||
if (auto const r = validatorManifests_.applyManifest(std::move(*m));
|
||||
auto const& supplied = *entry.manifest;
|
||||
Manifest m{
|
||||
supplied.serialized,
|
||||
supplied.masterKey,
|
||||
supplied.signingKey,
|
||||
supplied.sequence,
|
||||
supplied.domain};
|
||||
if (auto const r = validatorManifests_.applyManifest(
|
||||
std::move(m), ManifestRetention::protected_);
|
||||
r == ManifestDisposition::invalid)
|
||||
{
|
||||
JLOG(j_.warn()) << "List for " << strHex(pubKey)
|
||||
@@ -1132,6 +1170,98 @@ ValidatorList::updatePublisherList(
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
ValidatorList::rebuildPublisherCandidates(ValidatorList::lock_guard const&)
|
||||
{
|
||||
hash_set<PublicKey> candidateMasters;
|
||||
hash_set<PublicKey> contributingPublishers;
|
||||
std::size_t supplied = 0;
|
||||
|
||||
for (auto const& [pubKey, collection] : publisherLists_)
|
||||
{
|
||||
if (collection.status != PublisherStatus::available)
|
||||
continue;
|
||||
|
||||
for (auto const& candidate : collection.current.candidates)
|
||||
{
|
||||
++supplied;
|
||||
contributingPublishers.insert(pubKey);
|
||||
candidateMasters.insert(candidate.master);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<PublicKey> retained;
|
||||
retained.reserve(candidateMasters.size());
|
||||
for (auto const& master : candidateMasters)
|
||||
retained.push_back(master);
|
||||
std::sort(retained.begin(), retained.end());
|
||||
|
||||
auto const distinct = retained.size();
|
||||
if (retained.size() > maxPublisherCandidates)
|
||||
{
|
||||
for (auto it = retained.begin() + maxPublisherCandidates;
|
||||
it != retained.end();
|
||||
++it)
|
||||
candidateMasters.erase(*it);
|
||||
retained.erase(
|
||||
retained.begin() + maxPublisherCandidates, retained.end());
|
||||
|
||||
JLOG(j_.warn())
|
||||
<< "Publisher candidate tier exceeded global capacity: supplied="
|
||||
<< supplied << ", distinct=" << distinct
|
||||
<< ", retained=" << retained.size()
|
||||
<< ", dropped=" << (distinct - retained.size())
|
||||
<< ", publishers=" << contributingPublishers.size();
|
||||
}
|
||||
|
||||
// List tiers choose retention and relay policy only. ManifestCache remains
|
||||
// the sole high-water and signer-to-master namespace. Reconcile the whole
|
||||
// current source set so removed validators do not stay protected merely
|
||||
// because an older publisher generation once named them.
|
||||
hash_set<PublicKey> protectedMasters;
|
||||
protectedMasters.reserve(keyListings_.size() + candidateMasters.size());
|
||||
for (auto const& [master, _] : keyListings_)
|
||||
{
|
||||
(void)_;
|
||||
protectedMasters.insert(master);
|
||||
}
|
||||
for (auto const& master : candidateMasters)
|
||||
protectedMasters.insert(master);
|
||||
validatorManifests_.reconcileRetention(protectedMasters);
|
||||
|
||||
// Apply candidate manifests only after enforcing the global candidate
|
||||
// bound. They use ManifestCache's ordinary verification, collision and
|
||||
// sequence rules. Protected retention grants no consensus weight.
|
||||
for (auto const& [pubKey, collection] : publisherLists_)
|
||||
{
|
||||
if (collection.status != PublisherStatus::available)
|
||||
continue;
|
||||
for (auto const& candidate : collection.current.candidates)
|
||||
{
|
||||
if (!candidate.manifest ||
|
||||
!candidateMasters.contains(candidate.master))
|
||||
continue;
|
||||
|
||||
auto const& supplied = *candidate.manifest;
|
||||
Manifest manifest{
|
||||
supplied.serialized,
|
||||
supplied.masterKey,
|
||||
supplied.signingKey,
|
||||
supplied.sequence,
|
||||
supplied.domain};
|
||||
if (auto const result = validatorManifests_.applyManifest(
|
||||
std::move(manifest), ManifestRetention::protected_);
|
||||
result == ManifestDisposition::invalid)
|
||||
{
|
||||
JLOG(j_.warn()) << "List for " << strHex(pubKey)
|
||||
<< " contained invalid candidate manifest";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
publisherCandidateMasters_ = std::move(candidateMasters);
|
||||
}
|
||||
|
||||
ValidatorList::PublisherListStats
|
||||
ValidatorList::applyList(
|
||||
std::string const& globalManifest,
|
||||
@@ -1143,10 +1273,16 @@ ValidatorList::applyList(
|
||||
std::optional<uint256> const& hash,
|
||||
ValidatorList::lock_guard const& lock)
|
||||
{
|
||||
auto const& manifest = localManifest ? *localManifest : globalManifest;
|
||||
if (manifest.size() > kMaxManifestBase64)
|
||||
{
|
||||
JLOG(j_.warn()) << "UNL manifest exceeds maximum size";
|
||||
return PublisherListStats{ListDisposition::invalid};
|
||||
}
|
||||
|
||||
using namespace std::string_literals;
|
||||
|
||||
Json::Value list;
|
||||
auto const& manifest = localManifest ? *localManifest : globalManifest;
|
||||
auto [result, pubKeyOpt] = verify(lock, list, manifest, blob, signature);
|
||||
|
||||
if (!pubKeyOpt)
|
||||
@@ -1188,21 +1324,23 @@ ValidatorList::applyList(
|
||||
// Update publisher's list
|
||||
auto& pubCollection = publisherLists_[pubKey];
|
||||
auto const sequence = list[jss::sequence].asUInt();
|
||||
auto const accepted =
|
||||
auto const currentGeneration =
|
||||
(result == ListDisposition::accepted ||
|
||||
result == ListDisposition::expired);
|
||||
bool const available = result == ListDisposition::accepted;
|
||||
|
||||
if (accepted)
|
||||
pubCollection.status = result == ListDisposition::accepted
|
||||
? PublisherStatus::available
|
||||
: PublisherStatus::expired;
|
||||
if (currentGeneration)
|
||||
pubCollection.status =
|
||||
available ? PublisherStatus::available : PublisherStatus::expired;
|
||||
pubCollection.rawManifest = globalManifest;
|
||||
if (!pubCollection.maxSequence || sequence > *pubCollection.maxSequence)
|
||||
pubCollection.maxSequence = sequence;
|
||||
|
||||
Json::Value const& newList = list[jss::validators];
|
||||
Json::Value const* const newCandidates =
|
||||
list.isMember(jss::candidates) ? &list[jss::candidates] : nullptr;
|
||||
std::vector<PublicKey> oldList;
|
||||
if (accepted && pubCollection.remaining.count(sequence) != 0)
|
||||
if (currentGeneration && pubCollection.remaining.count(sequence) != 0)
|
||||
{
|
||||
// We've seen this list before and stored it in "remaining". The
|
||||
// normal expected process is that the processed list would have
|
||||
@@ -1211,7 +1349,7 @@ ValidatorList::applyList(
|
||||
// some of that work here.
|
||||
auto& publisher = pubCollection.current;
|
||||
// Copy the old validator list
|
||||
oldList = std::move(pubCollection.current.list);
|
||||
oldList = validatorMasters(pubCollection.current);
|
||||
// Move the publisher info from "remaining" to "current"
|
||||
publisher = std::move(pubCollection.remaining[sequence]);
|
||||
// Remove the entry in "remaining"
|
||||
@@ -1223,8 +1361,8 @@ ValidatorList::applyList(
|
||||
}
|
||||
else
|
||||
{
|
||||
auto& publisher = accepted ? pubCollection.current
|
||||
: pubCollection.remaining[sequence];
|
||||
auto& publisher = currentGeneration ? pubCollection.current
|
||||
: pubCollection.remaining[sequence];
|
||||
publisher.sequence = sequence;
|
||||
publisher.validFrom = TimeKeeper::time_point{TimeKeeper::duration{
|
||||
list.isMember(jss::effective) ? list[jss::effective].asUInt() : 0}};
|
||||
@@ -1237,11 +1375,11 @@ ValidatorList::applyList(
|
||||
if (hash)
|
||||
publisher.hash = *hash;
|
||||
|
||||
std::vector<PublicKey>& publisherList = publisher.list;
|
||||
std::vector<std::string>& manifests = publisher.manifests;
|
||||
std::vector<PublisherValidator>& publisherList = publisher.validators;
|
||||
std::vector<PublisherCandidate>& candidates = publisher.candidates;
|
||||
|
||||
// Copy the old validator list
|
||||
oldList = std::move(publisherList);
|
||||
oldList = validatorMasters(publisher);
|
||||
// Build the new validator list from "newList"
|
||||
publisherList.clear();
|
||||
publisherList.reserve(newList.size());
|
||||
@@ -1261,18 +1399,157 @@ ValidatorList::applyList(
|
||||
}
|
||||
else
|
||||
{
|
||||
publisherList.push_back(
|
||||
PublicKey(Slice{ret->data(), ret->size()}));
|
||||
}
|
||||
PublicKey key{Slice{ret->data(), ret->size()}};
|
||||
std::shared_ptr<Manifest const> validatorManifest;
|
||||
if (val.isMember(jss::manifest))
|
||||
{
|
||||
if (!val[jss::manifest].isString() ||
|
||||
val[jss::manifest].asString().size() >
|
||||
kMaxManifestBase64)
|
||||
{
|
||||
JLOG(j_.warn())
|
||||
<< "List for " << strHex(pubKey)
|
||||
<< " contained malformed validator manifest";
|
||||
}
|
||||
else
|
||||
{
|
||||
auto m = deserializeManifest(
|
||||
base64_decode(val[jss::manifest].asString()));
|
||||
if (!m || m->masterKey != key)
|
||||
{
|
||||
JLOG(j_.warn())
|
||||
<< "List for " << strHex(pubKey)
|
||||
<< " contained invalid or mismatched "
|
||||
"validator manifest";
|
||||
}
|
||||
else
|
||||
{
|
||||
validatorManifest =
|
||||
std::make_shared<Manifest const>(
|
||||
std::move(*m));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (val.isMember(jss::manifest) &&
|
||||
val[jss::manifest].isString())
|
||||
manifests.push_back(val[jss::manifest].asString());
|
||||
publisherList.push_back(
|
||||
PublisherValidator{key, std::move(validatorManifest)});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Standardize the list order by sorting
|
||||
std::sort(publisherList.begin(), publisherList.end());
|
||||
std::sort(
|
||||
publisherList.begin(),
|
||||
publisherList.end(),
|
||||
[](PublisherValidator const& lhs, PublisherValidator const& rhs) {
|
||||
return lhs.master < rhs.master;
|
||||
});
|
||||
|
||||
candidates.clear();
|
||||
if (newCandidates)
|
||||
{
|
||||
bool candidatePlaneValid = newCandidates->isArray() &&
|
||||
newCandidates->size() <= maxPublisherCandidates;
|
||||
if (!newCandidates->isArray())
|
||||
{
|
||||
JLOG(j_.error())
|
||||
<< "List for " << strHex(pubKey)
|
||||
<< " supplied a non-array candidate tier. Ignoring "
|
||||
"candidate tier only.";
|
||||
}
|
||||
else if (!candidatePlaneValid)
|
||||
{
|
||||
JLOG(j_.error())
|
||||
<< "List for " << strHex(pubKey) << " supplied "
|
||||
<< newCandidates->size() << " candidates; maximum is "
|
||||
<< maxPublisherCandidates
|
||||
<< ". Ignoring candidate tier only.";
|
||||
}
|
||||
else
|
||||
{
|
||||
std::vector<PublisherCandidate> parsed;
|
||||
parsed.reserve(newCandidates->size());
|
||||
hash_set<PublicKey> candidateKeys;
|
||||
candidateKeys.reserve(newCandidates->size());
|
||||
hash_set<PublicKey> validatorKeys;
|
||||
validatorKeys.reserve(publisherList.size());
|
||||
for (auto const& entry : publisherList)
|
||||
validatorKeys.insert(entry.master);
|
||||
|
||||
for (auto const& val : *newCandidates)
|
||||
{
|
||||
if (!val.isObject() ||
|
||||
!val.isMember(jss::validation_public_key) ||
|
||||
!val[jss::validation_public_key].isString())
|
||||
{
|
||||
candidatePlaneValid = false;
|
||||
break;
|
||||
}
|
||||
|
||||
auto const ret =
|
||||
strUnHex(val[jss::validation_public_key].asString());
|
||||
if (!ret || !publicKeyType(makeSlice(*ret)))
|
||||
{
|
||||
candidatePlaneValid = false;
|
||||
break;
|
||||
}
|
||||
|
||||
PublicKey key{Slice{ret->data(), ret->size()}};
|
||||
if (!candidateKeys.insert(key).second ||
|
||||
validatorKeys.contains(key))
|
||||
{
|
||||
candidatePlaneValid = false;
|
||||
break;
|
||||
}
|
||||
|
||||
std::shared_ptr<Manifest const> candidateManifest;
|
||||
if (val.isMember(jss::manifest))
|
||||
{
|
||||
if (!val[jss::manifest].isString() ||
|
||||
val[jss::manifest].asString().size() >
|
||||
kMaxManifestBase64)
|
||||
{
|
||||
candidatePlaneValid = false;
|
||||
break;
|
||||
}
|
||||
|
||||
auto m = deserializeManifest(
|
||||
base64_decode(val[jss::manifest].asString()));
|
||||
if (!m || m->masterKey != key || !m->verify())
|
||||
{
|
||||
candidatePlaneValid = false;
|
||||
break;
|
||||
}
|
||||
candidateManifest =
|
||||
std::make_shared<Manifest const>(std::move(*m));
|
||||
}
|
||||
parsed.push_back(
|
||||
PublisherCandidate{key, std::move(candidateManifest)});
|
||||
}
|
||||
|
||||
if (candidatePlaneValid)
|
||||
candidates = std::move(parsed);
|
||||
}
|
||||
|
||||
if (!candidatePlaneValid)
|
||||
{
|
||||
candidates.clear();
|
||||
JLOG(j_.error())
|
||||
<< "List for " << strHex(pubKey)
|
||||
<< " contained an invalid candidate tier; legacy "
|
||||
"validators remain accepted";
|
||||
}
|
||||
else
|
||||
{
|
||||
std::sort(
|
||||
candidates.begin(),
|
||||
candidates.end(),
|
||||
[](PublisherCandidate const& lhs,
|
||||
PublisherCandidate const& rhs) {
|
||||
return lhs.master < rhs.master;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
// If this publisher has ever sent a more updated version than the one
|
||||
// in this file, keep it. This scenario is unlikely, but legal.
|
||||
@@ -1287,9 +1564,20 @@ ValidatorList::applyList(
|
||||
PublisherListStats const applyResult{
|
||||
result, pubKey, pubCollection.status, *pubCollection.maxSequence};
|
||||
|
||||
if (accepted)
|
||||
if (currentGeneration)
|
||||
{
|
||||
updatePublisherList(pubKey, pubCollection.current, oldList, lock);
|
||||
// An expired generation is retained as newer publisher evidence, but
|
||||
// contributes no effective membership or manifest protection.
|
||||
if (!available)
|
||||
{
|
||||
pubCollection.current.validators.clear();
|
||||
pubCollection.current.candidates.clear();
|
||||
}
|
||||
auto const currentMasters = validatorMasters(pubCollection.current);
|
||||
updatePublisherList(pubKey, currentMasters, oldList, lock);
|
||||
if (available)
|
||||
ingestPublisherManifests(pubKey, pubCollection.current, lock);
|
||||
rebuildPublisherCandidates(lock);
|
||||
}
|
||||
|
||||
return applyResult;
|
||||
@@ -1366,13 +1654,15 @@ ValidatorList::verify(
|
||||
PublicKey masterPubKey = m->masterKey;
|
||||
auto const revoked = m->revoked();
|
||||
|
||||
auto const result = publisherManifests_.applyManifest(std::move(*m));
|
||||
auto const result = publisherManifests_.applyManifest(
|
||||
std::move(*m), ManifestRetention::protected_);
|
||||
|
||||
if (revoked && result == ManifestDisposition::accepted)
|
||||
{
|
||||
removePublisherList(lock, masterPubKey, PublisherStatus::revoked);
|
||||
// If the manifest is revoked, no future list is valid either
|
||||
publisherLists_[masterPubKey].remaining.clear();
|
||||
rebuildPublisherCandidates(lock);
|
||||
}
|
||||
|
||||
auto const signingKey = publisherManifests_.getSigningKey(masterPubKey);
|
||||
@@ -1446,6 +1736,15 @@ ValidatorList::listed(PublicKey const& identity) const
|
||||
return keyListings_.find(pubKey) != keyListings_.end();
|
||||
}
|
||||
|
||||
ValidatorManifestPolicy
|
||||
ValidatorList::manifestPolicy(PublicKey const& master) const
|
||||
{
|
||||
std::shared_lock readLock{mutex_};
|
||||
return {
|
||||
keyListings_.contains(master),
|
||||
publisherCandidateMasters_.contains(master)};
|
||||
}
|
||||
|
||||
bool
|
||||
ValidatorList::trusted(
|
||||
ValidatorList::shared_lock const&,
|
||||
@@ -1509,7 +1808,7 @@ ValidatorList::localPublicKey() const
|
||||
|
||||
bool
|
||||
ValidatorList::removePublisherList(
|
||||
ValidatorList::lock_guard const&,
|
||||
ValidatorList::lock_guard const& lock,
|
||||
PublicKey const& publisherKey,
|
||||
PublisherStatus reason)
|
||||
{
|
||||
@@ -1524,8 +1823,9 @@ ValidatorList::removePublisherList(
|
||||
JLOG(j_.debug()) << "Removing validator list for publisher "
|
||||
<< strHex(publisherKey);
|
||||
|
||||
for (auto const& val : iList->second.current.list)
|
||||
for (auto const& entry : iList->second.current.validators)
|
||||
{
|
||||
auto const& val = entry.master;
|
||||
auto const& iVal = keyListings_.find(val);
|
||||
if (iVal == keyListings_.end())
|
||||
continue;
|
||||
@@ -1536,7 +1836,8 @@ ValidatorList::removePublisherList(
|
||||
--iVal->second;
|
||||
}
|
||||
|
||||
iList->second.current.list.clear();
|
||||
iList->second.current.validators.clear();
|
||||
iList->second.current.candidates.clear();
|
||||
iList->second.status = reason;
|
||||
|
||||
return true;
|
||||
@@ -1545,7 +1846,7 @@ ValidatorList::removePublisherList(
|
||||
std::size_t
|
||||
ValidatorList::count(ValidatorList::shared_lock const&) const
|
||||
{
|
||||
return publisherLists_.size() + (localPublisherList.list.size() > 0);
|
||||
return publisherLists_.size() + (localPublisherList.validators.size() > 0);
|
||||
}
|
||||
|
||||
std::size_t
|
||||
@@ -1588,7 +1889,7 @@ ValidatorList::expires(ValidatorList::shared_lock const&) const
|
||||
}
|
||||
}
|
||||
|
||||
if (localPublisherList.list.size() > 0)
|
||||
if (localPublisherList.validators.size() > 0)
|
||||
{
|
||||
PublisherList collection = localPublisherList;
|
||||
// Unfetched
|
||||
@@ -1655,8 +1956,8 @@ ValidatorList::getJson() const
|
||||
Json::Value& jLocalStaticKeys =
|
||||
(res[jss::local_static_keys] = Json::arrayValue);
|
||||
|
||||
for (auto const& key : localPublisherList.list)
|
||||
jLocalStaticKeys.append(toBase58(TokenType::NodePublic, key));
|
||||
for (auto const& entry : localPublisherList.validators)
|
||||
jLocalStaticKeys.append(toBase58(TokenType::NodePublic, entry.master));
|
||||
|
||||
// Publisher lists
|
||||
Json::Value& jPublisherLists =
|
||||
@@ -1680,9 +1981,9 @@ ValidatorList::getJson() const
|
||||
if (publisherList.validFrom != TimeKeeper::time_point{})
|
||||
target[jss::effective] = to_string(publisherList.validFrom);
|
||||
Json::Value& keys = (target[jss::list] = Json::arrayValue);
|
||||
for (auto const& key : publisherList.list)
|
||||
for (auto const& entry : publisherList.validators)
|
||||
{
|
||||
keys.append(toBase58(TokenType::NodePublic, key));
|
||||
keys.append(toBase58(TokenType::NodePublic, entry.master));
|
||||
}
|
||||
};
|
||||
{
|
||||
@@ -1921,6 +2222,7 @@ ValidatorList::updateTrusted(
|
||||
|
||||
// Rotate pending and remove expired published lists
|
||||
bool good = true;
|
||||
bool publisherCandidatesDirty = false;
|
||||
// localPublisherList is not processed here. This is because the
|
||||
// Validators specified in the local config file do not expire nor do
|
||||
// they have a "remaining" section of PublisherList.
|
||||
@@ -1955,20 +2257,28 @@ ValidatorList::updateTrusted(
|
||||
candidate.validFrom <= closeTime,
|
||||
"ripple::ValidatorList::updateTrusted : maximum time");
|
||||
|
||||
auto const oldList = current.list;
|
||||
auto const oldList = validatorMasters(current);
|
||||
current = std::move(candidate);
|
||||
if (collection.status != PublisherStatus::available)
|
||||
collection.status = PublisherStatus::available;
|
||||
XRPL_ASSERT(
|
||||
current.sequence == sequence,
|
||||
"ripple::ValidatorList::updateTrusted : sequence match");
|
||||
// If the list is expired, remove the validators so they don't
|
||||
// get processed in. The expiration check below will do the rest
|
||||
// of the work
|
||||
if (current.validUntil <= closeTime)
|
||||
current.list.clear();
|
||||
|
||||
updatePublisherList(pubKey, current, oldList, lock);
|
||||
bool const expired = current.validUntil <= closeTime;
|
||||
collection.status = PublisherStatus::available;
|
||||
if (expired)
|
||||
{
|
||||
// Preserve the raw list for publisher history, but do not
|
||||
// retain parsed identities that were never counted in
|
||||
// keyListings_. A later refresh must not subtract them.
|
||||
current.validators.clear();
|
||||
current.candidates.clear();
|
||||
}
|
||||
auto const currentMasters = validatorMasters(current);
|
||||
|
||||
updatePublisherList(pubKey, currentMasters, oldList, lock);
|
||||
if (!expired)
|
||||
ingestPublisherManifests(pubKey, current, lock);
|
||||
publisherCandidatesDirty = true;
|
||||
|
||||
// Only broadcast the current, which will consequently only
|
||||
// send to peers that don't understand v2, or which are
|
||||
@@ -1995,11 +2305,14 @@ ValidatorList::updateTrusted(
|
||||
collection.current.validUntil <= closeTime)
|
||||
{
|
||||
removePublisherList(lock, pubKey, PublisherStatus::expired);
|
||||
publisherCandidatesDirty = true;
|
||||
ops.setUNLBlocked();
|
||||
}
|
||||
if (collection.status != PublisherStatus::available)
|
||||
good = false;
|
||||
}
|
||||
if (publisherCandidatesDirty)
|
||||
rebuildPublisherCandidates(lock);
|
||||
if (good)
|
||||
ops.clearUNLBlocked();
|
||||
|
||||
@@ -2091,7 +2404,7 @@ ValidatorList::updateTrusted(
|
||||
<< unlSize << ")";
|
||||
}
|
||||
|
||||
if ((publisherLists_.size() || localPublisherList.list.size()) &&
|
||||
if ((publisherLists_.size() || localPublisherList.validators.size()) &&
|
||||
unlSize == 0)
|
||||
{
|
||||
// No validators. Lock down.
|
||||
|
||||
@@ -20,6 +20,8 @@
|
||||
#include <xrpld/app/rdb/Wallet.h>
|
||||
#include <boost/format.hpp>
|
||||
|
||||
#include <cstddef>
|
||||
|
||||
namespace ripple {
|
||||
|
||||
std::unique_ptr<DatabaseCon>
|
||||
@@ -65,7 +67,7 @@ getManifests(
|
||||
continue;
|
||||
}
|
||||
|
||||
mCache.applyManifest(std::move(*mo));
|
||||
mCache.applyManifest(std::move(*mo), ManifestRetention::protected_);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -99,19 +101,27 @@ saveManifests(
|
||||
{
|
||||
soci::transaction tr(session);
|
||||
session << "DELETE FROM " << dbTable;
|
||||
std::size_t skipped = 0;
|
||||
for (auto const& v : map)
|
||||
{
|
||||
// Save all revocation manifests,
|
||||
// but only save trusted non-revocation manifests.
|
||||
// Preserve the existing wallet contract: terminal revocations are
|
||||
// saved, while non-revocation manifests are saved only for validators
|
||||
// selected by the caller. Retention remains an in-memory policy.
|
||||
if (!v.second.revoked() && !isTrusted(v.second.masterKey))
|
||||
{
|
||||
JLOG(j.info()) << "Untrusted manifest in cache not saved to db";
|
||||
++skipped;
|
||||
continue;
|
||||
}
|
||||
|
||||
saveManifest(session, dbTable, v.second.serialized);
|
||||
}
|
||||
tr.commit();
|
||||
|
||||
if (skipped != 0)
|
||||
{
|
||||
JLOG(j.info()) << skipped
|
||||
<< " untrusted manifest(s) in cache not saved to db";
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
|
||||
@@ -1027,6 +1027,27 @@ public:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/** Get the signing keys associated with current validations.
|
||||
|
||||
Includes trusted and untrusted validators. Calling this method also
|
||||
removes validations that have aged out under the normal validation
|
||||
freshness rules.
|
||||
*/
|
||||
auto
|
||||
getCurrentNodeKeys() -> hash_set<NodeKey>
|
||||
{
|
||||
hash_set<NodeKey> ret;
|
||||
std::lock_guard lock{mutex_};
|
||||
current(
|
||||
lock,
|
||||
[&](std::size_t numValidations) { ret.reserve(numValidations); },
|
||||
[&](NodeID const&, Validation const& validation) {
|
||||
ret.insert(validation.key());
|
||||
});
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/** Count the number of trusted full validations for the given ledger
|
||||
|
||||
@param ledgerID The identifier of ledger of interest
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
#ifndef RIPPLE_OVERLAY_MESSAGE_H_INCLUDED
|
||||
#define RIPPLE_OVERLAY_MESSAGE_H_INCLUDED
|
||||
|
||||
#include <xrpld/app/misc/Manifest.h>
|
||||
#include <xrpld/overlay/Compression.h>
|
||||
#include <xrpl/basics/ByteUtilities.h>
|
||||
#include <xrpl/protocol/PublicKey.h>
|
||||
@@ -37,6 +38,10 @@ namespace ripple {
|
||||
|
||||
constexpr std::size_t maximiumMessageSize = megabytes(64);
|
||||
|
||||
constexpr std::size_t manifestFramingBytes = 8;
|
||||
constexpr std::size_t maximumManifestsMessageSize =
|
||||
kMaxManifestsPerMessage * (kMaxManifestBytes + manifestFramingBytes);
|
||||
|
||||
// VFALCO NOTE If we forward declare Message and write out shared_ptr
|
||||
// instead of using the in-class type alias, we can remove the
|
||||
// entire ripple.pb.h from the main headers.
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include <xrpld/app/consensus/RCLValidations.h>
|
||||
#include <xrpld/app/ledger/LedgerMaster.h>
|
||||
#include <xrpld/app/misc/HashRouter.h>
|
||||
#include <xrpld/app/misc/NetworkOPs.h>
|
||||
@@ -36,6 +37,7 @@
|
||||
#include <xrpl/basics/random.h>
|
||||
#include <xrpl/beast/core/LexicalCast.h>
|
||||
#include <xrpl/protocol/STTx.h>
|
||||
#include <xrpl/resource/Fees.h>
|
||||
#include <xrpl/server/SimpleWriter.h>
|
||||
|
||||
#include <xrpld/core/ConfigSections.h>
|
||||
@@ -633,26 +635,93 @@ OverlayImpl::onManifests(
|
||||
std::shared_ptr<protocol::TMManifests> const& m,
|
||||
std::shared_ptr<PeerImp> const& from)
|
||||
{
|
||||
auto const n = m->list_size();
|
||||
auto const& journal = from->pjournal();
|
||||
|
||||
auto const total = static_cast<std::size_t>(m->list_size());
|
||||
if (total > kMaxManifestEntriesPerMessage)
|
||||
{
|
||||
from->charge(
|
||||
Resource::feeMalformedRequest, "too many manifest entries");
|
||||
JLOG(journal.warn())
|
||||
<< "Manifests: message had " << total << " entries; maximum is "
|
||||
<< kMaxManifestEntriesPerMessage;
|
||||
return;
|
||||
}
|
||||
|
||||
std::size_t untrusted = 0;
|
||||
bool skippedUntrusted = false;
|
||||
protocol::TMManifests relay;
|
||||
|
||||
for (std::size_t i = 0; i < n; ++i)
|
||||
for (std::size_t i = 0; i < total; ++i)
|
||||
{
|
||||
auto& s = m->list().Get(i).stobject();
|
||||
|
||||
if (auto mo = deserializeManifest(s))
|
||||
{
|
||||
auto const serialized = mo->serialized;
|
||||
auto const masterKey = mo->masterKey;
|
||||
|
||||
auto const result =
|
||||
app_.validatorManifests().applyManifest(std::move(*mo));
|
||||
auto policy = app_.validators().manifestPolicy(masterKey);
|
||||
if (!policy.relayEligible())
|
||||
{
|
||||
if (untrusted >= kMaxManifestsPerMessage)
|
||||
{
|
||||
skippedUntrusted = true;
|
||||
continue;
|
||||
}
|
||||
++untrusted;
|
||||
}
|
||||
|
||||
bool acceptedUpdate = false;
|
||||
ManifestDisposition result;
|
||||
if (policy.relayEligible())
|
||||
{
|
||||
result = app_.validatorManifests().applyManifest(
|
||||
std::move(*mo), ManifestRetention::protected_);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto const admission =
|
||||
app_.validatorManifests().applyManifestWithEviction(
|
||||
std::move(*mo), [this] {
|
||||
return app_.getValidations().getCurrentNodeKeys();
|
||||
});
|
||||
acceptedUpdate = admission.acceptedUpdate;
|
||||
result = admission.disposition;
|
||||
}
|
||||
|
||||
auto const latestPolicy =
|
||||
app_.validators().manifestPolicy(masterKey);
|
||||
if (!policy.relayEligible() && latestPolicy.relayEligible() &&
|
||||
(result == ManifestDisposition::accepted ||
|
||||
result == ManifestDisposition::untrustedCapacity))
|
||||
{
|
||||
// Protected membership may race bounded admission. Reapply the
|
||||
// already parsed bytes so the retained high-water is promoted
|
||||
// atomically by ManifestCache.
|
||||
mo = deserializeManifest(serialized);
|
||||
XRPL_ASSERT(
|
||||
mo,
|
||||
"ripple::OverlayImpl::onManifests : manifest "
|
||||
"deserialization succeeded for policy reconciliation");
|
||||
auto const reconciled = app_.validatorManifests().applyManifest(
|
||||
std::move(*mo), ManifestRetention::protected_);
|
||||
if (result == ManifestDisposition::untrustedCapacity)
|
||||
result = reconciled;
|
||||
else if (
|
||||
reconciled != ManifestDisposition::accepted &&
|
||||
reconciled != ManifestDisposition::stale)
|
||||
result = reconciled;
|
||||
}
|
||||
else if (policy.relayEligible() && !latestPolicy.relayEligible())
|
||||
{
|
||||
app_.validatorManifests().setRetention(
|
||||
masterKey, ManifestRetention::evictable);
|
||||
}
|
||||
policy = latestPolicy;
|
||||
|
||||
if (result == ManifestDisposition::accepted)
|
||||
{
|
||||
relay.add_list()->set_stobject(s);
|
||||
|
||||
// N.B.: this is important; the applyManifest call above moves
|
||||
// the loaded Manifest out of the optional so we need to
|
||||
// reload it here.
|
||||
@@ -664,7 +733,15 @@ OverlayImpl::onManifests(
|
||||
|
||||
app_.getOPs().pubManifest(*mo);
|
||||
|
||||
if (app_.validators().listed(mo->masterKey))
|
||||
// Do not live-relay a first-seen unlisted identity. Retained
|
||||
// identities may relay later rotations and revocations, while
|
||||
// listed identities always relay. If eviction later forgets
|
||||
// this identity, a reappearance is first-seen again and stops
|
||||
// locally instead of producing an immediate relay loop.
|
||||
if (policy.relayEligible() || acceptedUpdate)
|
||||
relay.add_list()->set_stobject(s);
|
||||
|
||||
if (policy.consensusListed)
|
||||
{
|
||||
auto db = app_.getWalletDB().checkoutDb();
|
||||
addValidatorManifest(*db, serialized);
|
||||
@@ -679,9 +756,22 @@ OverlayImpl::onManifests(
|
||||
}
|
||||
}
|
||||
|
||||
if (skippedUntrusted)
|
||||
{
|
||||
from->charge(
|
||||
Resource::feeMalformedRequest, "too many untrusted manifests");
|
||||
JLOG(journal.warn())
|
||||
<< "Manifests: message had " << total
|
||||
<< " entries; processed all trusted plus the first "
|
||||
<< kMaxManifestsPerMessage << " untrusted";
|
||||
}
|
||||
|
||||
if (!relay.list().empty())
|
||||
for_each([m2 = std::make_shared<Message>(relay, protocol::mtMANIFESTS)](
|
||||
std::shared_ptr<PeerImp>&& p) { p->send(m2); });
|
||||
for_each([m2 = std::make_shared<Message>(relay, protocol::mtMANIFESTS),
|
||||
source = from->id()](std::shared_ptr<PeerImp>&& p) {
|
||||
if (p->id() != source)
|
||||
p->send(m2);
|
||||
});
|
||||
}
|
||||
|
||||
void
|
||||
@@ -1180,34 +1270,86 @@ OverlayImpl::relay(
|
||||
return {};
|
||||
}
|
||||
|
||||
std::shared_ptr<Message>
|
||||
OverlayImpl::getManifestsMessage()
|
||||
std::vector<std::shared_ptr<Message>>
|
||||
OverlayImpl::getManifestsMessages()
|
||||
{
|
||||
std::lock_guard g(manifestLock_);
|
||||
|
||||
if (auto seq = app_.validatorManifests().sequence();
|
||||
seq != manifestListSeq_)
|
||||
auto const seq = app_.validatorManifests().sequence();
|
||||
if (seq != manifestListSeq_)
|
||||
{
|
||||
protocol::TMManifests tm;
|
||||
|
||||
std::vector<std::string> protectedManifests;
|
||||
std::vector<std::string> evictableManifests;
|
||||
app_.validatorManifests().for_each_manifest(
|
||||
[&tm](std::size_t s) { tm.mutable_list()->Reserve(s); },
|
||||
[&tm, &hr = app_.getHashRouter()](Manifest const& manifest) {
|
||||
tm.add_list()->set_stobject(
|
||||
manifest.serialized.data(), manifest.serialized.size());
|
||||
hr.addSuppression(manifest.hash());
|
||||
[&protectedManifests, &evictableManifests](std::size_t s) {
|
||||
protectedManifests.reserve(s);
|
||||
evictableManifests.reserve(s);
|
||||
},
|
||||
[&protectedManifests, &evictableManifests](
|
||||
Manifest const& manifest, ManifestRetention retention) {
|
||||
auto& entries = retention == ManifestRetention::protected_
|
||||
? protectedManifests
|
||||
: evictableManifests;
|
||||
entries.push_back(manifest.serialized);
|
||||
});
|
||||
|
||||
manifestMessage_.reset();
|
||||
std::shuffle(
|
||||
protectedManifests.begin(),
|
||||
protectedManifests.end(),
|
||||
default_prng());
|
||||
std::shuffle(
|
||||
evictableManifests.begin(),
|
||||
evictableManifests.end(),
|
||||
default_prng());
|
||||
|
||||
if (tm.list_size() != 0)
|
||||
manifestMessage_ =
|
||||
std::make_shared<Message>(tm, protocol::mtMANIFESTS);
|
||||
manifestMessages_.clear();
|
||||
protocol::TMManifests tm;
|
||||
auto flush = [this, &tm]() {
|
||||
if (tm.list_size() == 0)
|
||||
return;
|
||||
manifestMessages_.push_back(
|
||||
std::make_shared<Message>(tm, protocol::mtMANIFESTS));
|
||||
tm.Clear();
|
||||
};
|
||||
auto add = [this, &tm, &flush](std::string const& serialized) {
|
||||
if (static_cast<std::size_t>(tm.list_size()) ==
|
||||
kMaxManifestEntriesPerMessage)
|
||||
flush();
|
||||
|
||||
tm.add_list()->set_stobject(serialized.data(), serialized.size());
|
||||
if (Message::messageSize(tm) > maximumManifestsMessageSize)
|
||||
{
|
||||
tm.mutable_list()->RemoveLast();
|
||||
flush();
|
||||
|
||||
tm.add_list()->set_stobject(
|
||||
serialized.data(), serialized.size());
|
||||
if (Message::messageSize(tm) > maximumManifestsMessageSize)
|
||||
{
|
||||
tm.mutable_list()->RemoveLast();
|
||||
JLOG(journal_.warn())
|
||||
<< "Manifest exceeds peer snapshot message limit";
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// All protected entries are delivered across as many bounded frames as
|
||||
// needed. This covers local, listed, and publisher-candidate sources
|
||||
// without re-deriving provenance in the overlay.
|
||||
// The bounded evictable population remains best-effort and sampled.
|
||||
for (auto const& entry : protectedManifests)
|
||||
add(entry);
|
||||
|
||||
auto const take =
|
||||
std::min(kMaxManifestsPerMessage, evictableManifests.size());
|
||||
for (std::size_t i = 0; i < take; ++i)
|
||||
add(evictableManifests[i]);
|
||||
flush();
|
||||
|
||||
manifestListSeq_ = seq;
|
||||
}
|
||||
|
||||
return manifestMessage_;
|
||||
return manifestMessages_;
|
||||
}
|
||||
|
||||
void
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
namespace ripple {
|
||||
|
||||
@@ -124,11 +125,11 @@ private:
|
||||
// Transaction reduce-relay metrics
|
||||
metrics::TxMetrics txMetrics_;
|
||||
|
||||
// A message with the list of manifests we send to peers
|
||||
std::shared_ptr<Message> manifestMessage_;
|
||||
// Bounded messages containing the manifest snapshot sent to new peers.
|
||||
std::vector<std::shared_ptr<Message>> manifestMessages_;
|
||||
// Used to track whether we need to update the cached list of manifests
|
||||
std::optional<std::uint32_t> manifestListSeq_;
|
||||
// Protects the message and the sequence list of manifests
|
||||
// Protects the messages and the sequence list of manifests
|
||||
std::mutex manifestLock_;
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
@@ -241,8 +242,8 @@ public:
|
||||
std::optional<std::reference_wrapper<protocol::TMTransaction>> m,
|
||||
std::set<Peer::id_t> const& skip) override;
|
||||
|
||||
std::shared_ptr<Message>
|
||||
getManifestsMessage();
|
||||
std::vector<std::shared_ptr<Message>>
|
||||
getManifestsMessages();
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
//
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
#include <xrpld/app/ledger/TransactionMaster.h>
|
||||
#include <xrpld/app/misc/HashRouter.h>
|
||||
#include <xrpld/app/misc/LoadFeeTrack.h>
|
||||
#include <xrpld/app/misc/Manifest.h>
|
||||
#include <xrpld/app/misc/NetworkOPs.h>
|
||||
#include <xrpld/app/misc/Transaction.h>
|
||||
#include <xrpld/app/misc/ValidatorList.h>
|
||||
@@ -870,7 +871,7 @@ PeerImp::doProtocolStart()
|
||||
});
|
||||
}
|
||||
|
||||
if (auto m = overlay_.getManifestsMessage())
|
||||
for (auto const& m : overlay_.getManifestsMessages())
|
||||
send(m);
|
||||
|
||||
setTimer();
|
||||
@@ -1060,6 +1061,12 @@ PeerImp::onMessage(std::shared_ptr<protocol::TMManifests> const& m)
|
||||
if (s > 100)
|
||||
fee_.update(Resource::feeModerateBurdenPeer, "oversize");
|
||||
|
||||
// TODO(manifest): Charge proportionally to entry/signature-verification
|
||||
// work. Repeated frames of up to 100 manifests currently incur only the
|
||||
// trivial per-message peer charge.
|
||||
|
||||
// OverlayImpl bounds untrusted work and charges if the cap is exceeded;
|
||||
// trusted manifests are always processed and do not count against it.
|
||||
app_.getJobQueue().addJob(
|
||||
jtMANIFEST, "receiveManifests", [this, that = shared_from_this(), m]() {
|
||||
overlay_.onManifests(m, that);
|
||||
|
||||
@@ -382,6 +382,15 @@ invokeProtocolMessage(
|
||||
return result;
|
||||
}
|
||||
|
||||
// Drop an oversized TMManifests without penalizing an unpatched peer.
|
||||
if (header->message_type == protocol::mtMANIFESTS &&
|
||||
(header->payload_wire_size > maximumManifestsMessageSize ||
|
||||
header->uncompressed_size > maximumManifestsMessageSize))
|
||||
{
|
||||
result.first = header->total_wire_size;
|
||||
return result;
|
||||
}
|
||||
|
||||
bool success;
|
||||
|
||||
switch (header->message_type)
|
||||
|
||||
Reference in New Issue
Block a user