ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add experimental validation & proposal relay reduction support:

Browse Source 
- Add validation/proposal reduce-relay feature negotiation to
  the handshake
- Make squelch duration proportional to a number of peers that
  can be squelched
- Refactor makeRequest()/makeResponse() to facilitate handshake
  unit-testing
- Fix compression enable flag for inbound peer
- Fix compression algorithm parsing in the header parser
- Fix squelch duration in onMessage(TMSquelch)

This commit fixes 3624, fixes 3639 and fixes 3641
This commit is contained in:
Gregory Tsipenyuk
2020-10-21 12:41:04 -04:00
committed by Nik Bougalis
parent 44fe0e1fc4
commit 74d96ff4bd
22 changed files with 1076 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
src/test/overlay/compression_test.cpp
View File

@@ -25,6 +25,7 @@
#include <ripple/core/TimeKeeper.h>
#include <ripple/overlay/Compression.h>
#include <ripple/overlay/Message.h>
#include <ripple/overlay/impl/Handshake.h>
#include <ripple/overlay/impl/ProtocolMessage.h>
#include <ripple/overlay/impl/ZeroCopyStream.h>
#include <ripple/protocol/HashPrefix.h>
@@ -112,7 +113,7 @@ public:
BEAST_EXPECT(header);
if (header->algorithm == Algorithm::None)
if (!header || header->algorithm == Algorithm::None)
return;
std::vector<std::uint8_t> decompressed;
@@ -242,7 +243,7 @@ public:
uint256 const hash(ripple::sha512Half(123456789));
getLedger->set_ledgerhash(hash.begin(), hash.size());
getLedger->set_ledgerseq(123456789);
ripple::SHAMapNodeID sha(17, hash);
ripple::SHAMapNodeID sha(64, hash);
getLedger->add_nodeids(sha.getRawString());
getLedger->set_requestcookie(123456789);
getLedger->set_querytype(protocol::qtINDIRECT);
@@ -302,7 +303,7 @@ public:
uint256 hash(ripple::sha512Half(i));
auto object = getObject->add_objects();
object->set_hash(hash.data(), hash.size());
ripple::SHAMapNodeID sha(i % 55, hash);
ripple::SHAMapNodeID sha(64, hash);
object->set_nodeid(sha.getRawString());
object->set_index("");
object->set_data("");
@@ -458,14 +459,80 @@ public:
"TMValidatorListCollection");
}
void
testHandshake()
{
testcase("Handshake");
auto getEnv = [&](bool enable) {
Config c;
std::stringstream str;
str << "[reduce_relay]\n"
<< "vp_enable=1\n"
<< "vp_squelch=1\n"
<< "[compression]\n"
<< enable << "\n";
c.loadFromString(str.str());
auto env = std::make_shared<jtx::Env>(*this);
env->app().config().COMPRESSION = c.COMPRESSION;
env->app().config().VP_REDUCE_RELAY_ENABLE =
c.VP_REDUCE_RELAY_ENABLE;
env->app().config().VP_REDUCE_RELAY_SQUELCH =
c.VP_REDUCE_RELAY_SQUELCH;
return env;
};
auto handshake = [&](int outboundEnable, int inboundEnable) {
beast::IP::Address addr =
boost::asio::ip::address::from_string("172.1.1.100");
auto env = getEnv(outboundEnable);
auto request = ripple::makeRequest(
true,
env->app().config().COMPRESSION,
env->app().config().VP_REDUCE_RELAY_ENABLE);
http_request_type http_request;
http_request.version(request.version());
http_request.base() = request.base();
// feature enabled on the peer's connection only if both sides are
// enabled
auto const peerEnabled = inboundEnable && outboundEnable;
// inbound is enabled if the request's header has the feature
// enabled and the peer's configuration is enabled
auto const inboundEnabled = peerFeatureEnabled(
http_request, FEATURE_COMPR, "lz4", inboundEnable);
BEAST_EXPECT(!(peerEnabled ^ inboundEnabled));
env.reset();
env = getEnv(inboundEnable);
auto http_resp = ripple::makeResponse(
true,
http_request,
addr,
addr,
uint256{1},
1,
{1, 0},
env->app());
// outbound is enabled if the response's header has the feature
// enabled and the peer's configuration is enabled
auto const outboundEnabled = peerFeatureEnabled(
http_resp, FEATURE_COMPR, "lz4", outboundEnable);
BEAST_EXPECT(!(peerEnabled ^ outboundEnabled));
};
handshake(1, 1);
handshake(1, 0);
handshake(0, 1);
handshake(0, 0);
}
void
run() override
{
testProtocol();
testHandshake();
}
};
BEAST_DEFINE_TESTSUITE_MANUAL_PRIO(compression, ripple_data, ripple, 20);
BEAST_DEFINE_TESTSUITE_MANUAL(compression, ripple_data, ripple);
} // namespace test
} // namespace ripple

64
src/test/overlay/handshake_test.cpp Normal file
View File

@@ -0,0 +1,64 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2020 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <ripple/beast/unit_test.h>
#include <ripple/overlay/impl/Handshake.h>
namespace ripple {
namespace test {
class handshake_test : public beast::unit_test::suite
{
public:
handshake_test() = default;
void
testHandshake()
{
testcase("X-Protocol-Ctl");
boost::beast::http::fields headers;
headers.insert(
"X-Protocol-Ctl",
"feature1=v1,v2,v3; feature2=v4; feature3=10; feature4=1; "
"feature5=v6");
BEAST_EXPECT(!featureEnabled(headers, "feature1"));
BEAST_EXPECT(!isFeatureValue(headers, "feature1", "2"));
BEAST_EXPECT(isFeatureValue(headers, "feature1", "v1"));
BEAST_EXPECT(isFeatureValue(headers, "feature1", "v2"));
BEAST_EXPECT(isFeatureValue(headers, "feature1", "v3"));
BEAST_EXPECT(isFeatureValue(headers, "feature2", "v4"));
BEAST_EXPECT(!isFeatureValue(headers, "feature3", "1"));
BEAST_EXPECT(isFeatureValue(headers, "feature3", "10"));
BEAST_EXPECT(!isFeatureValue(headers, "feature4", "10"));
BEAST_EXPECT(isFeatureValue(headers, "feature4", "1"));
BEAST_EXPECT(!featureEnabled(headers, "v6"));
}
void
run() override
{
testHandshake();
}
};
BEAST_DEFINE_TESTSUITE(handshake, ripple_data, ripple);
} // namespace test
} // namespace ripple

256
src/test/overlay/reduce_relay_test.cpp
View File

@@ -21,6 +21,7 @@
#include <ripple/overlay/Message.h>
#include <ripple/overlay/Peer.h>
#include <ripple/overlay/Slot.h>
#include <ripple/overlay/impl/Handshake.h>
#include <ripple/protocol/SecretKey.h>
#include <ripple.pb.h>
#include <test/jtx/Env.h>
@@ -437,7 +438,8 @@ class PeerSim : public PeerPartial, public std::enable_shared_from_this<PeerSim>
{
public:
using id_t = Peer::id_t;
PeerSim(Overlay& overlay) : overlay_(overlay)
PeerSim(Overlay& overlay, beast::Journal journal)
: overlay_(overlay), squelch_(journal)
{
id_ = sid_++;
}
@@ -462,7 +464,7 @@ public:
{
auto validator = m->getValidatorKey();
assert(validator);
if (squelch_.isSquelched(*validator))
if (!squelch_.expireSquelch(*validator))
return;
overlay_.updateSlotAndSquelch({}, *validator, id(), f);
@@ -474,24 +476,28 @@ public:
{
auto validator = squelch.validatorpubkey();
PublicKey key(Slice(validator.data(), validator.size()));
squelch_.squelch(key, squelch.squelch(), squelch.squelchduration());
if (squelch.squelch())
squelch_.addSquelch(
key, std::chrono::seconds{squelch.squelchduration()});
else
squelch_.removeSquelch(key);
}
private:
inline static id_t sid_ = 0;
id_t id_;
Overlay& overlay_;
squelch::Squelch<ManualClock> squelch_;
reduce_relay::Squelch<ManualClock> squelch_;
};
class OverlaySim : public Overlay, public squelch::SquelchHandler
class OverlaySim : public Overlay, public reduce_relay::SquelchHandler
{
using Peers = std::unordered_map<Peer::id_t, PeerSPtr>;
public:
using id_t = Peer::id_t;
using clock_type = ManualClock;
OverlaySim(Application& app) : slots_(app, *this)
OverlaySim(Application& app) : slots_(app, *this), app_(app)
{
}
@@ -506,7 +512,7 @@ public:
}
std::uint16_t
inState(PublicKey const& validator, squelch::PeerState state)
inState(PublicKey const& validator, reduce_relay::PeerState state)
{
auto res = slots_.inState(validator, state);
return res ? *res : 0;
@@ -545,7 +551,7 @@ public:
Peer::id_t id;
if (peersCache_.empty() || !useCache)
{
peer = std::make_shared<PeerSim>(*this);
peer = std::make_shared<PeerSim>(*this, app_.journal("Squelch"));
id = peer->id();
}
else
@@ -602,7 +608,7 @@ public:
bool
isCountingState(PublicKey const& validator)
{
return slots_.inState(validator, squelch::SlotState::Counting);
return slots_.inState(validator, reduce_relay::SlotState::Counting);
}
std::set<id_t>
@@ -629,7 +635,7 @@ public:
std::unordered_map<
id_t,
std::tuple<
squelch::PeerState,
reduce_relay::PeerState,
std::uint16_t,
std::uint32_t,
std::uint32_t>>
@@ -664,7 +670,8 @@ private:
UnsquelchCB unsquelch_;
Peers peers_;
Peers peersCache_;
squelch::Slots<ManualClock> slots_;
reduce_relay::Slots<ManualClock> slots_;
Application& app_;
};
class Network
@@ -843,8 +850,8 @@ public:
for (auto& [_, v] : peers)
{
(void)_;
if (std::get<squelch::PeerState>(v) ==
squelch::PeerState::Squelched)
if (std::get<reduce_relay::PeerState>(v) ==
reduce_relay::PeerState::Squelched)
return false;
}
}
@@ -858,7 +865,7 @@ private:
class reduce_relay_test : public beast::unit_test::suite
{
using Slot = squelch::Slot<ManualClock>;
using Slot = reduce_relay::Slot<ManualClock>;
using id_t = Peer::id_t;
protected:
@@ -870,7 +877,7 @@ protected:
<< "num peers " << (int)network_.overlay().getNumPeers()
<< std::endl;
for (auto& [k, v] : peers)
std::cout << k << ":" << (int)std::get<squelch::PeerState>(v)
std::cout << k << ":" << (int)std::get<reduce_relay::PeerState>(v)
<< " ";
std::cout << std::endl;
}
@@ -950,7 +957,8 @@ protected:
str << s << " ";
if (log)
std::cout
<< (double)squelch::epoch<milliseconds>(now).count() /
<< (double)reduce_relay::epoch<milliseconds>(now)
.count() /
1000.
<< " random, squelched, validator: " << validator.id()
<< " peers: " << str.str() << std::endl;
@@ -958,7 +966,7 @@ protected:
network_.overlay().isCountingState(validator);
BEAST_EXPECT(
countingState == false &&
selected.size() == squelch::MAX_SELECTED_PEERS);
selected.size() == reduce_relay::MAX_SELECTED_PEERS);
}
// Trigger Link Down or Peer Disconnect event
@@ -1038,12 +1046,13 @@ protected:
event.isSelected_ =
network_.overlay().isSelected(event.key_, event.peer_);
auto peers = network_.overlay().getPeers(event.key_);
auto d = squelch::epoch<milliseconds>(now).count() -
auto d = reduce_relay::epoch<milliseconds>(now).count() -
std::get<3>(peers[event.peer_]);
mustHandle = event.isSelected_ &&
d > milliseconds(squelch::IDLED).count() &&
d > milliseconds(reduce_relay::IDLED).count() &&
network_.overlay().inState(
event.key_, squelch::PeerState::Squelched) > 0 &&
event.key_, reduce_relay::PeerState::Squelched) >
0 &&
peers.find(event.peer_) != peers.end();
}
network_.overlay().deleteIdlePeers(
@@ -1062,7 +1071,7 @@ protected:
}
if (event.state_ == State::WaitReset ||
(event.state_ == State::On &&
(now - event.time_ > (squelch::IDLED + seconds(2)))))
(now - event.time_ > (reduce_relay::IDLED + seconds(2)))))
{
bool handled =
event.state_ == State::WaitReset || !event.handled_;
@@ -1158,16 +1167,17 @@ protected:
if (squelched)
{
BEAST_EXPECT(
squelched == MAX_PEERS - squelch::MAX_SELECTED_PEERS);
squelched ==
MAX_PEERS - reduce_relay::MAX_SELECTED_PEERS);
n++;
}
},
1,
squelch::MAX_MESSAGE_THRESHOLD + 2,
reduce_relay::MAX_MESSAGE_THRESHOLD + 2,
purge,
resetClock);
auto selected = network_.overlay().getSelected(network_.validator(0));
BEAST_EXPECT(selected.size() == squelch::MAX_SELECTED_PEERS);
BEAST_EXPECT(selected.size() == reduce_relay::MAX_SELECTED_PEERS);
BEAST_EXPECT(n == 1); // only one selection round
auto res = checkCounting(network_.validator(0), false);
BEAST_EXPECT(res);
@@ -1231,7 +1241,7 @@ protected:
unsquelched++;
});
BEAST_EXPECT(
unsquelched == MAX_PEERS - squelch::MAX_SELECTED_PEERS);
unsquelched == MAX_PEERS - reduce_relay::MAX_SELECTED_PEERS);
BEAST_EXPECT(checkCounting(network_.validator(0), true));
});
}
@@ -1244,7 +1254,7 @@ protected:
doTest("Selected Peer Stops Relaying", log, [this](bool log) {
ManualClock::advance(seconds(601));
BEAST_EXPECT(propagateAndSquelch(log, true, false));
ManualClock::advance(squelch::IDLED + seconds(1));
ManualClock::advance(reduce_relay::IDLED + seconds(1));
std::uint16_t unsquelched = 0;
network_.overlay().deleteIdlePeers(
[&](PublicKey const& key, PeerWPtr const& peer) {
@@ -1252,7 +1262,7 @@ protected:
});
auto peers = network_.overlay().getPeers(network_.validator(0));
BEAST_EXPECT(
unsquelched == MAX_PEERS - squelch::MAX_SELECTED_PEERS);
unsquelched == MAX_PEERS - reduce_relay::MAX_SELECTED_PEERS);
BEAST_EXPECT(checkCounting(network_.validator(0), true));
});
}
@@ -1267,8 +1277,8 @@ protected:
BEAST_EXPECT(propagateAndSquelch(log, true, false));
auto peers = network_.overlay().getPeers(network_.validator(0));
auto it = std::find_if(peers.begin(), peers.end(), [&](auto it) {
return std::get<squelch::PeerState>(it.second) ==
squelch::PeerState::Squelched;
return std::get<reduce_relay::PeerState>(it.second) ==
reduce_relay::PeerState::Squelched;
});
assert(it != peers.end());
std::uint16_t unsquelched = 0;
@@ -1289,37 +1299,37 @@ protected:
std::string toLoad(R"rippleConfig(
[reduce_relay]
enable=1
squelch=1
vp_enable=1
vp_squelch=1
)rippleConfig");
c.loadFromString(toLoad);
BEAST_EXPECT(c.REDUCE_RELAY_ENABLE == true);
BEAST_EXPECT(c.REDUCE_RELAY_SQUELCH == true);
BEAST_EXPECT(c.VP_REDUCE_RELAY_ENABLE == true);
BEAST_EXPECT(c.VP_REDUCE_RELAY_SQUELCH == true);
Config c1;
toLoad = (R"rippleConfig(
[reduce_relay]
enable=0
squelch=0
vp_enable=0
vp_squelch=0
)rippleConfig");
c1.loadFromString(toLoad);
BEAST_EXPECT(c1.REDUCE_RELAY_ENABLE == false);
BEAST_EXPECT(c1.REDUCE_RELAY_SQUELCH == false);
BEAST_EXPECT(c1.VP_REDUCE_RELAY_ENABLE == false);
BEAST_EXPECT(c1.VP_REDUCE_RELAY_SQUELCH == false);
Config c2;
toLoad = R"rippleConfig(
[reduce_relay]
enabled=1
squelched=1
vp_enabled=1
vp_squelched=1
)rippleConfig";
c2.loadFromString(toLoad);
BEAST_EXPECT(c2.REDUCE_RELAY_ENABLE == false);
BEAST_EXPECT(c2.REDUCE_RELAY_SQUELCH == false);
BEAST_EXPECT(c2.VP_REDUCE_RELAY_ENABLE == false);
BEAST_EXPECT(c2.VP_REDUCE_RELAY_SQUELCH == false);
});
}
@@ -1354,7 +1364,7 @@ squelched=1
peers = network_.overlay().getPeers(network_.validator(0));
BEAST_EXPECT(std::get<1>(peers[0]) == (nMessages - 1));
// advance the clock
ManualClock::advance(squelch::IDLED + seconds(1));
ManualClock::advance(reduce_relay::IDLED + seconds(1));
network_.overlay().updateSlotAndSquelch(
key,
network_.validator(0),
@@ -1366,6 +1376,166 @@ squelched=1
});
}
struct Handler : public reduce_relay::SquelchHandler
{
Handler() : maxDuration_(0)
{
}
void
squelch(PublicKey const&, Peer::id_t, std::uint32_t duration)
const override
{
if (duration > maxDuration_)
maxDuration_ = duration;
}
void
unsquelch(PublicKey const&, Peer::id_t) const override
{
}
mutable int maxDuration_;
};
void
testRandomSquelch(bool l)
{
doTest("Random Squelch", l, [&](bool l) {
PublicKey validator = std::get<0>(randomKeyPair(KeyType::ed25519));
Handler handler;
auto run = [&](int npeers) {
handler.maxDuration_ = 0;
reduce_relay::Slots<ManualClock> slots(env_.app(), handler);
// 1st message from a new peer switches the slot
// to counting state and resets the counts of all peers +
// MAX_MESSAGE_THRESHOLD + 1 messages to reach the threshold
// and switch the slot's state to peer selection.
for (int m = 1; m <= reduce_relay::MAX_MESSAGE_THRESHOLD + 2;
m++)
{
for (int peer = 0; peer < npeers; peer++)
{
// make unique message hash to make the
// slot's internal hash router accept the message
std::uint64_t mid = m * 1000 + peer;
uint256 const message{mid};
slots.updateSlotAndSquelch(
message,
validator,
peer,
protocol::MessageType::mtVALIDATION);
}
}
// make Slot's internal hash router expire all messages
ManualClock::advance(hours(1));
};
using namespace reduce_relay;
// expect max duration less than MAX_UNSQUELCH_EXPIRE_DEFAULT with
// less than or equal to 60 peers
run(20);
BEAST_EXPECT(
handler.maxDuration_ >= MIN_UNSQUELCH_EXPIRE.count() &&
handler.maxDuration_ <= MAX_UNSQUELCH_EXPIRE_DEFAULT.count());
run(60);
BEAST_EXPECT(
handler.maxDuration_ >= MIN_UNSQUELCH_EXPIRE.count() &&
handler.maxDuration_ <= MAX_UNSQUELCH_EXPIRE_DEFAULT.count());
// expect max duration greater than MIN_UNSQUELCH_EXPIRE and less
// than MAX_UNSQUELCH_EXPIRE_PEERS with peers greater than 60
// and less than 360
run(350);
// can't make this condition stronger. squelch
// duration is probabilistic and max condition may still fail.
// log when the value is low
BEAST_EXPECT(
handler.maxDuration_ >= MIN_UNSQUELCH_EXPIRE.count() &&
handler.maxDuration_ <= MAX_UNSQUELCH_EXPIRE_PEERS.count());
using namespace beast::unit_test::detail;
if (handler.maxDuration_ <= MAX_UNSQUELCH_EXPIRE_DEFAULT.count())
log << make_reason(
"warning: squelch duration is low",
__FILE__,
__LINE__)
<< std::endl
<< std::flush;
// more than 400 is still less than MAX_UNSQUELCH_EXPIRE_PEERS
run(400);
BEAST_EXPECT(
handler.maxDuration_ >= MIN_UNSQUELCH_EXPIRE.count() &&
handler.maxDuration_ <= MAX_UNSQUELCH_EXPIRE_PEERS.count());
if (handler.maxDuration_ <= MAX_UNSQUELCH_EXPIRE_DEFAULT.count())
log << make_reason(
"warning: squelch duration is low",
__FILE__,
__LINE__)
<< std::endl
<< std::flush;
});
}
void
testHandshake(bool log)
{
doTest("Handshake", log, [&](bool log) {
auto setEnv = [&](bool enable) {
Config c;
std::stringstream str;
str << "[reduce_relay]\n"
<< "vp_enable=" << enable << "\n"
<< "vp_squelch=" << enable << "\n"
<< "[compression]\n"
<< "1\n";
c.loadFromString(str.str());
env_.app().config().VP_REDUCE_RELAY_ENABLE =
c.VP_REDUCE_RELAY_ENABLE;
env_.app().config().VP_REDUCE_RELAY_SQUELCH =
c.VP_REDUCE_RELAY_SQUELCH;
env_.app().config().COMPRESSION = c.COMPRESSION;
};
auto handshake = [&](int outboundEnable, int inboundEnable) {
beast::IP::Address addr =
boost::asio::ip::address::from_string("172.1.1.100");
setEnv(outboundEnable);
auto request = ripple::makeRequest(
true,
env_.app().config().COMPRESSION,
env_.app().config().VP_REDUCE_RELAY_ENABLE);
http_request_type http_request;
http_request.version(request.version());
http_request.base() = request.base();
// feature enabled on the peer's connection only if both sides
// are enabled
auto const peerEnabled = inboundEnable && outboundEnable;
// inbound is enabled if the request's header has the feature
// enabled and the peer's configuration is enabled
auto const inboundEnabled = peerFeatureEnabled(
http_request, FEATURE_VPRR, inboundEnable);
BEAST_EXPECT(!(peerEnabled ^ inboundEnabled));
setEnv(inboundEnable);
auto http_resp = ripple::makeResponse(
true,
http_request,
addr,
addr,
uint256{1},
1,
{1, 0},
env_.app());
// outbound is enabled if the response's header has the feature
// enabled and the peer's configuration is enabled
auto const outboundEnabled =
peerFeatureEnabled(http_resp, FEATURE_VPRR, outboundEnable);
BEAST_EXPECT(!(peerEnabled ^ outboundEnabled));
};
handshake(1, 1);
handshake(1, 0);
handshake(0, 1);
handshake(0, 0);
});
}
jtx::Env env_;
Network network_;
@@ -1387,6 +1557,8 @@ public:
testSelectedPeerDisconnects(log);
testSelectedPeerStopsRelaying(log);
testInternalHashRouter(log);
testRandomSquelch(log);
testHandshake(log);
}
};