ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2026-04-29 15:37:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add protocol message compression support:

Browse Source 
* Peers negotiate compression via HTTP Header "X-Offer-Compression: lz4"
* Messages greater than 70 bytes and protocol type messages MANIFESTS,
  ENDPOINTS, TRANSACTION, GET_LEDGER, LEDGER_DATA, GET_OBJECT,
  and VALIDATORLIST are compressed
* If the compressed message is larger than the uncompressed message
  then the uncompressed message is sent
* Compression flag and the compression algorithm type are included
  in the message header
* Only LZ4 block compression is currently supported
This commit is contained in:
Gregory Tsipenyuk
2020-02-15 10:50:52 -05:00
committed by manojsdoshi
parent ade5eb71cf
commit 758a3792eb
14 changed files with 875 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
src/ripple/overlay/impl/Message.cpp
View File

@@ -17,7 +17,6 @@
*/
//==============================================================================
#include <ripple/basics/safe_cast.h>
#include <ripple/overlay/Message.h>
#include <ripple/overlay/impl/TrafficCount.h>
#include <cstdint>
@@ -25,8 +24,9 @@
namespace ripple {
Message::Message (::google::protobuf::Message const& message, int type)
: mCategory(TrafficCount::categorize(message, type, false))
: category_(TrafficCount::categorize(message, type, false))
{
using namespace ripple::compression;
#if defined(GOOGLE_PROTOBUF_VERSION) && (GOOGLE_PROTOBUF_VERSION >= 3011000)
auto const messageBytes = message.ByteSizeLong ();
@@ -36,23 +36,129 @@ Message::Message (::google::protobuf::Message const& message, int type)
assert (messageBytes != 0);
/** Number of bytes in a message header. */
std::size_t constexpr headerBytes = 6;
buffer_.resize (headerBytes + messageBytes);
mBuffer.resize (headerBytes + messageBytes);
auto ptr = mBuffer.data();
*ptr++ = static_cast<std::uint8_t>((messageBytes >> 24) & 0xFF);
*ptr++ = static_cast<std::uint8_t>((messageBytes >> 16) & 0xFF);
*ptr++ = static_cast<std::uint8_t>((messageBytes >> 8) & 0xFF);
*ptr++ = static_cast<std::uint8_t>(messageBytes & 0xFF);
*ptr++ = static_cast<std::uint8_t>((type >> 8) & 0xFF);
*ptr++ = static_cast<std::uint8_t> (type & 0xFF);
setHeader(buffer_.data(), messageBytes, type, Algorithm::None, 0);
if (messageBytes != 0)
message.SerializeToArray(ptr, messageBytes);
message.SerializeToArray(buffer_.data() + headerBytes, messageBytes);
}
void
Message::compress()
{
using namespace ripple::compression;
auto const messageBytes = buffer_.size () - headerBytes;
auto type = getType(buffer_.data());
bool const compressible = [&]{
if (messageBytes <= 70)
return false;
switch(type)
{
case protocol::mtMANIFESTS:
case protocol::mtENDPOINTS:
case protocol::mtTRANSACTION:
case protocol::mtGET_LEDGER:
case protocol::mtLEDGER_DATA:
case protocol::mtGET_OBJECTS:
case protocol::mtVALIDATORLIST:
return true;
case protocol::mtPING:
case protocol::mtCLUSTER:
case protocol::mtPROPOSE_LEDGER:
case protocol::mtSTATUS_CHANGE:
case protocol::mtHAVE_SET:
case protocol::mtVALIDATION:
case protocol::mtGET_SHARD_INFO:
case protocol::mtSHARD_INFO:
case protocol::mtGET_PEER_SHARD_INFO:
case protocol::mtPEER_SHARD_INFO:
break;
}
return false;
}();
if (compressible)
{
auto payload = static_cast<void const*>(buffer_.data() + headerBytes);
auto compressedSize = ripple::compression::compress(
payload,
messageBytes,
[&](std::size_t inSize) { // size of required compressed buffer
bufferCompressed_.resize(inSize + headerBytesCompressed);
return (bufferCompressed_.data() + headerBytesCompressed);
});
if (compressedSize < (messageBytes - (headerBytesCompressed - headerBytes)))
{
bufferCompressed_.resize(headerBytesCompressed + compressedSize);
setHeader(bufferCompressed_.data(), compressedSize, type, Algorithm::LZ4, messageBytes);
}
else
bufferCompressed_.resize(0);
}
}
/** Set payload header
* Uncompressed message header
* 47-42 Set to 0
* 41-16 Payload size
* 15-0 Message Type
* Compressed message header
* 79 Set to 0, indicates the message is compressed
* 78-76 Compression algorithm, value 1-7. Set to 1 to indicate LZ4 compression
* 75-74 Set to 0
* 73-48 Payload size
* 47-32 Message Type
* 31-0 Uncompressed message size
*/
void
Message::setHeader(std::uint8_t* in, std::uint32_t payloadBytes, int type,
Algorithm comprAlgorithm, std::uint32_t uncompressedBytes)
{
auto h = in;
auto pack = [](std::uint8_t*& in, std::uint32_t size) {
*in++ = static_cast<std::uint8_t>((size >> 24) & 0x0F); // leftmost 4 are compression bits
*in++ = static_cast<std::uint8_t>((size >> 16) & 0xFF);
*in++ = static_cast<std::uint8_t>((size >> 8) & 0xFF);
*in++ = static_cast<std::uint8_t>(size & 0xFF);
};
pack(in, payloadBytes);
*in++ = static_cast<std::uint8_t>((type >> 8) & 0xFF);
*in++ = static_cast<std::uint8_t> (type & 0xFF);
if (comprAlgorithm != Algorithm::None)
{
pack(in, uncompressedBytes);
*h |= 0x80 | (static_cast<uint8_t>(comprAlgorithm) << 4);
}
}
std::vector <uint8_t> const&
Message::getBuffer (Compressed tryCompressed)
{
if (tryCompressed == Compressed::Off)
return buffer_;
std::call_once(once_flag_, &Message::compress, this);
if (bufferCompressed_.size() > 0)
return bufferCompressed_;
else
return buffer_;
}
int
Message::getType(std::uint8_t const* in) const
{
int type = (static_cast<int>(*(in + 4)) << 8) + *(in + 5);
return type;
}
}