ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add protocol message compression support:

Browse Source 
* Peers negotiate compression via HTTP Header "X-Offer-Compression: lz4"
* Messages greater than 70 bytes and protocol type messages MANIFESTS,
  ENDPOINTS, TRANSACTION, GET_LEDGER, LEDGER_DATA, GET_OBJECT,
  and VALIDATORLIST are compressed
* If the compressed message is larger than the uncompressed message
  then the uncompressed message is sent
* Compression flag and the compression algorithm type are included
  in the message header
* Only LZ4 block compression is currently supported
This commit is contained in:
Gregory Tsipenyuk
2020-02-15 10:50:52 -05:00
committed by manojsdoshi
parent ade5eb71cf
commit 758a3792eb
14 changed files with 875 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
src/ripple/overlay/impl/ProtocolMessage.h
View File

@@ -22,6 +22,7 @@
#include <ripple/basics/ByteUtilities.h>
#include <ripple/protocol/messages.h>
#include <ripple/overlay/Compression.h>
#include <ripple/overlay/Message.h>
#include <ripple/overlay/impl/ZeroCopyStream.h>
#include <boost/asio/buffer.hpp>
@@ -81,36 +82,66 @@ struct MessageHeader
/** The size of the payload on the wire. */
std::uint32_t payload_wire_size = 0;
/** Uncompressed message size if the message is compressed. */
std::uint32_t uncompressed_size = 0;
/** The type of the message. */
std::uint16_t message_type = 0;
/** Indicates which compression algorithm the payload is compressed with.
* Currenly only lz4 is supported. If None then the message is not compressed.
*/
compression::Algorithm algorithm = compression::Algorithm::None;
};
template<typename BufferSequence>
auto
buffersBegin(BufferSequence const &bufs)
{
return boost::asio::buffers_iterator<BufferSequence, std::uint8_t>::begin(bufs);
}
template <class BufferSequence>
boost::optional<MessageHeader> parseMessageHeader(
BufferSequence const& bufs,
std::size_t size)
{
auto iter = boost::asio::buffers_iterator<BufferSequence, std::uint8_t>::begin(bufs);
using namespace ripple::compression;
auto iter = buffersBegin(bufs);
MessageHeader hdr;
auto const compressed = (*iter & 0x80) == 0x80;
// Version 1 header: uncompressed payload.
// The top six bits of the first byte are 0.
if ((*iter & 0xFC) == 0)
// Check valid header
if ((*iter & 0xFC) == 0 || compressed)
{
hdr.header_size = 6;
hdr.header_size = compressed ? headerBytesCompressed : headerBytes;
if (size < hdr.header_size)
return {};
if (compressed)
{
uint8_t algorithm = (*iter & 0x70) >> 4;
if (algorithm != static_cast<std::uint8_t>(compression::Algorithm::LZ4))
return {};
hdr.algorithm = compression::Algorithm::LZ4;
}
for (int i = 0; i != 4; ++i)
hdr.payload_wire_size = (hdr.payload_wire_size << 8) + *iter++;
// clear the compression bits
hdr.payload_wire_size &= 0x03FFFFFF;
hdr.total_wire_size = hdr.header_size + hdr.payload_wire_size;
for (int i = 0; i != 2; ++i)
hdr.message_type = (hdr.message_type << 8) + *iter++;
if (compressed)
for (int i = 0; i != 4; ++i)
hdr.uncompressed_size = (hdr.uncompressed_size << 8) + *iter++;
return hdr;
}
@@ -130,7 +161,22 @@ invoke (
ZeroCopyInputStream<Buffers> stream(buffers);
stream.Skip(header.header_size);
if (! m->ParseFromZeroCopyStream(&stream))
if (header.algorithm != compression::Algorithm::None)
{
std::vector<std::uint8_t> payload;
payload.resize(header.uncompressed_size);
auto payloadSize = ripple::compression::decompress(
stream,
header.payload_wire_size,
payload.data(),
header.uncompressed_size,
header.algorithm);
if (payloadSize == 0 || !m->ParseFromArray(payload.data(), payloadSize))
return false;
}
else if (!m->ParseFromZeroCopyStream(&stream))
return false;
handler.onMessageBegin (header.message_type, m, header.payload_wire_size);