Add protocol message compression support:

* Peers negotiate compression via HTTP Header "X-Offer-Compression: lz4"
* Messages greater than 70 bytes and protocol type messages MANIFESTS,
  ENDPOINTS, TRANSACTION, GET_LEDGER, LEDGER_DATA, GET_OBJECT,
  and VALIDATORLIST are compressed
* If the compressed message is larger than the uncompressed message
  then the uncompressed message is sent
* Compression flag and the compression algorithm type are included
  in the message header
* Only LZ4 block compression is currently supported

src/ripple/overlay/impl/ConnectAttempt.cpp

@@ -197,7 +197,7 @@ ConnectAttempt::onHandshake (error_code ec)
     if (! sharedValue)
         return close(); // makeSharedValue logs
 
-    req_ = makeRequest(!overlay_.peerFinder().config().peerPrivate);
+    req_ = makeRequest(!overlay_.peerFinder().config().peerPrivate, app_.config().COMPRESSION);
 
     buildHandshake(req_, *sharedValue, overlay_.setup().networkID,
         overlay_.setup().public_ip, remote_endpoint_.address(), app_);
@@ -264,7 +264,7 @@ ConnectAttempt::onShutdown (error_code ec)
 //--------------------------------------------------------------------------
 
 auto
-ConnectAttempt::makeRequest (bool crawl) -> request_type
+ConnectAttempt::makeRequest (bool crawl, bool compressionEnabled) -> request_type
 {
     request_type m;
     m.method(boost::beast::http::verb::get);
@@ -275,6 +275,8 @@ ConnectAttempt::makeRequest (bool crawl) -> request_type
     m.insert ("Connection", "Upgrade");
     m.insert ("Connect-As", "Peer");
     m.insert ("Crawl", crawl ? "public" : "private");
+    if (compressionEnabled)
+        m.insert("X-Offer-Compression", "lz4");
     return m;
 }
 

src/ripple/overlay/impl/ConnectAttempt.h

@@ -93,7 +93,7 @@ private:
 
     static
     request_type
-    makeRequest (bool crawl);
+    makeRequest (bool crawl, bool compressionEnabled);
 
     void processResponse();
 

src/ripple/overlay/impl/Message.cpp

@@ -17,7 +17,6 @@
 */
 //==============================================================================
 
-#include <ripple/basics/safe_cast.h>
 #include <ripple/overlay/Message.h>
 #include <ripple/overlay/impl/TrafficCount.h>
 #include <cstdint>
@@ -25,8 +24,9 @@
 namespace ripple {
 
 Message::Message (::google::protobuf::Message const& message, int type)
-    : mCategory(TrafficCount::categorize(message, type, false))
+    : category_(TrafficCount::categorize(message, type, false))
 {
+    using namespace ripple::compression;
 
 #if defined(GOOGLE_PROTOBUF_VERSION) && (GOOGLE_PROTOBUF_VERSION >= 3011000)
     auto const messageBytes = message.ByteSizeLong ();
@@ -36,23 +36,129 @@ Message::Message (::google::protobuf::Message const& message, int type)
 
     assert (messageBytes != 0);
 
-    /** Number of bytes in a message header. */
-    std::size_t constexpr headerBytes = 6;
+    buffer_.resize (headerBytes + messageBytes);
 
-    mBuffer.resize (headerBytes + messageBytes);
-
-    auto ptr = mBuffer.data();
-
-    *ptr++ = static_cast<std::uint8_t>((messageBytes >> 24) & 0xFF);
-    *ptr++ = static_cast<std::uint8_t>((messageBytes >> 16) & 0xFF);
-    *ptr++ = static_cast<std::uint8_t>((messageBytes >> 8) & 0xFF);
-    *ptr++ = static_cast<std::uint8_t>(messageBytes & 0xFF);
-
-    *ptr++ = static_cast<std::uint8_t>((type >> 8) & 0xFF);
-    *ptr++ = static_cast<std::uint8_t> (type & 0xFF);
+    setHeader(buffer_.data(), messageBytes, type, Algorithm::None, 0);
 
     if (messageBytes != 0)
-        message.SerializeToArray(ptr, messageBytes);
+        message.SerializeToArray(buffer_.data() + headerBytes, messageBytes);
+}
+
+void
+Message::compress()
+{
+    using namespace ripple::compression;
+    auto const messageBytes = buffer_.size () - headerBytes;
+
+    auto type = getType(buffer_.data());
+
+    bool const compressible = [&]{
+        if (messageBytes <= 70)
+            return false;
+        switch(type)
+        {
+            case protocol::mtMANIFESTS:
+            case protocol::mtENDPOINTS:
+            case protocol::mtTRANSACTION:
+            case protocol::mtGET_LEDGER:
+            case protocol::mtLEDGER_DATA:
+            case protocol::mtGET_OBJECTS:
+            case protocol::mtVALIDATORLIST:
+                return true;
+            case protocol::mtPING:
+            case protocol::mtCLUSTER:
+            case protocol::mtPROPOSE_LEDGER:
+            case protocol::mtSTATUS_CHANGE:
+            case protocol::mtHAVE_SET:
+            case protocol::mtVALIDATION:
+            case protocol::mtGET_SHARD_INFO:
+            case protocol::mtSHARD_INFO:
+            case protocol::mtGET_PEER_SHARD_INFO:
+            case protocol::mtPEER_SHARD_INFO:
+                break;
+        }
+        return false;
+    }();
+
+    if (compressible)
+    {
+        auto payload = static_cast<void const*>(buffer_.data() + headerBytes);
+
+        auto compressedSize = ripple::compression::compress(
+                payload,
+                messageBytes,
+                [&](std::size_t inSize) { // size of required compressed buffer
+                    bufferCompressed_.resize(inSize + headerBytesCompressed);
+                    return (bufferCompressed_.data() + headerBytesCompressed);
+                });
+
+        if (compressedSize < (messageBytes - (headerBytesCompressed - headerBytes)))
+        {
+            bufferCompressed_.resize(headerBytesCompressed + compressedSize);
+            setHeader(bufferCompressed_.data(), compressedSize, type, Algorithm::LZ4, messageBytes);
+        }
+        else
+            bufferCompressed_.resize(0);
+    }
+}
+
+/** Set payload header
+ * Uncompressed message header
+ * 47-42    Set to 0
+ * 41-16    Payload size
+ * 15-0	    Message Type
+ * Compressed message header
+ * 79       Set to 0, indicates the message is compressed
+ * 78-76    Compression algorithm, value 1-7. Set to 1 to indicate LZ4 compression
+ * 75-74    Set to 0
+ * 73-48    Payload size
+ * 47-32	Message Type
+ * 31-0     Uncompressed message size
+*/
+void
+Message::setHeader(std::uint8_t* in, std::uint32_t payloadBytes, int type,
+                   Algorithm comprAlgorithm, std::uint32_t uncompressedBytes)
+{
+    auto h = in;
+
+    auto pack = [](std::uint8_t*& in, std::uint32_t size) {
+        *in++ = static_cast<std::uint8_t>((size >> 24) & 0x0F); // leftmost 4 are compression bits
+        *in++ = static_cast<std::uint8_t>((size >> 16) & 0xFF);
+        *in++ = static_cast<std::uint8_t>((size >> 8) & 0xFF);
+        *in++ = static_cast<std::uint8_t>(size & 0xFF);
+    };
+
+    pack(in, payloadBytes);
+
+    *in++ = static_cast<std::uint8_t>((type >> 8) & 0xFF);
+    *in++ = static_cast<std::uint8_t> (type & 0xFF);
+
+    if (comprAlgorithm != Algorithm::None)
+    {
+        pack(in, uncompressedBytes);
+        *h |= 0x80 | (static_cast<uint8_t>(comprAlgorithm) << 4);
+    }
+}
+
+std::vector <uint8_t> const&
+Message::getBuffer (Compressed tryCompressed)
+{
+    if (tryCompressed == Compressed::Off)
+        return buffer_;
+
+    std::call_once(once_flag_, &Message::compress, this);
+
+    if (bufferCompressed_.size() > 0)
+        return bufferCompressed_;
+    else
+        return buffer_;
+}
+
+int
+Message::getType(std::uint8_t const* in) const
+{
+    int type = (static_cast<int>(*(in + 4)) << 8) + *(in + 5);
+    return type;
 }
 
 }

src/ripple/overlay/impl/PeerImp.cpp

@@ -86,6 +86,7 @@ PeerImp::PeerImp (Application& app, id_t id,
     , slot_ (slot)
     , request_(std::move(request))
     , headers_(request_)
+    , compressionEnabled_(headers_["X-Offer-Compression"] == "lz4" ? Compressed::On : Compressed::Off)
 {
 }
 
@@ -219,7 +220,7 @@ PeerImp::send (std::shared_ptr<Message> const& m)
 
     overlay_.reportTraffic (
         safe_cast<TrafficCount::category>(m->getCategory()),
-        false, static_cast<int>(m->getBuffer().size()));
+        false, static_cast<int>(m->getBuffer(compressionEnabled_).size()));
 
     auto sendq_size = send_queue_.size();
 
@@ -246,7 +247,7 @@ PeerImp::send (std::shared_ptr<Message> const& m)
 
     boost::asio::async_write(
         stream_,
-        boost::asio::buffer(send_queue_.front()->getBuffer()),
+        boost::asio::buffer(send_queue_.front()->getBuffer(compressionEnabled_)),
         bind_executor(
             strand_,
             std::bind(
@@ -757,6 +758,8 @@ PeerImp::makeResponse (bool crawl,
     resp.insert("Connect-As", "Peer");
     resp.insert("Server", BuildInfo::getFullVersionString());
     resp.insert("Crawl", crawl ? "public" : "private");
+    if (req["X-Offer-Compression"] == "lz4" && app_.config().COMPRESSION)
+        resp.insert("X-Offer-Compression", "lz4");
 
     buildHandshake(resp, sharedValue, overlay_.setup().networkID,
         overlay_.setup().public_ip, remote_ip, app_);
@@ -945,7 +948,7 @@ PeerImp::onWriteMessage (error_code ec, std::size_t bytes_transferred)
         // Timeout on writes only
         return boost::asio::async_write(
             stream_,
-            boost::asio::buffer(send_queue_.front()->getBuffer()),
+            boost::asio::buffer(send_queue_.front()->getBuffer(compressionEnabled_)),
             bind_executor(
                 strand_,
                 std::bind(

src/ripple/overlay/impl/PeerImp.h

@@ -99,6 +99,7 @@ private:
     using address_type  = boost::asio::ip::address;
     using endpoint_type = boost::asio::ip::tcp::endpoint;
     using waitable_timer = boost::asio::basic_waitable_timer<std::chrono::steady_clock>;
+    using Compressed    = compression::Compressed;
 
     Application& app_;
     id_t const id_;
@@ -201,6 +202,8 @@ private:
     std::mutex mutable shardInfoMutex_;
     hash_map<PublicKey, ShardInfo> shardInfo_;
 
+    Compressed compressionEnabled_ = Compressed::Off;
+
     friend class OverlayImpl;
 
     class Metrics {
@@ -600,6 +603,9 @@ PeerImp::PeerImp (Application& app, std::unique_ptr<stream_type>&& stream_ptr,
     , slot_ (std::move(slot))
     , response_(std::move(response))
     , headers_(response_)
+    , compressionEnabled_(
+            headers_["X-Offer-Compression"] == "lz4" && app_.config().COMPRESSION
+                ? Compressed::On : Compressed::Off)
 {
     read_buffer_.commit (boost::asio::buffer_copy(read_buffer_.prepare(
         boost::asio::buffer_size(buffers)), buffers));

src/ripple/overlay/impl/ProtocolMessage.h

@@ -22,6 +22,7 @@
 
 #include <ripple/basics/ByteUtilities.h>
 #include <ripple/protocol/messages.h>
+#include <ripple/overlay/Compression.h>
 #include <ripple/overlay/Message.h>
 #include <ripple/overlay/impl/ZeroCopyStream.h>
 #include <boost/asio/buffer.hpp>
@@ -81,36 +82,66 @@ struct MessageHeader
     /** The size of the payload on the wire. */
     std::uint32_t payload_wire_size = 0;
 
+    /** Uncompressed message size if the message is compressed. */
+    std::uint32_t uncompressed_size = 0;
+
     /** The type of the message. */
     std::uint16_t message_type = 0;
+
+    /** Indicates which compression algorithm the payload is compressed with.
+     * Currenly only lz4 is supported. If None then the message is not compressed.
+     */
+    compression::Algorithm algorithm = compression::Algorithm::None;
 };
 
+template<typename BufferSequence>
+auto
+buffersBegin(BufferSequence const &bufs)
+{
+    return boost::asio::buffers_iterator<BufferSequence, std::uint8_t>::begin(bufs);
+}
+
 template <class BufferSequence>
 boost::optional<MessageHeader> parseMessageHeader(
     BufferSequence const& bufs,
     std::size_t size)
 {
-    auto iter = boost::asio::buffers_iterator<BufferSequence, std::uint8_t>::begin(bufs);
+    using namespace ripple::compression;
+    auto iter = buffersBegin(bufs);
 
     MessageHeader hdr;
+    auto const compressed = (*iter & 0x80) == 0x80;
 
-    // Version 1 header: uncompressed payload.
-    // The top six bits of the first byte are 0.
-    if ((*iter & 0xFC) == 0)
+    // Check valid header
+    if ((*iter & 0xFC) == 0 || compressed)
     {
-        hdr.header_size = 6;
+        hdr.header_size = compressed ? headerBytesCompressed : headerBytes;
 
         if (size < hdr.header_size)
             return {};
 
+        if (compressed)
+        {
+            uint8_t algorithm = (*iter & 0x70) >> 4;
+            if (algorithm != static_cast<std::uint8_t>(compression::Algorithm::LZ4))
+                return {};
+            hdr.algorithm = compression::Algorithm::LZ4;
+        }
+
         for (int i = 0; i != 4; ++i)
             hdr.payload_wire_size = (hdr.payload_wire_size << 8) + *iter++;
+        // clear the compression bits
+        hdr.payload_wire_size &= 0x03FFFFFF;
 
         hdr.total_wire_size = hdr.header_size + hdr.payload_wire_size;
 
         for (int i = 0; i != 2; ++i)
             hdr.message_type = (hdr.message_type << 8) + *iter++;
 
+        if (compressed)
+            for (int i = 0; i != 4; ++i)
+                hdr.uncompressed_size = (hdr.uncompressed_size << 8) + *iter++;
+
         return hdr;
     }
 
@@ -130,7 +161,22 @@ invoke (
     ZeroCopyInputStream<Buffers> stream(buffers);
     stream.Skip(header.header_size);
 
-    if (! m->ParseFromZeroCopyStream(&stream))
+    if (header.algorithm != compression::Algorithm::None)
+    {
+        std::vector<std::uint8_t> payload;
+        payload.resize(header.uncompressed_size);
+
+        auto payloadSize = ripple::compression::decompress(
+            stream,
+            header.payload_wire_size,
+            payload.data(),
+            header.uncompressed_size,
+            header.algorithm);
+
+        if (payloadSize == 0 || !m->ParseFromArray(payload.data(), payloadSize))
+            return false;
+    }
+    else if (!m->ParseFromZeroCopyStream(&stream))
         return false;
 
     handler.onMessageBegin (header.message_type, m, header.payload_wire_size);