Improve message buffering (RIPD-1699):

Specially crafted messages could cause the server to buffer large
amounts of memory which could increase memory pressure.

This commit changes how messages are buffered and imposes a limit
on the amount of data that the server is willing to buffer.

Acknowledgements:
Aaron Hook for responsibly disclosing this issue.

Bug Bounties and Responsible Disclosures:
We welcome reviews of the rippled code and urge researchers to
responsibly disclose any issues they may find. For information
on Ripple's Bug Bounty program, please visit:

    https://ripple.com/bug-bounty

src/ripple/overlay/Message.h

@@ -51,8 +51,19 @@ public:
 
 public:
     /** Number of bytes in a message header.
+
+        A message will not be processed unless a full header is received and
+        no messages smaller than this will be serialized.
     */
-    static size_t const kHeaderBytes = 6;
+    static std::size_t constexpr kHeaderBytes = 6;
+
+    /** The largest size that a message can be.
+
+        Sending a message whose size exceeds this may result in the connection
+        being dropped. A larger message size may be supported in the future or
+        negotiated as part of a protocol upgrade.
+     */
+    static std::size_t constexpr kMaxMessageSize = 64 * 1024 * 1024;
 
     Message (::google::protobuf::Message const& message, int type);
 

src/ripple/overlay/impl/ProtocolMessage.h

@@ -106,13 +106,26 @@ invokeProtocolMessage (Buffers const& buffers, Handler& handler)
     std::pair<std::size_t,boost::system::error_code> result = { 0, {} };
     boost::system::error_code& ec = result.second;
 
-    auto const type = Message::type(buffers);
-    if (type == 0)
+    auto const bs = boost::asio::buffer_size(buffers);
+
+    // If we don't even have enough bytes for the header, there's no point
+    // in doing any work.
+    if (bs < Message::kHeaderBytes)
         return result;
+
+    if (bs > Message::kMaxMessageSize)
+    {
+        result.second = make_error_code(boost::system::errc::message_size);
+        return result;
+    }
+
     auto const size = Message::kHeaderBytes + Message::size(buffers);
-    if (boost::asio::buffer_size(buffers) < size)
+
+    if (bs < size)
         return result;
 
+    auto const type = Message::type(buffers);
+
     switch (type)
     {
     case protocol::mtHELLO:         ec = detail::invoke<protocol::TMHello> (type, buffers, handler); break;