Improve message buffering (RIPD-1699):

Specially crafted messages could cause the server to buffer large
amounts of memory which could increase memory pressure.

This commit changes how messages are buffered and imposes a limit
on the amount of data that the server is willing to buffer.

Acknowledgements:
Aaron Hook for responsibly disclosing this issue.

Bug Bounties and Responsible Disclosures:
We welcome reviews of the rippled code and urge researchers to
responsibly disclose any issues they may find. For information
on Ripple's Bug Bounty program, please visit:

    https://ripple.com/bug-bounty

src/ripple/overlay/impl/ProtocolMessage.h

@@ -106,13 +106,26 @@ invokeProtocolMessage (Buffers const& buffers, Handler& handler)
     std::pair<std::size_t,boost::system::error_code> result = { 0, {} };
     boost::system::error_code& ec = result.second;
 
-    auto const type = Message::type(buffers);
-    if (type == 0)
+    auto const bs = boost::asio::buffer_size(buffers);
+
+    // If we don't even have enough bytes for the header, there's no point
+    // in doing any work.
+    if (bs < Message::kHeaderBytes)
         return result;
+
+    if (bs > Message::kMaxMessageSize)
+    {
+        result.second = make_error_code(boost::system::errc::message_size);
+        return result;
+    }
+
     auto const size = Message::kHeaderBytes + Message::size(buffers);
-    if (boost::asio::buffer_size(buffers) < size)
+
+    if (bs < size)
         return result;
 
+    auto const type = Message::type(buffers);
+
     switch (type)
     {
     case protocol::mtHELLO:         ec = detail::invoke<protocol::TMHello> (type, buffers, handler); break;