Merge branch 'dev' into fix/rpc-queue-limit

Co-Authored-By: Denis Angell <dangell@transia.co>

Builds/CMake/deps/Boost.cmake

@@ -44,7 +44,7 @@ else()
 endif()
 # TBD:
 # Boost_USE_DEBUG_RUNTIME:  When ON, uses Boost libraries linked against the
-find_package(Boost 1.70 REQUIRED
+find_package(Boost 1.86 REQUIRED
   COMPONENTS
     chrono
     container

build-full.sh

@@ -90,11 +90,11 @@ echo "-- Install Cmake 3.23.1 --" &&
 pwd &&
 ( wget -nc -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) &&
 tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ &&
-echo "-- Install Boost 1.75.0 --" &&
+echo "-- Install Boost 1.86.0 --" &&
 pwd &&
-( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) &&
-tar -xzf boost_1_75_0.tar.gz &&
-cd boost_1_75_0 && ./bootstrap.sh && ./b2  link=static -j$3 && ./b2 install &&
+( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.86.0/source/boost_1_86_0.tar.gz; echo "" ) &&
+tar -xzf boost_1_86_0.tar.gz &&
+cd boost_1_86_0 && ./bootstrap.sh && ./b2  link=static -j$3 && ./b2 install &&
 cd ../ &&
 echo "-- Install Protobuf 3.20.0 --" &&
 pwd &&
@@ -127,9 +127,9 @@ echo "-- Build WasmEdge --" &&
 cd WasmEdge-0.11.2 &&
 ( mkdir build; echo "" ) &&
 cd build &&
-export BOOST_ROOT="/usr/local/src/boost_1_75_0" &&
+export BOOST_ROOT="/usr/local/src/boost_1_86_0" &&
 export Boost_LIBRARY_DIRS="/usr/local/lib" &&
-export BOOST_INCLUDEDIR="/usr/local/src/boost_1_75_0" &&
+export BOOST_INCLUDEDIR="/usr/local/src/boost_1_86_0" &&
 export PATH=`echo $PATH | sed -E "s/devtoolset-7/devtoolset-9/g"` &&
 cmake .. \
     -DCMAKE_BUILD_TYPE=Release \

cfg/rippled-standalone.cfg

@@ -144,4 +144,12 @@ D686F2538F410C9D0D856788E98E3579595DAF7B38D38887F81ECAC934B06040 HooksUpdate1
 86E83A7D2ECE3AD5FA87AB2195AE015C950469ABF0B72EAACED318F74886AE90 CryptoConditionsSuite
 3C43D9A973AA4443EF3FC38E42DD306160FBFFDAB901CD8BAA15D09F2597EB87 NonFungibleTokensV1
 0285B7E5E08E1A8E4C15636F0591D87F73CB6A7B6452A932AD72BBC8E5D1CBE3 fixNFTokenDirV1
-36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer
+36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer
+4C499D17719BB365B69010A436B64FD1A82AAB199FC1CEB06962EBD01059FB09 fixXahauV1
+215181D23BF5C173314B5FDB9C872C92DE6CC918483727DE037C0C13E7E6EE9D fixXahauV2
+0D8BF22FF7570D58598D1EF19EBB6E142AD46E59A223FD3816262FBB69345BEA Remit
+7CA0426E7F411D39BB014E57CD9E08F61DE1750F0D41FCD428D9FB80BB7596B0 ZeroB2M
+4B8466415FAB32FFA89D9DCBE166A42340115771DF611A7160F8D7439C87ECD8 fixNSDelete
+EDB4EE4C524E16BDD91D9A529332DED08DCAAA51CC6DC897ACFA1A0ED131C5B6 fix240819
+8063140E9260799D6716756B891CEC3E7006C4E4F277AB84670663A88F94B9C4 fixPageCap
+88693F108C3CD8A967F3F4253A32DEF5E35F9406ACD2A11B88B11D90865763A9 fix240911

src/ripple/app/reporting/ETLSource.cpp

@@ -194,8 +194,14 @@ ETLSource::onResolve(
     {
         boost::beast::get_lowest_layer(*ws_).expires_after(
             std::chrono::seconds(30));
+
+        // Use async_connect with the entire results
         boost::beast::get_lowest_layer(*ws_).async_connect(
-            results, [this](auto ec, auto ep) { onConnect(ec, ep); });
+            results,
+            [this](
+                boost::beast::error_code ec,
+                boost::asio::ip::tcp::resolver::results_type::endpoint_type
+                    ep) { onConnect(ec, ep); });
     }
 }
 

src/ripple/basics/Log.h

@@ -24,6 +24,7 @@
 #include <ripple/beast/utility/Journal.h>
 #include <boost/beast/core/string.hpp>
 #include <boost/filesystem.hpp>
+#include <fstream>
 #include <map>
 #include <memory>
 #include <mutex>

src/ripple/basics/impl/FileUtilities.cpp

@@ -18,6 +18,7 @@
 //==============================================================================
 
 #include <ripple/basics/FileUtilities.h>
+#include <fstream>
 
 namespace ripple {
 
@@ -41,7 +42,7 @@ getFileContents(
         return {};
     }
 
-    ifstream fileStream(fullPath, std::ios::in);
+    std::ifstream fileStream(fullPath.string(), std::ios::in);
 
     if (!fileStream)
     {
@@ -71,7 +72,8 @@ writeFileContents(
     using namespace boost::filesystem;
     using namespace boost::system::errc;
 
-    ofstream fileStream(destPath, std::ios::out | std::ios::trunc);
+    std::ofstream fileStream(
+        destPath.string(), std::ios::out | std::ios::trunc);
 
     if (!fileStream)
     {

src/ripple/net/impl/DatabaseBody.ipp

@@ -18,6 +18,7 @@
 //==============================================================================
 
 #include <ripple/app/rdb/Download.h>
+#include <fstream>
 
 namespace ripple {
 

src/ripple/net/impl/RPCCall.cpp

@@ -1894,7 +1894,9 @@ fromNetwork(
     constexpr auto RPC_REPLY_MAX_BYTES = megabytes(256);
 
     using namespace std::chrono_literals;
-    auto constexpr RPC_NOTIFY = 10min;
+    // auto constexpr RPC_NOTIFY = 10min; // Wietse: lolwut 10 minutes for one
+    // HTTP call?
+    auto constexpr RPC_NOTIFY = 30s;
 
     HTTPClient::request(
         bSSL,

src/ripple/net/impl/RPCSub.cpp

@@ -78,12 +78,14 @@ public:
     {
         std::lock_guard sl(mLock);
 
-        if (mDeque.size() >= eventQueueMax)
-        {
-            // Drop the previous event.
-            JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
-            mDeque.pop_back();
-        }
+        // Wietse: we're not going to limit this, this is admin-port only, scale
+        // accordingly Dropping events just like this results in inconsistent
+        // data on the receiving end if (mDeque.size() >= eventQueueMax)
+        // {
+        //     // Drop the previous event.
+        //     JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
+        //     mDeque.pop_back();
+        // }
 
         auto jm = broadcast ? j_.debug() : j_.info();
         JLOG(jm) << "RPCCall::fromNetwork push: " << jvObj;
@@ -182,7 +184,8 @@ private:
     }
 
 private:
-    enum { eventQueueMax = 32 };
+    // Wietse: we're not going to limit this, this is admin-port only, scale
+    // accordingly enum { eventQueueMax = 32 };
 
     boost::asio::io_service& m_io_service;
     JobQueue& m_jobQueue;

src/ripple/overlay/impl/Handshake.cpp

@@ -44,7 +44,7 @@ getFeatureValue(
         return {};
     boost::smatch match;
     boost::regex rx(feature + "=([^;\\s]+)");
-    auto const value = header->value().to_string();
+    auto const value = std::string(header->value());
     if (boost::regex_search(value, match, rx))
         return {match[1]};
     return {};
@@ -233,7 +233,7 @@ verifyHandshake(
 {
     if (auto const iter = headers.find("Server-Domain"); iter != headers.end())
     {
-        if (!isProperlyFormedTomlDomain(iter->value().to_string()))
+        if (!isProperlyFormedTomlDomain(std::string(iter->value())))
             throw std::runtime_error("Invalid server domain");
     }
 
@@ -243,7 +243,8 @@ verifyHandshake(
         uint32_t peer_nid = 0;
         if (auto const iter = headers.find("Network-ID"); iter != headers.end())
         {
-            if (!beast::lexicalCastChecked(peer_nid, iter->value().to_string()))
+            if (!beast::lexicalCastChecked(
+                    peer_nid, std::string(iter->value())))
                 throw std::runtime_error("Invalid peer network identifier");
         }
 
@@ -255,7 +256,7 @@ verifyHandshake(
     if (auto const iter = headers.find("Network-Time"); iter != headers.end())
     {
         auto const netTime =
-            [str = iter->value().to_string()]() -> TimeKeeper::time_point {
+            [str = std::string(iter->value())]() -> TimeKeeper::time_point {
             TimeKeeper::duration::rep val;
 
             if (beast::lexicalCastChecked(val, str))
@@ -291,7 +292,7 @@ verifyHandshake(
         if (auto const iter = headers.find("Public-Key"); iter != headers.end())
         {
             auto pk = parseBase58<PublicKey>(
-                TokenType::NodePublic, iter->value().to_string());
+                TokenType::NodePublic, std::string(iter->value()));
 
             if (pk)
             {
@@ -317,7 +318,7 @@ verifyHandshake(
         if (iter == headers.end())
             throw std::runtime_error("No session signature specified");
 
-        auto sig = base64_decode(iter->value().to_string());
+        auto sig = base64_decode(std::string(iter->value()));
 
         if (!verifyDigest(publicKey, sharedValue, makeSlice(sig), false))
             throw std::runtime_error("Failed to verify session");
@@ -330,7 +331,7 @@ verifyHandshake(
     {
         boost::system::error_code ec;
         auto const local_ip = boost::asio::ip::address::from_string(
-            iter->value().to_string(), ec);
+            std::string(iter->value()), ec);
 
         if (ec)
             throw std::runtime_error("Invalid Local-IP");
@@ -345,7 +346,7 @@ verifyHandshake(
     {
         boost::system::error_code ec;
         auto const remote_ip = boost::asio::ip::address::from_string(
-            iter->value().to_string(), ec);
+            std::string(iter->value()), ec);
 
         if (ec)
             throw std::runtime_error("Invalid Remote-IP");

src/ripple/overlay/impl/PeerImp.cpp

@@ -176,7 +176,7 @@ PeerImp::run()
     if (auto const iter = headers_.find("Closed-Ledger");
         iter != headers_.end())
     {
-        closed = parseLedgerHash(iter->value().to_string());
+        closed = parseLedgerHash(std::string(iter->value()));
 
         if (!closed)
             fail("Malformed handshake data (1)");
@@ -185,7 +185,7 @@ PeerImp::run()
     if (auto const iter = headers_.find("Previous-Ledger");
         iter != headers_.end())
     {
-        previous = parseLedgerHash(iter->value().to_string());
+        previous = parseLedgerHash(std::string(iter->value()));
 
         if (!previous)
             fail("Malformed handshake data (2)");
@@ -372,8 +372,8 @@ std::string
 PeerImp::getVersion() const
 {
     if (inbound_)
-        return headers_["User-Agent"].to_string();
-    return headers_["Server"].to_string();
+        return std::string(headers_["User-Agent"]);
+    return std::string(headers_["Server"]);
 }
 
 Json::Value
@@ -399,7 +399,7 @@ PeerImp::json()
     if (auto const d = domain(); !d.empty())
         ret[jss::server_domain] = domain();
 
-    if (auto const nid = headers_["Network-ID"].to_string(); !nid.empty())
+    if (auto const nid = std::string(headers_["Network-ID"]); !nid.empty())
         ret[jss::network_id] = nid;
 
     ret[jss::load] = usage_.balance();
@@ -839,7 +839,7 @@ PeerImp::name() const
 std::string
 PeerImp::domain() const
 {
-    return headers_["Server-Domain"].to_string();
+    return std::string(headers_["Server-Domain"]);
 }
 
 //------------------------------------------------------------------------------

src/ripple/peerfinder/impl/Checker.h

@@ -21,7 +21,6 @@
 #define RIPPLE_PEERFINDER_CHECKER_H_INCLUDED
 
 #include <ripple/beast/net/IPAddressConversion.h>
-#include <boost/asio/detail/handler_invoke_helpers.hpp>
 #include <boost/asio/io_service.hpp>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/intrusive/list.hpp>

src/ripple/rpc/impl/ServerHandlerImp.cpp

@@ -247,11 +247,11 @@ build_map(boost::beast::http::fields const& h)
     std::map<std::string, std::string> c;
     for (auto const& e : h)
     {
-        auto key(e.name_string().to_string());
+        auto key(std::string(e.name_string()));
         std::transform(key.begin(), key.end(), key.begin(), [](auto kc) {
             return std::tolower(static_cast<unsigned char>(kc));
         });
-        c[key] = e.value().to_string();
+        c[key] = std::string(e.value());
     }
     return c;
 }

src/ripple/rpc/impl/WSInfoSub.h

@@ -50,7 +50,7 @@ public:
         {
             auto it = h.find("X-User");
             if (it != h.end())
-                user_ = it->value().to_string();
+                user_ = std::string(it->value());
             fwdfor_ = std::string(forwardedFor(h));
         }
     }

src/test/app/LedgerLoad_test.cpp

@@ -154,7 +154,7 @@ class LedgerLoad_test : public beast::unit_test::suite
         copy_file(
             sd.ledgerFile,
             ledgerFileCorrupt,
-            copy_option::overwrite_if_exists,
+            copy_options::overwrite_existing,
             ec);
         if (!BEAST_EXPECTS(!ec, ec.message()))
             return;

src/test/jtx/TrustedPublisherServer.h

@@ -29,6 +29,7 @@
 #include <boost/asio.hpp>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/ssl/stream.hpp>
+#include <boost/beast/core/flat_buffer.hpp>
 #include <boost/beast/http.hpp>
 #include <boost/beast/ssl.hpp>
 #include <boost/beast/version.hpp>
@@ -574,7 +575,7 @@ private:
                 if (ec)
                     break;
 
-                auto path = req.target().to_string();
+                auto path = req.target();  //.to_string();
                 res.insert("Server", "TrustedPublisherServer");
                 res.version(req.version());
                 res.keep_alive(req.keep_alive());
@@ -677,7 +678,8 @@ private:
                     // unknown request
                     res.result(boost::beast::http::status::not_found);
                     res.insert("Content-Type", "text/html");
-                    res.body() = "The file '" + path + "' was not found";
+                    res.body() =
+                        "The file '" + std::string(path) + "' was not found";
                 }
 
                 if (prepare)

src/test/server/ServerStatus_test.cpp

@@ -674,10 +674,10 @@ class ServerStatus_test : public beast::unit_test::suite,
             resp.result() == boost::beast::http::status::switching_protocols);
         BEAST_EXPECT(
             resp.find("Upgrade") != resp.end() &&
-            resp["Upgrade"] == "websocket");
+            std::string(resp["Upgrade"]) == "websocket");
         BEAST_EXPECT(
             resp.find("Connection") != resp.end() &&
-            resp["Connection"] == "upgrade");
+            std::string(resp["Connection"]) == "Upgrade");
     }
 
     void