From 1e45d4120c687346e24fc374a849ec094167eda3 Mon Sep 17 00:00:00 2001 From: RichardAH Date: Thu, 17 Oct 2024 08:37:38 +1000 Subject: [PATCH 1/5] Update to boost186 (#377) Co-Authored-By: Denis Angell --- Builds/CMake/deps/Boost.cmake | 2 +- build-full.sh | 12 ++++++------ src/ripple/app/reporting/ETLSource.cpp | 8 +++++++- src/ripple/basics/Log.h | 1 + src/ripple/basics/impl/FileUtilities.cpp | 6 ++++-- src/ripple/net/impl/DatabaseBody.ipp | 1 + src/ripple/overlay/impl/Handshake.cpp | 17 +++++++++-------- src/ripple/overlay/impl/PeerImp.cpp | 12 ++++++------ src/ripple/peerfinder/impl/Checker.h | 1 - src/ripple/rpc/impl/ServerHandlerImp.cpp | 4 ++-- src/ripple/rpc/impl/WSInfoSub.h | 2 +- src/test/app/LedgerLoad_test.cpp | 2 +- src/test/jtx/TrustedPublisherServer.h | 6 ++++-- src/test/server/ServerStatus_test.cpp | 4 ++-- 14 files changed, 45 insertions(+), 33 deletions(-) diff --git a/Builds/CMake/deps/Boost.cmake b/Builds/CMake/deps/Boost.cmake index ba3086c5c..5038234bc 100644 --- a/Builds/CMake/deps/Boost.cmake +++ b/Builds/CMake/deps/Boost.cmake @@ -44,7 +44,7 @@ else() endif() # TBD: # Boost_USE_DEBUG_RUNTIME: When ON, uses Boost libraries linked against the -find_package(Boost 1.70 REQUIRED +find_package(Boost 1.86 REQUIRED COMPONENTS chrono container diff --git a/build-full.sh b/build-full.sh index 4731b6649..4ea18fea0 100755 --- a/build-full.sh +++ b/build-full.sh @@ -90,11 +90,11 @@ echo "-- Install Cmake 3.23.1 --" && pwd && ( wget -nc -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) && tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ && -echo "-- Install Boost 1.75.0 --" && +echo "-- Install Boost 1.86.0 --" && pwd && -( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) && -tar -xzf boost_1_75_0.tar.gz && -cd boost_1_75_0 && ./bootstrap.sh && ./b2 link=static -j$3 && ./b2 install && +( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.86.0/source/boost_1_86_0.tar.gz; echo "" ) && +tar -xzf boost_1_86_0.tar.gz && +cd boost_1_86_0 && ./bootstrap.sh && ./b2 link=static -j$3 && ./b2 install && cd ../ && echo "-- Install Protobuf 3.20.0 --" && pwd && @@ -127,9 +127,9 @@ echo "-- Build WasmEdge --" && cd WasmEdge-0.11.2 && ( mkdir build; echo "" ) && cd build && -export BOOST_ROOT="/usr/local/src/boost_1_75_0" && +export BOOST_ROOT="/usr/local/src/boost_1_86_0" && export Boost_LIBRARY_DIRS="/usr/local/lib" && -export BOOST_INCLUDEDIR="/usr/local/src/boost_1_75_0" && +export BOOST_INCLUDEDIR="/usr/local/src/boost_1_86_0" && export PATH=`echo $PATH | sed -E "s/devtoolset-7/devtoolset-9/g"` && cmake .. \ -DCMAKE_BUILD_TYPE=Release \ diff --git a/src/ripple/app/reporting/ETLSource.cpp b/src/ripple/app/reporting/ETLSource.cpp index be31f4fdf..41a2b143d 100644 --- a/src/ripple/app/reporting/ETLSource.cpp +++ b/src/ripple/app/reporting/ETLSource.cpp @@ -194,8 +194,14 @@ ETLSource::onResolve( { boost::beast::get_lowest_layer(*ws_).expires_after( std::chrono::seconds(30)); + + // Use async_connect with the entire results boost::beast::get_lowest_layer(*ws_).async_connect( - results, [this](auto ec, auto ep) { onConnect(ec, ep); }); + results, + [this]( + boost::beast::error_code ec, + boost::asio::ip::tcp::resolver::results_type::endpoint_type + ep) { onConnect(ec, ep); }); } } diff --git a/src/ripple/basics/Log.h b/src/ripple/basics/Log.h index 929225c04..6722ab813 100644 --- a/src/ripple/basics/Log.h +++ b/src/ripple/basics/Log.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include diff --git a/src/ripple/basics/impl/FileUtilities.cpp b/src/ripple/basics/impl/FileUtilities.cpp index be60a3e5c..d94f588f1 100644 --- a/src/ripple/basics/impl/FileUtilities.cpp +++ b/src/ripple/basics/impl/FileUtilities.cpp @@ -18,6 +18,7 @@ //============================================================================== #include +#include namespace ripple { @@ -41,7 +42,7 @@ getFileContents( return {}; } - ifstream fileStream(fullPath, std::ios::in); + std::ifstream fileStream(fullPath.string(), std::ios::in); if (!fileStream) { @@ -71,7 +72,8 @@ writeFileContents( using namespace boost::filesystem; using namespace boost::system::errc; - ofstream fileStream(destPath, std::ios::out | std::ios::trunc); + std::ofstream fileStream( + destPath.string(), std::ios::out | std::ios::trunc); if (!fileStream) { diff --git a/src/ripple/net/impl/DatabaseBody.ipp b/src/ripple/net/impl/DatabaseBody.ipp index cdc7da2bc..a43b0c721 100644 --- a/src/ripple/net/impl/DatabaseBody.ipp +++ b/src/ripple/net/impl/DatabaseBody.ipp @@ -18,6 +18,7 @@ //============================================================================== #include +#include namespace ripple { diff --git a/src/ripple/overlay/impl/Handshake.cpp b/src/ripple/overlay/impl/Handshake.cpp index ebb9b2716..937570888 100644 --- a/src/ripple/overlay/impl/Handshake.cpp +++ b/src/ripple/overlay/impl/Handshake.cpp @@ -44,7 +44,7 @@ getFeatureValue( return {}; boost::smatch match; boost::regex rx(feature + "=([^;\\s]+)"); - auto const value = header->value().to_string(); + auto const value = std::string(header->value()); if (boost::regex_search(value, match, rx)) return {match[1]}; return {}; @@ -233,7 +233,7 @@ verifyHandshake( { if (auto const iter = headers.find("Server-Domain"); iter != headers.end()) { - if (!isProperlyFormedTomlDomain(iter->value().to_string())) + if (!isProperlyFormedTomlDomain(std::string(iter->value()))) throw std::runtime_error("Invalid server domain"); } @@ -243,7 +243,8 @@ verifyHandshake( uint32_t peer_nid = 0; if (auto const iter = headers.find("Network-ID"); iter != headers.end()) { - if (!beast::lexicalCastChecked(peer_nid, iter->value().to_string())) + if (!beast::lexicalCastChecked( + peer_nid, std::string(iter->value()))) throw std::runtime_error("Invalid peer network identifier"); } @@ -255,7 +256,7 @@ verifyHandshake( if (auto const iter = headers.find("Network-Time"); iter != headers.end()) { auto const netTime = - [str = iter->value().to_string()]() -> TimeKeeper::time_point { + [str = std::string(iter->value())]() -> TimeKeeper::time_point { TimeKeeper::duration::rep val; if (beast::lexicalCastChecked(val, str)) @@ -291,7 +292,7 @@ verifyHandshake( if (auto const iter = headers.find("Public-Key"); iter != headers.end()) { auto pk = parseBase58( - TokenType::NodePublic, iter->value().to_string()); + TokenType::NodePublic, std::string(iter->value())); if (pk) { @@ -317,7 +318,7 @@ verifyHandshake( if (iter == headers.end()) throw std::runtime_error("No session signature specified"); - auto sig = base64_decode(iter->value().to_string()); + auto sig = base64_decode(std::string(iter->value())); if (!verifyDigest(publicKey, sharedValue, makeSlice(sig), false)) throw std::runtime_error("Failed to verify session"); @@ -330,7 +331,7 @@ verifyHandshake( { boost::system::error_code ec; auto const local_ip = boost::asio::ip::address::from_string( - iter->value().to_string(), ec); + std::string(iter->value()), ec); if (ec) throw std::runtime_error("Invalid Local-IP"); @@ -345,7 +346,7 @@ verifyHandshake( { boost::system::error_code ec; auto const remote_ip = boost::asio::ip::address::from_string( - iter->value().to_string(), ec); + std::string(iter->value()), ec); if (ec) throw std::runtime_error("Invalid Remote-IP"); diff --git a/src/ripple/overlay/impl/PeerImp.cpp b/src/ripple/overlay/impl/PeerImp.cpp index 85844f342..adb05ddf6 100644 --- a/src/ripple/overlay/impl/PeerImp.cpp +++ b/src/ripple/overlay/impl/PeerImp.cpp @@ -176,7 +176,7 @@ PeerImp::run() if (auto const iter = headers_.find("Closed-Ledger"); iter != headers_.end()) { - closed = parseLedgerHash(iter->value().to_string()); + closed = parseLedgerHash(std::string(iter->value())); if (!closed) fail("Malformed handshake data (1)"); @@ -185,7 +185,7 @@ PeerImp::run() if (auto const iter = headers_.find("Previous-Ledger"); iter != headers_.end()) { - previous = parseLedgerHash(iter->value().to_string()); + previous = parseLedgerHash(std::string(iter->value())); if (!previous) fail("Malformed handshake data (2)"); @@ -372,8 +372,8 @@ std::string PeerImp::getVersion() const { if (inbound_) - return headers_["User-Agent"].to_string(); - return headers_["Server"].to_string(); + return std::string(headers_["User-Agent"]); + return std::string(headers_["Server"]); } Json::Value @@ -399,7 +399,7 @@ PeerImp::json() if (auto const d = domain(); !d.empty()) ret[jss::server_domain] = domain(); - if (auto const nid = headers_["Network-ID"].to_string(); !nid.empty()) + if (auto const nid = std::string(headers_["Network-ID"]); !nid.empty()) ret[jss::network_id] = nid; ret[jss::load] = usage_.balance(); @@ -839,7 +839,7 @@ PeerImp::name() const std::string PeerImp::domain() const { - return headers_["Server-Domain"].to_string(); + return std::string(headers_["Server-Domain"]); } //------------------------------------------------------------------------------ diff --git a/src/ripple/peerfinder/impl/Checker.h b/src/ripple/peerfinder/impl/Checker.h index 3a5ef9013..e913d4c09 100644 --- a/src/ripple/peerfinder/impl/Checker.h +++ b/src/ripple/peerfinder/impl/Checker.h @@ -21,7 +21,6 @@ #define RIPPLE_PEERFINDER_CHECKER_H_INCLUDED #include -#include #include #include #include diff --git a/src/ripple/rpc/impl/ServerHandlerImp.cpp b/src/ripple/rpc/impl/ServerHandlerImp.cpp index f269283b8..81075a5c0 100644 --- a/src/ripple/rpc/impl/ServerHandlerImp.cpp +++ b/src/ripple/rpc/impl/ServerHandlerImp.cpp @@ -247,11 +247,11 @@ build_map(boost::beast::http::fields const& h) std::map c; for (auto const& e : h) { - auto key(e.name_string().to_string()); + auto key(std::string(e.name_string())); std::transform(key.begin(), key.end(), key.begin(), [](auto kc) { return std::tolower(static_cast(kc)); }); - c[key] = e.value().to_string(); + c[key] = std::string(e.value()); } return c; } diff --git a/src/ripple/rpc/impl/WSInfoSub.h b/src/ripple/rpc/impl/WSInfoSub.h index 8f386c8eb..a1f2f5bb4 100644 --- a/src/ripple/rpc/impl/WSInfoSub.h +++ b/src/ripple/rpc/impl/WSInfoSub.h @@ -50,7 +50,7 @@ public: { auto it = h.find("X-User"); if (it != h.end()) - user_ = it->value().to_string(); + user_ = std::string(it->value()); fwdfor_ = std::string(forwardedFor(h)); } } diff --git a/src/test/app/LedgerLoad_test.cpp b/src/test/app/LedgerLoad_test.cpp index 2685014e4..93fc002ae 100644 --- a/src/test/app/LedgerLoad_test.cpp +++ b/src/test/app/LedgerLoad_test.cpp @@ -154,7 +154,7 @@ class LedgerLoad_test : public beast::unit_test::suite copy_file( sd.ledgerFile, ledgerFileCorrupt, - copy_option::overwrite_if_exists, + copy_options::overwrite_existing, ec); if (!BEAST_EXPECTS(!ec, ec.message())) return; diff --git a/src/test/jtx/TrustedPublisherServer.h b/src/test/jtx/TrustedPublisherServer.h index 985c1bfd6..266f8d922 100644 --- a/src/test/jtx/TrustedPublisherServer.h +++ b/src/test/jtx/TrustedPublisherServer.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -574,7 +575,7 @@ private: if (ec) break; - auto path = req.target().to_string(); + auto path = req.target(); //.to_string(); res.insert("Server", "TrustedPublisherServer"); res.version(req.version()); res.keep_alive(req.keep_alive()); @@ -677,7 +678,8 @@ private: // unknown request res.result(boost::beast::http::status::not_found); res.insert("Content-Type", "text/html"); - res.body() = "The file '" + path + "' was not found"; + res.body() = + "The file '" + std::string(path) + "' was not found"; } if (prepare) diff --git a/src/test/server/ServerStatus_test.cpp b/src/test/server/ServerStatus_test.cpp index 684532c4f..29bf26980 100644 --- a/src/test/server/ServerStatus_test.cpp +++ b/src/test/server/ServerStatus_test.cpp @@ -674,10 +674,10 @@ class ServerStatus_test : public beast::unit_test::suite, resp.result() == boost::beast::http::status::switching_protocols); BEAST_EXPECT( resp.find("Upgrade") != resp.end() && - resp["Upgrade"] == "websocket"); + std::string(resp["Upgrade"]) == "websocket"); BEAST_EXPECT( resp.find("Connection") != resp.end() && - resp["Connection"] == "upgrade"); + std::string(resp["Connection"]) == "Upgrade"); } void From 4239124750cc7cfa10a29cc7ab7233474d237373 Mon Sep 17 00:00:00 2001 From: tequ Date: Fri, 25 Oct 2024 08:10:45 +0900 Subject: [PATCH 2/5] Update amendments for rippled-standalone.cfg (#385) --- cfg/rippled-standalone.cfg | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cfg/rippled-standalone.cfg b/cfg/rippled-standalone.cfg index 79c5c9004..3301299ca 100755 --- a/cfg/rippled-standalone.cfg +++ b/cfg/rippled-standalone.cfg @@ -144,4 +144,12 @@ D686F2538F410C9D0D856788E98E3579595DAF7B38D38887F81ECAC934B06040 HooksUpdate1 86E83A7D2ECE3AD5FA87AB2195AE015C950469ABF0B72EAACED318F74886AE90 CryptoConditionsSuite 3C43D9A973AA4443EF3FC38E42DD306160FBFFDAB901CD8BAA15D09F2597EB87 NonFungibleTokensV1 0285B7E5E08E1A8E4C15636F0591D87F73CB6A7B6452A932AD72BBC8E5D1CBE3 fixNFTokenDirV1 -36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer \ No newline at end of file +36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer +4C499D17719BB365B69010A436B64FD1A82AAB199FC1CEB06962EBD01059FB09 fixXahauV1 +215181D23BF5C173314B5FDB9C872C92DE6CC918483727DE037C0C13E7E6EE9D fixXahauV2 +0D8BF22FF7570D58598D1EF19EBB6E142AD46E59A223FD3816262FBB69345BEA Remit +7CA0426E7F411D39BB014E57CD9E08F61DE1750F0D41FCD428D9FB80BB7596B0 ZeroB2M +4B8466415FAB32FFA89D9DCBE166A42340115771DF611A7160F8D7439C87ECD8 fixNSDelete +EDB4EE4C524E16BDD91D9A529332DED08DCAAA51CC6DC897ACFA1A0ED131C5B6 fix240819 +8063140E9260799D6716756B891CEC3E7006C4E4F277AB84670663A88F94B9C4 fixPageCap +88693F108C3CD8A967F3F4253A32DEF5E35F9406ACD2A11B88B11D90865763A9 fix240911 From 287c01ad0427410b81ede73946eee8e1dc1a9c67 Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Fri, 25 Oct 2024 02:10:14 +0200 Subject: [PATCH 3/5] Improve Admin command RPC Post (#384) * Improve ADMIN HTTP POST RPC notifications: no queue limit, shorter HTTP call TTL --- src/ripple/net/impl/RPCCall.cpp | 4 +++- src/ripple/net/impl/RPCSub.cpp | 17 ++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/ripple/net/impl/RPCCall.cpp b/src/ripple/net/impl/RPCCall.cpp index f11f831e0..cb63d997c 100644 --- a/src/ripple/net/impl/RPCCall.cpp +++ b/src/ripple/net/impl/RPCCall.cpp @@ -1894,7 +1894,9 @@ fromNetwork( constexpr auto RPC_REPLY_MAX_BYTES = megabytes(256); using namespace std::chrono_literals; - auto constexpr RPC_NOTIFY = 10min; + // auto constexpr RPC_NOTIFY = 10min; // Wietse: lolwut 10 minutes for one + // HTTP call? + auto constexpr RPC_NOTIFY = 30s; HTTPClient::request( bSSL, diff --git a/src/ripple/net/impl/RPCSub.cpp b/src/ripple/net/impl/RPCSub.cpp index 8b052e817..5423a1132 100644 --- a/src/ripple/net/impl/RPCSub.cpp +++ b/src/ripple/net/impl/RPCSub.cpp @@ -78,12 +78,14 @@ public: { std::lock_guard sl(mLock); - if (mDeque.size() >= eventQueueMax) - { - // Drop the previous event. - JLOG(j_.warn()) << "RPCCall::fromNetwork drop"; - mDeque.pop_back(); - } + // Wietse: we're not going to limit this, this is admin-port only, scale + // accordingly Dropping events just like this results in inconsistent + // data on the receiving end if (mDeque.size() >= eventQueueMax) + // { + // // Drop the previous event. + // JLOG(j_.warn()) << "RPCCall::fromNetwork drop"; + // mDeque.pop_back(); + // } auto jm = broadcast ? j_.debug() : j_.info(); JLOG(jm) << "RPCCall::fromNetwork push: " << jvObj; @@ -182,7 +184,8 @@ private: } private: - enum { eventQueueMax = 32 }; + // Wietse: we're not going to limit this, this is admin-port only, scale + // accordingly enum { eventQueueMax = 32 }; boost::asio::io_service& m_io_service; JobQueue& m_jobQueue; From 766f5d7ee1d04085823e7b8e30f30a8977ea7ec9 Mon Sep 17 00:00:00 2001 From: tequ Date: Fri, 25 Oct 2024 09:10:43 +0900 Subject: [PATCH 4/5] Update macro.h (#366) --- hook/macro.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hook/macro.h b/hook/macro.h index 38a534bfc..838c2b55c 100644 --- a/hook/macro.h +++ b/hook/macro.h @@ -1,5 +1,5 @@ /** - * These are helper macros for writing hooks, all of them are optional as is including hookmacro.h at all + * These are helper macros for writing hooks, all of them are optional as is including macro.h at all */ #include From 08f13b7cfeed4ccdf6d7aba1dbd08feac65c73ee Mon Sep 17 00:00:00 2001 From: Wietse Wind Date: Fri, 25 Oct 2024 03:13:42 +0200 Subject: [PATCH 5/5] Fix account_tx sluggishness as per https://github.com/XRPLF/rippled/commit/2e9261cb (#308) --- src/ripple/app/rdb/backend/detail/impl/Node.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ripple/app/rdb/backend/detail/impl/Node.cpp b/src/ripple/app/rdb/backend/detail/impl/Node.cpp index 11c43191a..c80038ef7 100644 --- a/src/ripple/app/rdb/backend/detail/impl/Node.cpp +++ b/src/ripple/app/rdb/backend/detail/impl/Node.cpp @@ -1132,7 +1132,7 @@ accountTxPage( { sql = boost::str( boost::format( - prefix + (R"(AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u' + prefix + (R"(AccountTransactions.LedgerSeq BETWEEN %u AND %u ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s LIMIT %u;)")) % @@ -1155,12 +1155,14 @@ accountTxPage( FROM AccountTransactions, Transactions WHERE (AccountTransactions.TransID = Transactions.TransID AND AccountTransactions.Account = '%s' AND - AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u') - OR + AccountTransactions.LedgerSeq BETWEEN %u AND %u) + UNION + SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,Status,RawTxn,TxnMeta + FROM AccountTransactions, Transactions WHERE (AccountTransactions.TransID = Transactions.TransID AND AccountTransactions.Account = '%s' AND - AccountTransactions.LedgerSeq = '%u' AND - AccountTransactions.TxnSeq %s '%u') + AccountTransactions.LedgerSeq = %u AND + AccountTransactions.TxnSeq %s %u) ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s LIMIT %u;