diff --git a/CMakeLists.txt b/CMakeLists.txt index fc9786257..6befb16ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,9 @@ if(VERBOSE) set(FETCHCONTENT_QUIET FALSE CACHE STRING "Verbose FetchContent()") endif() +#c++20 removed std::result_of but boost 1.75 is still using it. +add_definitions(-DBOOST_ASIO_HAS_STD_INVOKE_RESULT=1) + add_library(clio) target_compile_features(clio PUBLIC cxx_std_20) target_include_directories(clio PUBLIC src) diff --git a/src/backend/BackendInterface.h b/src/backend/BackendInterface.h index 2634650a7..e70d33079 100644 --- a/src/backend/BackendInterface.h +++ b/src/backend/BackendInterface.h @@ -111,7 +111,7 @@ synchronous(F&& f) * R is the currently executing coroutine that is about to get passed. * If corountine types do not match, the current one's type is stored. */ - using R = typename std::result_of::type; + using R = typename boost::result_of::type; if constexpr (!std::is_same::value) { /** diff --git a/src/main/main.cpp b/src/main/main.cpp index 859ff2092..9afc51c22 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -12,9 +12,10 @@ static std::uint32_t const MAX_RETRIES = 5; static std::chrono::seconds const WAIT_TIME = std::chrono::seconds(60); +static std::uint32_t const NFT_WRITE_BATCH_SIZE = 10000; static void -wait(boost::asio::steady_timer& timer, std::string const reason) +wait(boost::asio::steady_timer& timer, std::string const& reason) { BOOST_LOG_TRIVIAL(info) << reason << ". Waiting"; timer.expires_after(WAIT_TIME); @@ -22,18 +23,31 @@ wait(boost::asio::steady_timer& timer, std::string const reason) BOOST_LOG_TRIVIAL(info) << "Done"; } -static void +static std::vector doNFTWrite( - std::vector& nfts, + std::vector&& nfts, Backend::CassandraBackend& backend, - std::string const tag) + std::string const& tag) { - if (nfts.size() <= 0) - return; + if (nfts.size() == 0) + return nfts; auto const size = nfts.size(); backend.writeNFTs(std::move(nfts)); backend.sync(); BOOST_LOG_TRIVIAL(info) << tag << ": Wrote " << size << " records"; + nfts.clear(); + return nfts; +} + +static std::vector +maybeDoNFTWrite( + std::vector&& nfts, + Backend::CassandraBackend& backend, + std::string const& tag) +{ + if (nfts.size() < NFT_WRITE_BATCH_SIZE) + return nfts; + return doNFTWrite(std::move(nfts), backend, tag); } static std::optional @@ -122,6 +136,8 @@ doMigration( return; } + std::vector toWrite; + /* * Step 1 - Look at all NFT transactions recorded in * `nf_token_transactions` and reload any NFTokenMint transactions. These @@ -140,8 +156,6 @@ doMigration( // For all NFT txs, paginated in groups of 1000... while (morePages) { - std::vector toWrite; - CassResult const* result = doTryGetTxPageResult(nftTxQuery, timer, backend); @@ -195,7 +209,7 @@ doMigration( std::get<1>(getNFTDataFromTx(txMeta, sttx)).value()); } - doNFTWrite(toWrite, backend, "TX"); + toWrite = maybeDoNFTWrite(std::move(toWrite), backend, "TX"); morePages = cass_result_has_more_pages(result); if (morePages) @@ -205,6 +219,7 @@ doMigration( } cass_statement_free(nftTxQuery); + toWrite = doNFTWrite(std::move(toWrite), backend, "TX"); BOOST_LOG_TRIVIAL(info) << "\nDone with transaction loading!\n"; /* @@ -216,21 +231,27 @@ doMigration( */ std::optional cursor; + // For each object page in initial ledger do { auto const page = doTryFetchLedgerPage( timer, backend, cursor, ledgerRange->minSequence, yield); + + // For each object in page of 2000 for (auto const& object : page.objects) { - std::vector toWrite = getNFTDataFromObj( + std::vector objectNFTs = getNFTDataFromObj( ledgerRange->minSequence, - ripple::to_string(object.key), + std::string(object.key.begin(), object.key.end()), std::string(object.blob.begin(), object.blob.end())); - doNFTWrite(toWrite, backend, "OBJ"); + toWrite.insert(toWrite.end(), objectNFTs.begin(), objectNFTs.end()); } + + toWrite = maybeDoNFTWrite(std::move(toWrite), backend, "OBJ"); cursor = page.cursor; } while (cursor.has_value()); + toWrite = doNFTWrite(std::move(toWrite), backend, "OBJ"); BOOST_LOG_TRIVIAL(info) << "\nDone with object loading!\n"; /* diff --git a/src/rpc/RPC.cpp b/src/rpc/RPC.cpp index 35ae1e731..32e13a986 100644 --- a/src/rpc/RPC.cpp +++ b/src/rpc/RPC.cpp @@ -178,7 +178,7 @@ public: { for (auto const& handler : handlers) { - handlerMap_[handler.method] = move(handler); + handlerMap_[handler.method] = std::move(handler); } }