From d4e2240126890acad53922a5d192e4e3d1a36b78 Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 01:39:20 -0400 Subject: [PATCH 1/4] attempt to speed up tx loading --- src/main/main.cpp | 74 ++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/src/main/main.cpp b/src/main/main.cpp index 6776c19fe..8b3a02a8c 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -17,10 +17,10 @@ static std::uint32_t const NFT_WRITE_BATCH_SIZE = 10000; static void wait(boost::asio::steady_timer& timer, std::string const& reason) { - BOOST_LOG_TRIVIAL(info) << reason << ". Waiting"; + BOOST_LOG_TRIVIAL(info) << reason << ". Waiting then retrying"; timer.expires_after(WAIT_TIME); timer.wait(); - BOOST_LOG_TRIVIAL(info) << "Done"; + BOOST_LOG_TRIVIAL(info) << "Done waiting"; } static std::vector @@ -49,25 +49,26 @@ maybeDoNFTWrite( return doNFTWrite(nfts, backend, tag); } -static std::optional -doTryFetchTransaction( +static std::vector +doTryFetchTransactions( boost::asio::steady_timer& timer, Backend::CassandraBackend& backend, - ripple::uint256 const& hash, + std::vector const& hashes, boost::asio::yield_context& yield, std::uint32_t const attempts = 0) { try { - return backend.fetchTransaction(hash, yield); + return backend.fetchTransactions(hashes, yield); } catch (Backend::DatabaseTimeout const& e) { if (attempts >= MAX_RETRIES) throw e; - wait(timer, "Transaction read error"); - return doTryFetchTransaction(timer, backend, hash, yield, attempts + 1); + wait(timer, "Transactions read error"); + return doTryFetchTransactions( + timer, backend, hashes, yield, attempts + 1); } } @@ -158,6 +159,8 @@ doMigration( CassResult const* result = doTryGetTxPageResult(nftTxQuery, timer, backend); + std::vector txHashes; + // For each tx in page... CassIterator* txPageIterator = cass_iterator_from_result(result); while (cass_iterator_next(txPageIterator)) @@ -178,36 +181,35 @@ doMigration( "Could not retrieve hash from nf_token_transactions"); } - auto const txHash = ripple::uint256::fromVoid(buf); - auto const tx = - doTryFetchTransaction(timer, backend, txHash, yield); - if (!tx) - { - cass_iterator_free(txPageIterator); - cass_result_free(result); - cass_statement_free(nftTxQuery); - std::stringstream ss; - ss << "Could not fetch tx with hash " - << ripple::to_string(txHash); - throw std::runtime_error(ss.str()); - } - - // Not really sure how cassandra paging works, but we want to skip - // any transactions that were loaded since the migration started - if (tx->ledgerSequence > ledgerRange->maxSequence) - continue; - - ripple::STTx const sttx{ripple::SerialIter{ - tx->transaction.data(), tx->transaction.size()}}; - if (sttx.getTxnType() != ripple::TxType::ttNFTOKEN_MINT) - continue; - - ripple::TxMeta const txMeta{ - sttx.getTransactionID(), tx->ledgerSequence, tx->metadata}; - toWrite.push_back( - std::get<1>(getNFTDataFromTx(txMeta, sttx)).value()); + txHashes.push_back(ripple::uint256::fromVoid(buf)); } + auto txs = doTryFetchTransactions(timer, backend, txHashes, yield); + txs.erase( + std::remove_if( + txs.begin(), + txs.end(), + [&ledgerRange](Backend::TransactionAndMetadata const& tx) { + if (tx.ledgerSequence > ledgerRange->maxSequence) + return true; + ripple::STTx const sttx{ripple::SerialIter{ + tx.transaction.data(), tx.transaction.size()}}; + return sttx.getTxnType() != ripple::TxType::ttNFTOKEN_MINT; + }), + txs.end()); + + std::transform( + txs.begin(), + txs.end(), + toWrite.end(), + [](Backend::TransactionAndMetadata const& tx) { + ripple::STTx const sttx{ripple::SerialIter{ + tx.transaction.data(), tx.transaction.size()}}; + ripple::TxMeta const txMeta{ + sttx.getTransactionID(), tx.ledgerSequence, tx.metadata}; + return std::get<1>(getNFTDataFromTx(txMeta, sttx)).value(); + }); + toWrite = maybeDoNFTWrite(toWrite, backend, "TX"); morePages = cass_result_has_more_pages(result); From a8641c1b3bb299ef47bd99c8d9508cbcaf41b73d Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 01:44:54 -0400 Subject: [PATCH 2/4] loop instead of transform --- src/main/main.cpp | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/main/main.cpp b/src/main/main.cpp index 8b3a02a8c..93398812a 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -198,17 +198,15 @@ doMigration( }), txs.end()); - std::transform( - txs.begin(), - txs.end(), - toWrite.end(), - [](Backend::TransactionAndMetadata const& tx) { - ripple::STTx const sttx{ripple::SerialIter{ - tx.transaction.data(), tx.transaction.size()}}; - ripple::TxMeta const txMeta{ - sttx.getTransactionID(), tx.ledgerSequence, tx.metadata}; - return std::get<1>(getNFTDataFromTx(txMeta, sttx)).value(); - }); + for (auto const& tx : txs) + { + ripple::STTx const sttx{ripple::SerialIter{ + tx.transaction.data(), tx.transaction.size()}}; + ripple::TxMeta const txMeta{ + sttx.getTransactionID(), tx.ledgerSequence, tx.metadata}; + toWrite.push_back( + std::get<1>(getNFTDataFromTx(txMeta, sttx)).value()); + } toWrite = maybeDoNFTWrite(toWrite, backend, "TX"); From 44f7c77b0f2e2f2158fcb9042e0b6c4e7837667d Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 01:48:35 -0400 Subject: [PATCH 3/4] silence logging --- src/backend/CassandraBackend.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/CassandraBackend.cpp b/src/backend/CassandraBackend.cpp index d5f9e6ef8..978d48ba7 100644 --- a/src/backend/CassandraBackend.cpp +++ b/src/backend/CassandraBackend.cpp @@ -550,7 +550,7 @@ CassandraBackend::fetchTransactions( std::vector results{numHashes}; std::vector>> cbs; cbs.reserve(numHashes); - auto timeDiff = util::timed([&]() { + [[maybe_unused]] auto timeDiff = util::timed([&]() { for (std::size_t i = 0; i < hashes.size(); ++i) { CassandraStatement statement{selectTransaction_}; @@ -580,9 +580,9 @@ CassandraBackend::fetchTransactions( throw DatabaseTimeout(); } - log_.debug() << "Fetched " << numHashes - << " transactions from Cassandra in " << timeDiff - << " milliseconds"; + // log_.debug() << "Fetched " << numHashes + // << " transactions from Cassandra in " << timeDiff + // << " milliseconds"; return results; } From ee397a2e277133ce352b3bbae8ccfca3eb4434a5 Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 01:55:09 -0400 Subject: [PATCH 4/4] speed up even more --- src/main/main.cpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/main.cpp b/src/main/main.cpp index 93398812a..dc6b1fcc9 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -185,23 +185,17 @@ doMigration( } auto txs = doTryFetchTransactions(timer, backend, txHashes, yield); - txs.erase( - std::remove_if( - txs.begin(), - txs.end(), - [&ledgerRange](Backend::TransactionAndMetadata const& tx) { - if (tx.ledgerSequence > ledgerRange->maxSequence) - return true; - ripple::STTx const sttx{ripple::SerialIter{ - tx.transaction.data(), tx.transaction.size()}}; - return sttx.getTxnType() != ripple::TxType::ttNFTOKEN_MINT; - }), - txs.end()); for (auto const& tx : txs) { + if (tx.ledgerSequence > ledgerRange->maxSequence) + continue; + ripple::STTx const sttx{ripple::SerialIter{ tx.transaction.data(), tx.transaction.size()}}; + if (sttx.getTxnType() != ripple::TxType::ttNFTOKEN_MINT) + continue; + ripple::TxMeta const txMeta{ sttx.getTransactionID(), tx.ledgerSequence, tx.metadata}; toWrite.push_back(