From fbdd8c92e1f21a4f246a870154bf0b2366e76c2f Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Wed, 3 May 2023 23:25:42 -0400 Subject: [PATCH 1/9] fix loading from ledger --- CMakeLists.txt | 3 +++ src/backend/BackendInterface.h | 2 +- src/main/main.cpp | 2 +- src/rpc/RPC.cpp | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc978625..6befb16e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,9 @@ if(VERBOSE) set(FETCHCONTENT_QUIET FALSE CACHE STRING "Verbose FetchContent()") endif() +#c++20 removed std::result_of but boost 1.75 is still using it. +add_definitions(-DBOOST_ASIO_HAS_STD_INVOKE_RESULT=1) + add_library(clio) target_compile_features(clio PUBLIC cxx_std_20) target_include_directories(clio PUBLIC src) diff --git a/src/backend/BackendInterface.h b/src/backend/BackendInterface.h index 2634650a..e70d3307 100644 --- a/src/backend/BackendInterface.h +++ b/src/backend/BackendInterface.h @@ -111,7 +111,7 @@ synchronous(F&& f) * R is the currently executing coroutine that is about to get passed. * If corountine types do not match, the current one's type is stored. */ - using R = typename std::result_of::type; + using R = typename boost::result_of::type; if constexpr (!std::is_same::value) { /** diff --git a/src/main/main.cpp b/src/main/main.cpp index 859ff209..db35b8dd 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -224,7 +224,7 @@ doMigration( { std::vector toWrite = getNFTDataFromObj( ledgerRange->minSequence, - ripple::to_string(object.key), + std::string(object.key.begin(), object.key.end()), std::string(object.blob.begin(), object.blob.end())); doNFTWrite(toWrite, backend, "OBJ"); } diff --git a/src/rpc/RPC.cpp b/src/rpc/RPC.cpp index 35ae1e73..32e13a98 100644 --- a/src/rpc/RPC.cpp +++ b/src/rpc/RPC.cpp @@ -178,7 +178,7 @@ public: { for (auto const& handler : handlers) { - handlerMap_[handler.method] = move(handler); + handlerMap_[handler.method] = std::move(handler); } } From 6f20306884f52cb38ac7e0c775f9fb80059a1f41 Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Thu, 4 May 2023 06:52:29 +0000 Subject: [PATCH 2/9] improve speed of initial ledger loading --- src/main/main.cpp | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/src/main/main.cpp b/src/main/main.cpp index db35b8dd..9afc51c2 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -12,9 +12,10 @@ static std::uint32_t const MAX_RETRIES = 5; static std::chrono::seconds const WAIT_TIME = std::chrono::seconds(60); +static std::uint32_t const NFT_WRITE_BATCH_SIZE = 10000; static void -wait(boost::asio::steady_timer& timer, std::string const reason) +wait(boost::asio::steady_timer& timer, std::string const& reason) { BOOST_LOG_TRIVIAL(info) << reason << ". Waiting"; timer.expires_after(WAIT_TIME); @@ -22,18 +23,31 @@ wait(boost::asio::steady_timer& timer, std::string const reason) BOOST_LOG_TRIVIAL(info) << "Done"; } -static void +static std::vector doNFTWrite( - std::vector& nfts, + std::vector&& nfts, Backend::CassandraBackend& backend, - std::string const tag) + std::string const& tag) { - if (nfts.size() <= 0) - return; + if (nfts.size() == 0) + return nfts; auto const size = nfts.size(); backend.writeNFTs(std::move(nfts)); backend.sync(); BOOST_LOG_TRIVIAL(info) << tag << ": Wrote " << size << " records"; + nfts.clear(); + return nfts; +} + +static std::vector +maybeDoNFTWrite( + std::vector&& nfts, + Backend::CassandraBackend& backend, + std::string const& tag) +{ + if (nfts.size() < NFT_WRITE_BATCH_SIZE) + return nfts; + return doNFTWrite(std::move(nfts), backend, tag); } static std::optional @@ -122,6 +136,8 @@ doMigration( return; } + std::vector toWrite; + /* * Step 1 - Look at all NFT transactions recorded in * `nf_token_transactions` and reload any NFTokenMint transactions. These @@ -140,8 +156,6 @@ doMigration( // For all NFT txs, paginated in groups of 1000... while (morePages) { - std::vector toWrite; - CassResult const* result = doTryGetTxPageResult(nftTxQuery, timer, backend); @@ -195,7 +209,7 @@ doMigration( std::get<1>(getNFTDataFromTx(txMeta, sttx)).value()); } - doNFTWrite(toWrite, backend, "TX"); + toWrite = maybeDoNFTWrite(std::move(toWrite), backend, "TX"); morePages = cass_result_has_more_pages(result); if (morePages) @@ -205,6 +219,7 @@ doMigration( } cass_statement_free(nftTxQuery); + toWrite = doNFTWrite(std::move(toWrite), backend, "TX"); BOOST_LOG_TRIVIAL(info) << "\nDone with transaction loading!\n"; /* @@ -216,21 +231,27 @@ doMigration( */ std::optional cursor; + // For each object page in initial ledger do { auto const page = doTryFetchLedgerPage( timer, backend, cursor, ledgerRange->minSequence, yield); + + // For each object in page of 2000 for (auto const& object : page.objects) { - std::vector toWrite = getNFTDataFromObj( + std::vector objectNFTs = getNFTDataFromObj( ledgerRange->minSequence, std::string(object.key.begin(), object.key.end()), std::string(object.blob.begin(), object.blob.end())); - doNFTWrite(toWrite, backend, "OBJ"); + toWrite.insert(toWrite.end(), objectNFTs.begin(), objectNFTs.end()); } + + toWrite = maybeDoNFTWrite(std::move(toWrite), backend, "OBJ"); cursor = page.cursor; } while (cursor.has_value()); + toWrite = doNFTWrite(std::move(toWrite), backend, "OBJ"); BOOST_LOG_TRIVIAL(info) << "\nDone with object loading!\n"; /* From fca45f39987f584a6f3e8a98ac8285f30e2887e0 Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 04:12:30 +0000 Subject: [PATCH 3/9] remove l values --- src/main/main.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/main.cpp b/src/main/main.cpp index 9afc51c2..6776c19f 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -25,7 +25,7 @@ wait(boost::asio::steady_timer& timer, std::string const& reason) static std::vector doNFTWrite( - std::vector&& nfts, + std::vector& nfts, Backend::CassandraBackend& backend, std::string const& tag) { @@ -35,19 +35,18 @@ doNFTWrite( backend.writeNFTs(std::move(nfts)); backend.sync(); BOOST_LOG_TRIVIAL(info) << tag << ": Wrote " << size << " records"; - nfts.clear(); - return nfts; + return {}; } static std::vector maybeDoNFTWrite( - std::vector&& nfts, + std::vector& nfts, Backend::CassandraBackend& backend, std::string const& tag) { if (nfts.size() < NFT_WRITE_BATCH_SIZE) return nfts; - return doNFTWrite(std::move(nfts), backend, tag); + return doNFTWrite(nfts, backend, tag); } static std::optional @@ -209,7 +208,7 @@ doMigration( std::get<1>(getNFTDataFromTx(txMeta, sttx)).value()); } - toWrite = maybeDoNFTWrite(std::move(toWrite), backend, "TX"); + toWrite = maybeDoNFTWrite(toWrite, backend, "TX"); morePages = cass_result_has_more_pages(result); if (morePages) @@ -219,7 +218,7 @@ doMigration( } cass_statement_free(nftTxQuery); - toWrite = doNFTWrite(std::move(toWrite), backend, "TX"); + toWrite = doNFTWrite(toWrite, backend, "TX"); BOOST_LOG_TRIVIAL(info) << "\nDone with transaction loading!\n"; /* @@ -247,11 +246,11 @@ doMigration( toWrite.insert(toWrite.end(), objectNFTs.begin(), objectNFTs.end()); } - toWrite = maybeDoNFTWrite(std::move(toWrite), backend, "OBJ"); + toWrite = maybeDoNFTWrite(toWrite, backend, "OBJ"); cursor = page.cursor; } while (cursor.has_value()); - toWrite = doNFTWrite(std::move(toWrite), backend, "OBJ"); + toWrite = doNFTWrite(toWrite, backend, "OBJ"); BOOST_LOG_TRIVIAL(info) << "\nDone with object loading!\n"; /* From d4e2240126890acad53922a5d192e4e3d1a36b78 Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 01:39:20 -0400 Subject: [PATCH 4/9] attempt to speed up tx loading --- src/main/main.cpp | 74 ++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/src/main/main.cpp b/src/main/main.cpp index 6776c19f..8b3a02a8 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -17,10 +17,10 @@ static std::uint32_t const NFT_WRITE_BATCH_SIZE = 10000; static void wait(boost::asio::steady_timer& timer, std::string const& reason) { - BOOST_LOG_TRIVIAL(info) << reason << ". Waiting"; + BOOST_LOG_TRIVIAL(info) << reason << ". Waiting then retrying"; timer.expires_after(WAIT_TIME); timer.wait(); - BOOST_LOG_TRIVIAL(info) << "Done"; + BOOST_LOG_TRIVIAL(info) << "Done waiting"; } static std::vector @@ -49,25 +49,26 @@ maybeDoNFTWrite( return doNFTWrite(nfts, backend, tag); } -static std::optional -doTryFetchTransaction( +static std::vector +doTryFetchTransactions( boost::asio::steady_timer& timer, Backend::CassandraBackend& backend, - ripple::uint256 const& hash, + std::vector const& hashes, boost::asio::yield_context& yield, std::uint32_t const attempts = 0) { try { - return backend.fetchTransaction(hash, yield); + return backend.fetchTransactions(hashes, yield); } catch (Backend::DatabaseTimeout const& e) { if (attempts >= MAX_RETRIES) throw e; - wait(timer, "Transaction read error"); - return doTryFetchTransaction(timer, backend, hash, yield, attempts + 1); + wait(timer, "Transactions read error"); + return doTryFetchTransactions( + timer, backend, hashes, yield, attempts + 1); } } @@ -158,6 +159,8 @@ doMigration( CassResult const* result = doTryGetTxPageResult(nftTxQuery, timer, backend); + std::vector txHashes; + // For each tx in page... CassIterator* txPageIterator = cass_iterator_from_result(result); while (cass_iterator_next(txPageIterator)) @@ -178,36 +181,35 @@ doMigration( "Could not retrieve hash from nf_token_transactions"); } - auto const txHash = ripple::uint256::fromVoid(buf); - auto const tx = - doTryFetchTransaction(timer, backend, txHash, yield); - if (!tx) - { - cass_iterator_free(txPageIterator); - cass_result_free(result); - cass_statement_free(nftTxQuery); - std::stringstream ss; - ss << "Could not fetch tx with hash " - << ripple::to_string(txHash); - throw std::runtime_error(ss.str()); - } - - // Not really sure how cassandra paging works, but we want to skip - // any transactions that were loaded since the migration started - if (tx->ledgerSequence > ledgerRange->maxSequence) - continue; - - ripple::STTx const sttx{ripple::SerialIter{ - tx->transaction.data(), tx->transaction.size()}}; - if (sttx.getTxnType() != ripple::TxType::ttNFTOKEN_MINT) - continue; - - ripple::TxMeta const txMeta{ - sttx.getTransactionID(), tx->ledgerSequence, tx->metadata}; - toWrite.push_back( - std::get<1>(getNFTDataFromTx(txMeta, sttx)).value()); + txHashes.push_back(ripple::uint256::fromVoid(buf)); } + auto txs = doTryFetchTransactions(timer, backend, txHashes, yield); + txs.erase( + std::remove_if( + txs.begin(), + txs.end(), + [&ledgerRange](Backend::TransactionAndMetadata const& tx) { + if (tx.ledgerSequence > ledgerRange->maxSequence) + return true; + ripple::STTx const sttx{ripple::SerialIter{ + tx.transaction.data(), tx.transaction.size()}}; + return sttx.getTxnType() != ripple::TxType::ttNFTOKEN_MINT; + }), + txs.end()); + + std::transform( + txs.begin(), + txs.end(), + toWrite.end(), + [](Backend::TransactionAndMetadata const& tx) { + ripple::STTx const sttx{ripple::SerialIter{ + tx.transaction.data(), tx.transaction.size()}}; + ripple::TxMeta const txMeta{ + sttx.getTransactionID(), tx.ledgerSequence, tx.metadata}; + return std::get<1>(getNFTDataFromTx(txMeta, sttx)).value(); + }); + toWrite = maybeDoNFTWrite(toWrite, backend, "TX"); morePages = cass_result_has_more_pages(result); From a8641c1b3bb299ef47bd99c8d9508cbcaf41b73d Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 01:44:54 -0400 Subject: [PATCH 5/9] loop instead of transform --- src/main/main.cpp | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/main/main.cpp b/src/main/main.cpp index 8b3a02a8..93398812 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -198,17 +198,15 @@ doMigration( }), txs.end()); - std::transform( - txs.begin(), - txs.end(), - toWrite.end(), - [](Backend::TransactionAndMetadata const& tx) { - ripple::STTx const sttx{ripple::SerialIter{ - tx.transaction.data(), tx.transaction.size()}}; - ripple::TxMeta const txMeta{ - sttx.getTransactionID(), tx.ledgerSequence, tx.metadata}; - return std::get<1>(getNFTDataFromTx(txMeta, sttx)).value(); - }); + for (auto const& tx : txs) + { + ripple::STTx const sttx{ripple::SerialIter{ + tx.transaction.data(), tx.transaction.size()}}; + ripple::TxMeta const txMeta{ + sttx.getTransactionID(), tx.ledgerSequence, tx.metadata}; + toWrite.push_back( + std::get<1>(getNFTDataFromTx(txMeta, sttx)).value()); + } toWrite = maybeDoNFTWrite(toWrite, backend, "TX"); From 44f7c77b0f2e2f2158fcb9042e0b6c4e7837667d Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 01:48:35 -0400 Subject: [PATCH 6/9] silence logging --- src/backend/CassandraBackend.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/CassandraBackend.cpp b/src/backend/CassandraBackend.cpp index d5f9e6ef..978d48ba 100644 --- a/src/backend/CassandraBackend.cpp +++ b/src/backend/CassandraBackend.cpp @@ -550,7 +550,7 @@ CassandraBackend::fetchTransactions( std::vector results{numHashes}; std::vector>> cbs; cbs.reserve(numHashes); - auto timeDiff = util::timed([&]() { + [[maybe_unused]] auto timeDiff = util::timed([&]() { for (std::size_t i = 0; i < hashes.size(); ++i) { CassandraStatement statement{selectTransaction_}; @@ -580,9 +580,9 @@ CassandraBackend::fetchTransactions( throw DatabaseTimeout(); } - log_.debug() << "Fetched " << numHashes - << " transactions from Cassandra in " << timeDiff - << " milliseconds"; + // log_.debug() << "Fetched " << numHashes + // << " transactions from Cassandra in " << timeDiff + // << " milliseconds"; return results; } From ee397a2e277133ce352b3bbae8ccfca3eb4434a5 Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 01:55:09 -0400 Subject: [PATCH 7/9] speed up even more --- src/main/main.cpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/main.cpp b/src/main/main.cpp index 93398812..dc6b1fcc 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -185,23 +185,17 @@ doMigration( } auto txs = doTryFetchTransactions(timer, backend, txHashes, yield); - txs.erase( - std::remove_if( - txs.begin(), - txs.end(), - [&ledgerRange](Backend::TransactionAndMetadata const& tx) { - if (tx.ledgerSequence > ledgerRange->maxSequence) - return true; - ripple::STTx const sttx{ripple::SerialIter{ - tx.transaction.data(), tx.transaction.size()}}; - return sttx.getTxnType() != ripple::TxType::ttNFTOKEN_MINT; - }), - txs.end()); for (auto const& tx : txs) { + if (tx.ledgerSequence > ledgerRange->maxSequence) + continue; + ripple::STTx const sttx{ripple::SerialIter{ tx.transaction.data(), tx.transaction.size()}}; + if (sttx.getTxnType() != ripple::TxType::ttNFTOKEN_MINT) + continue; + ripple::TxMeta const txMeta{ sttx.getTransactionID(), tx.ledgerSequence, tx.metadata}; toWrite.push_back( From a71e40616b2d429fd14e1e1b1472c2056202e1aa Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 03:34:39 -0400 Subject: [PATCH 8/9] improve README --- README.md | 79 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 52b4e559..0fbaa9f0 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,81 @@ # CLIO MIGRATOR (ONE OFF!) -This tool is a (really) hacky way of migrating some data from +This tool allows you to backfill data from [clio](https://github.com/XRPLF/clio) due to the [specific pull request 313](https://github.com/XRPLF/clio/pull/313) in that repo. Specifically, it is meant to migrate NFT data such that: -* The new `nf_token_uris` table is populated with all URIs for all NFTs known -* The new `issuer_nf_tokens_v2` table is populated with all NFTs known -* The old `issuer_nf_tokens` table is dropped. This table was never used prior - to the above-referenced PR, so it is very safe to drop. +- The new `nf_token_uris` table is populated with all URIs for all NFTs known +- The new `issuer_nf_tokens_v2` table is populated with all NFTs known +- The old `issuer_nf_tokens` table is dropped. This table was never used prior +to the above-referenced PR, so it is very safe to drop. + +## How to use This tool should be used as follows, with regard to the above update: -1) Stop serving requests from your clio -2) Stop your clio and upgrade it to the version after the after PR -3) Start your clio -4) Now, your clio is writing new data correctly. This tool will update your -old data, while your new clio is running. -5) Run this tool, using the _exact_ same config as what you are using for your +1. __Stop serving requests from your clio__ If you need to achieve zero downtime, you have two options: + - Temporarily point your traffic to someone else's clio that has already performed this + migration. The XRPL Foundation should have performed this on their servers before this + release. Ask in our Discord what server to point traffic to. + - Create a new temporary `clio` instance running _the prior release_ and make sure + that its config.json specifies `read_only: true`. You can safely serve data + from this separate instance. +2. __Stop your clio and upgrade it to the version after the above PR__ +3. __Start your clio__ Now, your clio is writing new data correctly. This tool will update your +old data, while your new clio is running and writing new ledgers. +5. __Run this tool__, using the _exact_ same config as what you are using for your production clio. -6) Once this tool terminates successfully, you can resume serving requests +6. __Once this tool terminates successfully__, you can resume serving requests from your clio. -## Compiling +## Notes on timing + +The amount of time that this migration takes depends greatly on what your data +looks like. This migration migrates data in three steps: + +1. __Transaction loading__ + - Pull all successful transactions that relate to NFTs. + The hashes of these transactions are stored in the `nf_token_transctions` table. + - For each of these transactions, discard any that were posted after the + migration started + - For each of these transactions, discard any that are not NFTokenMint + transactions + - For any remaning transactions, pull the associated NFT data from them and + write them to the database. +2. __Initial ledger loading__ We need to also scan all objects in the initial +ledger, looking for any NFTokenPage objects that would not have an associated +transaction recorded. + - Pull all objects from the initial ledger + - For each object, if it is not an NFTokenPage, discard it. + - Otherwise, load all NFTs stored in the NFTokenPage +3. __Drop the old (and unused) `issuer_nf_tokens` table__. This step is completely +safe, since this table is not used for anything in clio. It was meant to drive +a clio-only API called `nfts_by_issuer`, which is still in development. +However, we decided that for performance reasons its schema needed to change +to the schema we have in `issuer_nf_tokens_v2`. Since the API in question is +not yet part of clio, removing this table will not affect anything. + + +Step 1 is highly performance optimized. If you have a full-history clio +set-up, this migration make take only a few minutes. We tested it on a +full-history server and it completed in about 9 minutes. + +However Step 2 is not well-optimized and unfortuntely cannot be. If you have a +clio server whose `start_sequence` is relatively recent (even if the +`start_sequence` indicates a ledger prior to NFTs being enabled on your +network), the migration will take longer. We tested it on a clio with a +`start_sequence` of about 1 week prior to testing and it completed in about 6 +hours. + +As a result, we recommend _assuming_ the worst case: that this migration will take about 8 +hours. + + + +## Compiling and running Git-clone this project to your server. Then from the top-level directory: ``` @@ -39,5 +90,3 @@ you should copy your existing clio config somewhere and: ``` ./clio_migrator ``` - -This migration will take a few hours to complete. From 57ea5ae6fdc058ea88e48ed20df78b1404a74e55 Mon Sep 17 00:00:00 2001 From: ledhed2222 Date: Fri, 5 May 2023 04:14:22 -0400 Subject: [PATCH 9/9] more docs improvements and poss step 2 speed increase --- README.md | 14 +++---- src/main/main.cpp | 102 ++++++++++++++++++++++++++++++---------------- 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 0fbaa9f0..32cb45ce 100644 --- a/README.md +++ b/README.md @@ -15,20 +15,20 @@ to the above-referenced PR, so it is very safe to drop. This tool should be used as follows, with regard to the above update: -1. __Stop serving requests from your clio__ If you need to achieve zero downtime, you have two options: - - Temporarily point your traffic to someone else's clio that has already performed this +1. __Compile or download the new version of `clio`__, but don't run it just yet. +2. __Stop serving requests from your existing `clio`__. If you need to achieve zero downtime, you have two options: + - Temporarily point your traffic to someone else's `clio` that has already performed this migration. The XRPL Foundation should have performed this on their servers before this release. Ask in our Discord what server to point traffic to. - Create a new temporary `clio` instance running _the prior release_ and make sure that its config.json specifies `read_only: true`. You can safely serve data from this separate instance. -2. __Stop your clio and upgrade it to the version after the above PR__ -3. __Start your clio__ Now, your clio is writing new data correctly. This tool will update your -old data, while your new clio is running and writing new ledgers. +3. __Stop your `clio` and restart it, running the new version__. Now, your `clio` is writing new data correctly. This tool will update your +old data, while your upgraded `clio` is running and writing new ledgers. 5. __Run this tool__, using the _exact_ same config as what you are using for your -production clio. +production `clio`. 6. __Once this tool terminates successfully__, you can resume serving requests -from your clio. +from your `clio`. ## Notes on timing diff --git a/src/main/main.cpp b/src/main/main.cpp index dc6b1fcc..4d9baa13 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -83,7 +83,7 @@ doTryFetchLedgerPage( { try { - return backend.fetchLedgerPage(cursor, sequence, 2000, false, yield); + return backend.fetchLedgerPage(cursor, sequence, 10000, false, yield); } catch (Backend::DatabaseTimeout const& e) { @@ -118,26 +118,12 @@ doTryGetTxPageResult( } static void -doMigration( +doMigrationStepOne( Backend::CassandraBackend& backend, boost::asio::steady_timer& timer, - boost::asio::yield_context& yield) + boost::asio::yield_context& yield, + Backend::LedgerRange const& ledgerRange) { - BOOST_LOG_TRIVIAL(info) << "Beginning migration"; - auto const ledgerRange = backend.hardFetchLedgerRangeNoThrow(yield); - - /* - * Step 0 - If we haven't downloaded the initial ledger yet, just short - * circuit. - */ - if (!ledgerRange) - { - BOOST_LOG_TRIVIAL(info) << "There is no data to migrate"; - return; - } - - std::vector toWrite; - /* * Step 1 - Look at all NFT transactions recorded in * `nf_token_transactions` and reload any NFTokenMint transactions. These @@ -146,6 +132,9 @@ doMigration( * the tokens in `nf_tokens` because we also want to cover the extreme * edge case of a token that is re-minted with a different URI. */ + std::string const stepTag = "Step 1 - transaction loading"; + std::vector toWrite; + std::stringstream query; query << "SELECT hash FROM " << backend.tablePrefix() << "nf_token_transactions"; @@ -184,11 +173,12 @@ doMigration( txHashes.push_back(ripple::uint256::fromVoid(buf)); } - auto txs = doTryFetchTransactions(timer, backend, txHashes, yield); + auto const txs = + doTryFetchTransactions(timer, backend, txHashes, yield); for (auto const& tx : txs) { - if (tx.ledgerSequence > ledgerRange->maxSequence) + if (tx.ledgerSequence > ledgerRange.maxSequence) continue; ripple::STTx const sttx{ripple::SerialIter{ @@ -202,7 +192,7 @@ doMigration( std::get<1>(getNFTDataFromTx(txMeta, sttx)).value()); } - toWrite = maybeDoNFTWrite(toWrite, backend, "TX"); + toWrite = maybeDoNFTWrite(toWrite, backend, stepTag); morePages = cass_result_has_more_pages(result); if (morePages) @@ -212,9 +202,16 @@ doMigration( } cass_statement_free(nftTxQuery); - toWrite = doNFTWrite(toWrite, backend, "TX"); - BOOST_LOG_TRIVIAL(info) << "\nDone with transaction loading!\n"; + doNFTWrite(toWrite, backend, stepTag); +} +static void +doMigrationStepTwo( + Backend::CassandraBackend& backend, + boost::asio::steady_timer& timer, + boost::asio::yield_context& yield, + Backend::LedgerRange const& ledgerRange) +{ /* * Step 2 - Pull every object from our initial ledger and load all NFTs * found in any NFTokenPage object. Prior to this migration, we were not @@ -222,38 +219,43 @@ doMigration( * missed. This will also record the URI of any NFTs minted prior to the * start sequence. */ + std::string const stepTag = "Step 2 - initial ledger loading"; + std::vector toWrite; std::optional cursor; // For each object page in initial ledger do { auto const page = doTryFetchLedgerPage( - timer, backend, cursor, ledgerRange->minSequence, yield); + timer, backend, cursor, ledgerRange.minSequence, yield); - // For each object in page of 2000 + // For each object in page for (auto const& object : page.objects) { - std::vector objectNFTs = getNFTDataFromObj( - ledgerRange->minSequence, + auto const objectNFTs = getNFTDataFromObj( + ledgerRange.minSequence, std::string(object.key.begin(), object.key.end()), std::string(object.blob.begin(), object.blob.end())); toWrite.insert(toWrite.end(), objectNFTs.begin(), objectNFTs.end()); } - toWrite = maybeDoNFTWrite(toWrite, backend, "OBJ"); + toWrite = maybeDoNFTWrite(toWrite, backend, stepTag); cursor = page.cursor; } while (cursor.has_value()); - toWrite = doNFTWrite(toWrite, backend, "OBJ"); - BOOST_LOG_TRIVIAL(info) << "\nDone with object loading!\n"; + doNFTWrite(toWrite, backend, stepTag); +} +static bool +doMigrationStepThree(Backend::CassandraBackend& backend) +{ /* * Step 3 - Drop the old `issuer_nf_tokens` table, which is replaced by * `issuer_nf_tokens_v2`. Normally, we should probably not drop old tables * in migrations, but here it is safe since the old table wasn't yet being * used to serve any data anyway. */ - query.str(""); + std::stringstream query; query << "DROP TABLE " << backend.tablePrefix() << "issuer_nf_tokens"; CassStatement* issuerDropTableQuery = cass_statement_new(query.str().c_str(), 0); @@ -263,12 +265,42 @@ doMigration( cass_future_free(fut); cass_statement_free(issuerDropTableQuery); backend.sync(); - if (rc != CASS_OK) - BOOST_LOG_TRIVIAL(warning) << "\nCould not drop old issuer_nf_tokens " + return rc == CASS_OK; +} + +static void +doMigration( + Backend::CassandraBackend& backend, + boost::asio::steady_timer& timer, + boost::asio::yield_context& yield) +{ + BOOST_LOG_TRIVIAL(info) << "Beginning migration"; + auto const ledgerRange = backend.hardFetchLedgerRangeNoThrow(yield); + + /* + * Step 0 - If we haven't downloaded the initial ledger yet, just short + * circuit. + */ + if (!ledgerRange) + { + BOOST_LOG_TRIVIAL(info) << "There is no data to migrate"; + return; + } + + doMigrationStepOne(backend, timer, yield, *ledgerRange); + BOOST_LOG_TRIVIAL(info) << "\nStep 1 done!\n"; + + doMigrationStepTwo(backend, timer, yield, *ledgerRange); + BOOST_LOG_TRIVIAL(info) << "\nStep 2 done!\n"; + + auto const stepThreeResult = doMigrationStepThree(backend); + BOOST_LOG_TRIVIAL(info) << "\nStep 3 done!"; + if (stepThreeResult) + BOOST_LOG_TRIVIAL(info) << "Dropped old 'issuer_nf_tokens' table!\n"; + else + BOOST_LOG_TRIVIAL(warning) << "Could not drop old issuer_nf_tokens " "table. If it still exists, " "you should drop it yourself\n"; - else - BOOST_LOG_TRIVIAL(info) << "\nDropped old 'issuer_nf_tokens' table!\n"; BOOST_LOG_TRIVIAL(info) << "\nCompleted migration from " << ledgerRange->minSequence << " to "