diff --git a/README.md b/README.md index 6828bce84..0c0e6440f 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ Specifically, it is meant to migrate NFT data such that: - The old `issuer_nf_tokens` table is dropped. This table was never used prior to the above-referenced PR, so it is very safe to drop. -## How to use - +## Overall Migration Steps This tool should be used as follows, with regard to the above update: +0. __Copy your current clio configuration file to this repo__. 1. __Compile or download the new version of `clio`__, but don't run it just yet. 2. __Stop serving requests from your existing `clio`__. If you need to achieve zero downtime, you have two options: - Temporarily point your traffic to someone else's `clio` that has already performed this @@ -23,16 +23,46 @@ This tool should be used as follows, with regard to the above update: - Create a new temporary `clio` instance running _the prior release_ and make sure that its config.json specifies `read_only: true`. You can safely serve data from this separate instance. -3. __Stop your `clio` and restart it, running the new version__. Now, your `clio` is writing new data correctly. This tool will update your +3. __Stop your `clio` and restart it, running the new version__. You may make any changes necessary to the config, +but do not change the old configuration you copied in Step 0. +4. __Now, your `clio` is writing new data correctly__. This tool will update your old data, while your upgraded `clio` is running and writing new ledgers. -5. __Run this tool__, using the _exact_ same config as what you are using for your -production `clio`. +5. __Run this tool__, using the configuration file you copied in Step 0. + Details on how to run the tool are below in the "usage" section. 6. __Once this tool terminates successfully__, you can resume serving requests from your `clio`. +7. __Optionally__ you can now use the included `clio_verifier` executable to + ensure that URIs were migrated correctly. This executable exit with a 0 + status code if everything is OK. It will take a long time to run, and is + meant to be run while you are already fully running on the new version. + Details on how to run this tool are below in the "usage" section. +## Usage +### Compiling +Git-clone this project to your server. Then from the top-level directory: +```bash +mkdir build +cd build +cmake .. +cmake --build . -j 4 +``` -## Notes on timing +### Running the migration +Once this completes, the migrator will be compiled as `clio_migrator`. Then +you should use the old config file you copied in Step 0 above to run it like +so: +```bash +./clio_migrator +``` +### OPTIONAL: running the verifier +After the migration completes, it is optional to perform a database verification to ensure the URIs are migrated correctly. +Again, use the old config file you copied in Step 0 above. +```bash +./clio_verifier +``` + +## Technical details and notes on timing The amount of time that this migration takes depends greatly on what your data looks like. This migration migrates data in three steps: @@ -45,7 +75,7 @@ looks like. This migration migrates data in three steps: transactions - For any remaning transactions, pull the associated NFT data from them and write them to the database. -2. __Initial ledger loading__ We need to also scan all objects in the initial +2. __Initial ledger loading__. We need to also scan all objects in the initial ledger, looking for any NFTokenPage objects that would not have an associated transaction recorded. - Pull all objects from the initial ledger @@ -60,10 +90,10 @@ not yet part of clio, removing this table will not affect anything. Step 1 is highly performance optimized. If you have a full-history clio -set-up, this migration make take only a few minutes. We tested it on a +set-up, this migration may take only a few minutes. We tested it on a full-history server and it completed in about 9 minutes. -However Step 2 is not well-optimized and unfortuntely cannot be. If you have a +However Step 2 is not well-optimized and unfortunately cannot be. If you have a clio server whose `start_sequence` is relatively recent (even if the `start_sequence` indicates a ledger prior to NFTs being enabled on your network), the migration will take longer. We tested it on a clio with a @@ -72,27 +102,3 @@ hours. As a result, we recommend _assuming_ the worst case: that this migration will take about 8 hours. - - - -## Compiling and running - -Git-clone this project to your server. Then from the top-level directory: -``` -mkdir build -cd build -cmake .. -cmake --build . -j 4 -``` - -Once this completes, the migrator will be compiled as `clio_migrator`. Then -you should copy your existing clio config somewhere and: -``` -./clio_migrator -``` - -After the migration completes, it is optional to perform a database verification to ensure the fields are migrated correctly: -``` -./clio_verifier -``` - diff --git a/src/main/main.cpp b/src/main/main.cpp index 6632999ae..24b38b6f7 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -29,9 +29,9 @@ doNFTWrite( Backend::CassandraBackend& backend, std::string const& tag) { - if (nfts.size() == 0) - return nfts; auto const size = nfts.size(); + if (size == 0) + return nfts; backend.writeNFTs(std::move(nfts)); backend.sync(); BOOST_LOG_TRIVIAL(info) << tag << ": Wrote " << size << " records"; @@ -246,7 +246,7 @@ doMigrationStepTwo( doNFTWrite(toWrite, backend, stepTag); } -static bool +static void doMigrationStepThree(Backend::CassandraBackend& backend) { /* @@ -265,7 +265,11 @@ doMigrationStepThree(Backend::CassandraBackend& backend) cass_future_free(fut); cass_statement_free(issuerDropTableQuery); backend.sync(); - return rc == CASS_OK; + + if (rc != CASS_OK) + BOOST_LOG_TRIVIAL(warning) << "Could not drop old issuer_nf_tokens " + "table. If it still exists, " + "you should drop it yourself\n"; } static void @@ -293,14 +297,8 @@ doMigration( doMigrationStepTwo(backend, timer, yield, *ledgerRange); BOOST_LOG_TRIVIAL(info) << "\nStep 2 done!\n"; - auto const stepThreeResult = doMigrationStepThree(backend); - BOOST_LOG_TRIVIAL(info) << "\nStep 3 done!"; - if (stepThreeResult) - BOOST_LOG_TRIVIAL(info) << "Dropped old 'issuer_nf_tokens' table!\n"; - else - BOOST_LOG_TRIVIAL(warning) << "Could not drop old issuer_nf_tokens " - "table. If it still exists, " - "you should drop it yourself\n"; + doMigrationStepThree(backend); + BOOST_LOG_TRIVIAL(info) << "\nStep 3 done!\n"; BOOST_LOG_TRIVIAL(info) << "\nCompleted migration from " << ledgerRange->minSequence << " to " @@ -326,7 +324,7 @@ main(int argc, char* argv[]) } auto const type = config.value("database.type"); - if (!(boost::iequals(type, "cassandra") || boost::iequals(type, "cassandra-new"))) + if (!boost::iequals(type, "cassandra")) { std::cerr << "Migration only for cassandra dbs" << std::endl; return EXIT_FAILURE; diff --git a/src/main/verify.cpp b/src/main/verify.cpp index 077451764..3b8a0285d 100644 --- a/src/main/verify.cpp +++ b/src/main/verify.cpp @@ -4,15 +4,17 @@ #include #include
-#include #include #include #include +#include #include static std::uint32_t const MAX_RETRIES = 5; static std::chrono::seconds const WAIT_TIME = std::chrono::seconds(60); +static std::uint32_t const MIN_VERIFICATION_BATCH = 2000; + using Blob = std::vector; static void @@ -48,32 +50,79 @@ doTryFetchLedgerPage( } } -static void +static std::optional +doTryGetNFT( + boost::asio::steady_timer& timer, + Backend::CassandraBackend& backend, + ripple::uint256 const& nftID, + std::uint32_t const seq, + boost::asio::yield_context& yield, + std::uint32_t const attempts = 0) +{ + try + { + return backend.fetchNFT(nftID, seq, yield); + } + catch (Backend::DatabaseTimeout const& e) + { + if (attempts >= MAX_RETRIES) + throw e; + + wait(timer, "NFT read error"); + return doTryGetNFT(timer, backend, nftID, seq, yield, attempts + 1); + } +} + +static std::vector verifyNFTs( + boost::asio::steady_timer& timer, + std::uint32_t const seq, std::vector& nfts, Backend::CassandraBackend& backend, boost::asio::yield_context& yield) { if (nfts.size() <= 0) - return; + return nfts; - for(auto const& nft : nfts){ - std::optional writtenNFT = backend.fetchNFT(nft.tokenID, nft.ledgerSequence, yield); + for (auto const& nft : nfts) + { + auto const writtenNFT = + doTryGetNFT(timer, backend, nft.tokenID, seq, yield); - if(!writtenNFT.has_value()) + if (!writtenNFT.has_value()) throw std::runtime_error("NFT was not written!"); Blob writtenUriBlob = writtenNFT->uri; std::string writtenUriStr = ripple::strHex(writtenUriBlob); auto fetchOldUri = nft.uri; - std::string oldUriStr = nft.uri.has_value() ? ripple::strHex(nft.uri.value()) : ""; + std::string oldUriStr = + nft.uri.has_value() ? ripple::strHex(nft.uri.value()) : ""; - if(oldUriStr.compare(writtenUriStr) != 0){ - BOOST_LOG_TRIVIAL(warning) <<"\nNFTokenID "<< to_string(nft.tokenID) << " failed to match URIs!\n"; + if (oldUriStr.compare(writtenUriStr) != 0) + { + BOOST_LOG_TRIVIAL(warning) + << "\nNFTokenID " << to_string(nft.tokenID) + << " failed to match URIs!\n"; throw std::runtime_error("Failed to match!"); } } + + BOOST_LOG_TRIVIAL(info) << "Verified " << nfts.size() << " NFTs"; + return {}; +} + +static std::vector +maybeVerifyNFTs( + boost::asio::steady_timer& timer, + std::uint32_t const seq, + std::vector& nfts, + Backend::CassandraBackend& backend, + boost::asio::yield_context& yield) +{ + if (nfts.size() < MIN_VERIFICATION_BATCH) + return nfts; + return verifyNFTs(timer, seq, nfts, backend, yield); } static void @@ -95,32 +144,34 @@ doVerification( return; } + std::vector toVerify; + /* * Find all NFTokenPage objects and compare the URIs with what has been * written by the migrator - */ + */ std::optional cursor; - int nftCnt = 0; do { auto const page = doTryFetchLedgerPage( timer, backend, cursor, ledgerRange->maxSequence, yield); for (auto const& object : page.objects) { - std::vector toVerify = getNFTDataFromObj( + auto const nfts = getNFTDataFromObj( ledgerRange->maxSequence, std::string(object.key.begin(), object.key.end()), std::string(object.blob.begin(), object.blob.end())); - - //helper function to verify vector of NFTs - verifyNFTs(toVerify, backend, yield); - nftCnt += toVerify.size(); + toVerify.insert(toVerify.end(), nfts.begin(), nfts.end()); } + + toVerify = maybeVerifyNFTs( + timer, ledgerRange->maxSequence, toVerify, backend, yield); cursor = page.cursor; } while (cursor.has_value()); - BOOST_LOG_TRIVIAL(info) << "\nLedger range: " << ledgerRange->minSequence << "-" << ledgerRange->maxSequence << "\n"; - BOOST_LOG_TRIVIAL(info) << "\nVerified " << nftCnt << " NFTs!\n"; + verifyNFTs(timer, ledgerRange->maxSequence, toVerify, backend, yield); + BOOST_LOG_TRIVIAL(info) << "\nLedger range: " << ledgerRange->minSequence + << "-" << ledgerRange->maxSequence << "\n"; BOOST_LOG_TRIVIAL(info) << "\nDone with verification!\n"; }