From eb629592165b0d3d4a2bd81aefbbff457d93089e Mon Sep 17 00:00:00 2001 From: Edward Hennis Date: Wed, 9 Dec 2015 19:30:05 -0500 Subject: [PATCH] Clear old Validations during online delete (RIPD-870): * Add Validations.LedgerSeq and .InitialSeq fields. * Clean up logging. * Lower online delete minimum for standalone mode. * Unit tests of online_delete. --- Builds/VisualStudio2015/RippleD.vcxproj | 4 + .../VisualStudio2015/RippleD.vcxproj.filters | 3 + src/ripple/app/ledger/Ledger.cpp | 65 +- src/ripple/app/ledger/LedgerHistory.cpp | 4 +- src/ripple/app/ledger/impl/LedgerMaster.cpp | 2 +- src/ripple/app/main/DBInit.cpp | 15 + src/ripple/app/misc/SHAMapStore.h | 1 + src/ripple/app/misc/SHAMapStoreImp.cpp | 167 ++++- src/ripple/app/misc/SHAMapStoreImp.h | 16 +- src/ripple/app/misc/Validations.cpp | 41 +- src/ripple/app/tests/SHAMapStore_test.cpp | 654 ++++++++++++++++++ src/ripple/core/impl/DatabaseCon.cpp | 4 +- src/ripple/rpc/handlers/CanDelete.cpp | 9 +- src/ripple/unity/app_tests.cpp | 1 + 14 files changed, 917 insertions(+), 69 deletions(-) create mode 100644 src/ripple/app/tests/SHAMapStore_test.cpp diff --git a/Builds/VisualStudio2015/RippleD.vcxproj b/Builds/VisualStudio2015/RippleD.vcxproj index 5b8eeb824..4d66dc499 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj +++ b/Builds/VisualStudio2015/RippleD.vcxproj @@ -1531,6 +1531,10 @@ True True + + True + True + True True diff --git a/Builds/VisualStudio2015/RippleD.vcxproj.filters b/Builds/VisualStudio2015/RippleD.vcxproj.filters index ad78f1211..9b6454663 100644 --- a/Builds/VisualStudio2015/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2015/RippleD.vcxproj.filters @@ -2112,6 +2112,9 @@ ripple\app\tests + + ripple\app\tests + ripple\app\tests diff --git a/src/ripple/app/ledger/Ledger.cpp b/src/ripple/app/ledger/Ledger.cpp index 3e163dc08..f54d34796 100644 --- a/src/ripple/app/ledger/Ledger.cpp +++ b/src/ripple/app/ledger/Ledger.cpp @@ -890,16 +890,6 @@ static bool saveValidatedLedger ( "DELETE FROM AccountTransactions WHERE LedgerSeq = %u;"); static boost::format deleteAcctTrans ( "DELETE FROM AccountTransactions WHERE TransID = '%s';"); - static boost::format transExists ( - "SELECT Status FROM Transactions WHERE TransID = '%s';"); - static boost::format updateTx ( - "UPDATE Transactions SET LedgerSeq = %u, Status = '%c', TxnMeta = %s " - "WHERE TransID = '%s';"); - static boost::format addLedger ( - "INSERT OR REPLACE INTO Ledgers " - "(LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime," - "CloseTimeRes,CloseFlags,AccountSetHash,TransSetHash) VALUES " - "('%s','%u','%s','%s','%u','%u','%d','%u','%s','%s');"); auto seq = ledger->info().seq; @@ -1032,18 +1022,53 @@ static bool saveValidatedLedger ( } { + static std::string addLedger( + R"sql(INSERT OR REPLACE INTO Ledgers + (LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime, + CloseTimeRes,CloseFlags,AccountSetHash,TransSetHash) + VALUES + (:ledgerHash,:ledgerSeq,:prevHash,:totalCoins,:closingTime,:prevClosingTime, + :closeTimeRes,:closeFlags,:accountSetHash,:transSetHash);)sql"); + static std::string updateVal( + R"sql(UPDATE Validations SET LedgerSeq = :ledgerSeq, InitialSeq = :initialSeq + WHERE LedgerHash = :ledgerHash;)sql"); + auto db (app.getLedgerDB ().checkoutDb ()); - // TODO(tom): ARG! - *db << boost::str ( - addLedger % - to_string (ledger->info().hash) % seq % to_string (ledger->info().parentHash) % - to_string (ledger->info().drops) % - ledger->info().closeTime.time_since_epoch().count() % - ledger->info().parentCloseTime.time_since_epoch().count() % - ledger->info().closeTimeResolution.count() % - ledger->info().closeFlags % to_string (ledger->info().accountHash) % - to_string (ledger->info().txHash)); + soci::transaction tr(*db); + + auto const hash = to_string (ledger->info().hash); + auto const parentHash = to_string (ledger->info().parentHash); + auto const drops = to_string (ledger->info().drops); + auto const closeTime = + ledger->info().closeTime.time_since_epoch().count(); + auto const parentCloseTime = + ledger->info().parentCloseTime.time_since_epoch().count(); + auto const closeTimeResolution = + ledger->info().closeTimeResolution.count(); + auto const closeFlags = ledger->info().closeFlags; + auto const accountHash = to_string (ledger->info().accountHash); + auto const txHash = to_string (ledger->info().txHash); + + *db << addLedger, + soci::use(hash), + soci::use(seq), + soci::use(parentHash), + soci::use(drops), + soci::use(closeTime), + soci::use(parentCloseTime), + soci::use(closeTimeResolution), + soci::use(closeFlags), + soci::use(accountHash), + soci::use(txHash); + + + *db << updateVal, + soci::use(seq), + soci::use(seq), + soci::use(hash); + + tr.commit(); } // Clients can now trust the database for diff --git a/src/ripple/app/ledger/LedgerHistory.cpp b/src/ripple/app/ledger/LedgerHistory.cpp index 714392b91..0bdaa048c 100644 --- a/src/ripple/app/ledger/LedgerHistory.cpp +++ b/src/ripple/app/ledger/LedgerHistory.cpp @@ -499,7 +499,9 @@ void LedgerHistory::clearLedgerCachePrior (LedgerIndex seq) { for (LedgerHash it: m_ledgers_by_hash.getKeys()) { - if (getLedgerByHash (it)->info().seq < seq) + auto const ledger = getLedgerByHash (it); + assert(ledger); + if (!ledger || ledger->info().seq < seq) m_ledgers_by_hash.del (it, false); } } diff --git a/src/ripple/app/ledger/impl/LedgerMaster.cpp b/src/ripple/app/ledger/impl/LedgerMaster.cpp index 244b855cd..9144a6960 100644 --- a/src/ripple/app/ledger/impl/LedgerMaster.cpp +++ b/src/ripple/app/ledger/impl/LedgerMaster.cpp @@ -984,7 +984,7 @@ LedgerMaster::getLedgerHashForHistory (LedgerIndex index) if (! ret) ret = walkHashBySeq (index); - return *ret; + return ret; } bool diff --git a/src/ripple/app/main/DBInit.cpp b/src/ripple/app/main/DBInit.cpp index 96e9824e9..2a404fa3d 100644 --- a/src/ripple/app/main/DBInit.cpp +++ b/src/ripple/app/main/DBInit.cpp @@ -90,14 +90,29 @@ const char* LedgerDBInit[] = );", "CREATE INDEX IF NOT EXISTS SeqLedger ON Ledgers(LedgerSeq);", + // InitialSeq field is the current ledger seq when the row + // is inserted. Only relevant during online delete "CREATE TABLE IF NOT EXISTS Validations ( \ + LedgerSeq BIGINT UNSIGNED, \ + InitialSeq BIGINT UNSIGNED, \ LedgerHash CHARACTER(64), \ NodePubKey CHARACTER(56), \ SignTime BIGINT UNSIGNED, \ RawData BLOB \ );", + // This will error out if the column already exists, + // but DatabaseCon intentionally ignores errors. + "ALTER TABLE Validations \ + ADD COLUMN LedgerSeq BIGINT UNSIGNED;", + "ALTER TABLE Validations \ + ADD COLUMN InitialSeq BIGINT UNSIGNED;", + "CREATE INDEX IF NOT EXISTS ValidationsByHash ON \ Validations(LedgerHash);", + "CREATE INDEX IF NOT EXISTS ValidationsBySeq ON \ + Validations(LedgerSeq);", + "CREATE INDEX IF NOT EXISTS ValidationsByInitialSeq ON \ + Validations(InitialSeq, LedgerSeq);", "CREATE INDEX IF NOT EXISTS ValidationsByTime ON \ Validations(SignTime);", diff --git a/src/ripple/app/misc/SHAMapStore.h b/src/ripple/app/misc/SHAMapStore.h index 742e7541e..ad85c87cd 100644 --- a/src/ripple/app/misc/SHAMapStore.h +++ b/src/ripple/app/misc/SHAMapStore.h @@ -41,6 +41,7 @@ class SHAMapStore public: struct Setup { + bool standalone = false; std::uint32_t deleteInterval = 0; bool advisoryDelete = false; std::uint32_t ledgerHistory = 0; diff --git a/src/ripple/app/misc/SHAMapStoreImp.cpp b/src/ripple/app/misc/SHAMapStoreImp.cpp index 41ff380e9..3eeb001a9 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.cpp +++ b/src/ripple/app/misc/SHAMapStoreImp.cpp @@ -29,6 +29,7 @@ #include #include #include +#include namespace ripple { void SHAMapStoreImp::SavedStateDB::init (BasicConfig const& config, @@ -179,21 +180,24 @@ SHAMapStoreImp::SHAMapStoreImp ( , scheduler_ (scheduler) , journal_ (journal) , nodeStoreJournal_ (nodeStoreJournal) + , rotating_(false) , transactionMaster_ (transactionMaster) , canDelete_ (std::numeric_limits ::max()) { if (setup_.deleteInterval) { - if (setup_.deleteInterval < minimumDeletionInterval_) + auto const minInterval = setup.standalone ? + minimumDeletionIntervalSA_ : minimumDeletionInterval_; + if (setup_.deleteInterval < minInterval) { Throw ("online_delete must be at least " + - std::to_string (minimumDeletionInterval_)); + std::to_string (minInterval)); } if (setup_.ledgerHistory > setup_.deleteInterval) { Throw ( - "online_delete must be less than ledger_history (currently " + + "online_delete must not be less than ledger_history (currently " + std::to_string (setup_.ledgerHistory) + ")"); } @@ -283,7 +287,8 @@ SHAMapStoreImp::run() while (1) { healthy_ = true; - validatedLedger_.reset(); + std::shared_ptr validatedLedger; + rotating_ = false; { std::unique_lock lock (mutex_); @@ -294,12 +299,15 @@ SHAMapStoreImp::run() } cond_.wait (lock); if (newLedger_) - validatedLedger_ = std::move (newLedger_); + { + rotating_ = true; + validatedLedger = std::move(newLedger_); + } else continue; } - LedgerIndex validatedSeq = validatedLedger_->info().seq; + LedgerIndex validatedSeq = validatedLedger->info().seq; if (!lastRotated) { lastRotated = validatedSeq; @@ -340,7 +348,7 @@ SHAMapStoreImp::run() } std::uint64_t nodeCount = 0; - validatedLedger_->stateMap().snapShot ( + validatedLedger->stateMap().snapShot ( false)->visitNodes ( std::bind (&SHAMapStoreImp::copyNode, this, std::ref(nodeCount), std::placeholders::_1)); @@ -504,7 +512,7 @@ SHAMapStoreImp::makeDatabaseRotating (std::string const& name, readThreads, writableBackend, archiveBackend, nodeStoreJournal_); } -void +bool SHAMapStoreImp::clearSql (DatabaseCon& database, LedgerIndex lastRotated, std::string const& minQuery, @@ -517,12 +525,12 @@ SHAMapStoreImp::clearSql (DatabaseCon& database, boost::optional m; *db << minQuery, soci::into(m); if (!m) - return; + return false; min = *m; } - if (health() != Health::ok) - return; + if(min > lastRotated || health() != Health::ok) + return false; boost::format formattedDeleteQuery (deleteQuery); @@ -530,19 +538,19 @@ SHAMapStoreImp::clearSql (DatabaseCon& database, "start: " << deleteQuery << " from " << min << " to " << lastRotated; while (min < lastRotated) { - min = (min + setup_.deleteBatch >= lastRotated) ? lastRotated : - min + setup_.deleteBatch; + min = std::min(lastRotated, min + setup_.deleteBatch); { auto db = database.checkoutDb (); *db << boost::str (formattedDeleteQuery % min); } if (health()) - return; + return true; if (min < lastRotated) std::this_thread::sleep_for ( std::chrono::milliseconds (setup_.backOff)); } JLOG(journal_.debug) << "finished: " << deleteQuery; + return true; } void @@ -566,25 +574,10 @@ SHAMapStoreImp::freshenCaches() void SHAMapStoreImp::clearPrior (LedgerIndex lastRotated) { - ledgerMaster_->clearPriorLedgers (lastRotated); if (health()) return; - // TODO This won't remove validations for ledgers that do not get - // validated. That will likely require inserting LedgerSeq into - // the validations table. - // - // This query has poor performance with large data sets. - // The schema needs to be redesigned to avoid the JOIN, or an - // RDBMS that supports concurrency should be used. - /* - clearSql (*ledgerDb_, lastRotated, - "SELECT MIN(LedgerSeq) FROM Ledgers;", - "DELETE FROM Validations WHERE LedgerHash IN " - "(SELECT Ledgers.LedgerHash FROM Validations JOIN Ledgers ON " - "Validations.LedgerHash=Ledgers.LedgerHash WHERE Ledgers.LedgerSeq < %u);"); - */ - + ledgerMaster_->clearPriorLedgers (lastRotated); if (health()) return; @@ -594,6 +587,118 @@ SHAMapStoreImp::clearPrior (LedgerIndex lastRotated) if (health()) return; + { + /* + Steps for migration: + Assume: online_delete = 100, lastRotated = 1000, + Last shutdown was at ledger # 1080. + The current network validated ledger is 1090. + Implies: Ledgers has entries from 900 to 1080. + Validations has entries for all 1080 ledgers, + including orphan validations that were not included + in a validated ledger. + 1) Columns are created in Validations with default NULL values. + 2) During syncing, Ledgers and Validations for 1080 - 1090 + are received from the network. Records are created in + Validations with InitialSeq approximately 1080 (exact value + doesn't matter), and later validated with the matching + LedgerSeq value. + 3) rippled participates in ledgers 1091-1100. Validations + received are created with InitialSeq in that range, and + appropriate LedgerSeqs. Maybe some of those ledgers are + not accepted, so LedgerSeq stays null. + 4) At ledger 1100, this function is called with + lastRotated = 1000. The first query tries to delete + rows WHERE LedgerSeq < 1000. It finds none. + 5) The second round of deletions does not run. + 6) Ledgers continue to advance from 1100-1200 as described + in step 3. + 7) At ledger 1200, this function is called again with + lastRotated = 1100. The first query again tries to delete + rows WHERE LedgerSeq < 1100. It finds the rows for 1080-1099. + 8) The second round of deletions runs. It gets + WHERE v.LedgerSeq is NULL AND + (v.InitialSeq IS NULL OR v.InitialSeq < 1100) + The rows that are found include (a) ALL of the Validations + for the first 1080 ledgers. (b) Any orphan validations that + were created in step 3. + 9) This continues. The next rotation cycle does the same as steps + 7 & 8, except that none of the original Validations (8a) exist + anymore, and 8b gets the orphans from step 6. + */ + + static auto anyValDeleted = false; + auto const valDeleted = clearSql(*ledgerDb_, lastRotated, + "SELECT MIN(LedgerSeq) FROM Validations;", + "DELETE FROM Validations WHERE LedgerSeq < %u;"); + anyValDeleted |= valDeleted; + + if (health()) + return; + + if (anyValDeleted) + { + /* Delete the old NULL LedgerSeqs - the Validations that + aren't linked to a validated ledger - but only if we + deleted rows in the matching `clearSql` call, and only + for those created with an old InitialSeq. + */ + using namespace std::chrono; + auto const deleteBatch = setup_.deleteBatch; + auto const continueLimit = (deleteBatch + 1) / 2; + + std::string const deleteQuery( + R"sql(DELETE FROM Validations + WHERE LedgerHash IN + ( + SELECT v.LedgerHash + FROM Validations v + WHERE v.LedgerSeq is NULL AND + (v.InitialSeq IS NULL OR v.InitialSeq < )sql" + + std::to_string(lastRotated) + + ") LIMIT " + + std::to_string (deleteBatch) + + ");"); + + JLOG(journal_.debug) << "start: " << deleteQuery << " of " + << deleteBatch << " rows."; + long long totalRowsAffected = 0; + long long rowsAffected; + soci::statement st = [&] + { + auto db = ledgerDb_->checkoutDb(); + return (db->prepare << deleteQuery); + }(); + if (health()) + return; + do + { + { + auto db = ledgerDb_->checkoutDb(); + auto const start = high_resolution_clock::now(); + st.execute(true); + rowsAffected = st.get_affected_rows(); + totalRowsAffected += rowsAffected; + auto const ms = duration_cast( + high_resolution_clock::now() - start).count(); + JLOG(journal_.trace) << "step: deleted " << rowsAffected + << " rows in " << ms << "ms."; + } + if (health()) + return; + if (rowsAffected >= continueLimit) + std::this_thread::sleep_for( + std::chrono::milliseconds(setup_.backOff)); + } + while (rowsAffected && rowsAffected >= continueLimit); + JLOG(journal_.debug) << "finished: " << deleteQuery << ". Deleted " + << totalRowsAffected << " rows."; + } + } + + if (health()) + return; + clearSql (*transactionDb_, lastRotated, "SELECT MIN(LedgerSeq) FROM Transactions;", "DELETE FROM Transactions WHERE LedgerSeq < %u;"); @@ -675,6 +780,8 @@ setup_SHAMapStore (Config const& c) { SHAMapStore::Setup setup; + setup.standalone = c.RUN_STANDALONE; + // Get existing settings and add some default values if not specified: setup.nodeDatabase = c.section (ConfigSection::nodeDatabase ()); diff --git a/src/ripple/app/misc/SHAMapStoreImp.h b/src/ripple/app/misc/SHAMapStoreImp.h index db38a62d1..797db777c 100644 --- a/src/ripple/app/misc/SHAMapStoreImp.h +++ b/src/ripple/app/misc/SHAMapStoreImp.h @@ -80,7 +80,9 @@ private: // check health/stop status as records are copied std::uint64_t const checkHealthInterval_ = 1000; // minimum # of ledgers to maintain for health of network - std::uint32_t minimumDeletionInterval_ = 256; + static std::uint32_t const minimumDeletionInterval_ = 256; + // minimum # of ledgers required for standalone mode. + static std::uint32_t const minimumDeletionIntervalSA_ = 8; Setup setup_; NodeStore::Scheduler& scheduler_; @@ -94,7 +96,7 @@ private: mutable std::condition_variable cond_; mutable std::mutex mutex_; std::shared_ptr newLedger_; - std::shared_ptr validatedLedger_; + std::atomic rotating_; TransactionMaster& transactionMaster_; std::atomic canDelete_; // these do not exist upon SHAMapStore creation, but do exist @@ -106,6 +108,12 @@ private: DatabaseCon* transactionDb_ = nullptr; DatabaseCon* ledgerDb_ = nullptr; +public: + bool rotating() const + { + return rotating_; + } + public: SHAMapStoreImp (Application& app, Setup const& setup, @@ -203,8 +211,10 @@ private: /** delete from sqlite table in batches to not lock the db excessively * pause briefly to extend access time to other users * call with mutex object unlocked + * @return true if any deletable rows were found (though not + * necessarily deleted. */ - void clearSql (DatabaseCon& database, LedgerIndex lastRotated, + bool clearSql (DatabaseCon& database, LedgerIndex lastRotated, std::string const& minQuery, std::string const& deleteQuery); void clearCaches (LedgerIndex validatedSeq); void freshenCaches(); diff --git a/src/ripple/app/misc/Validations.cpp b/src/ripple/app/misc/Validations.cpp index 856536529..dcf241717 100644 --- a/src/ripple/app/misc/Validations.cpp +++ b/src/ripple/app/misc/Validations.cpp @@ -461,8 +461,10 @@ private: void doWrite () { LoadEvent::autoptr event (app_.getJobQueue ().getLoadEventAP (jtDISK, "ValidationWrite")); - boost::format insVal ("INSERT INTO Validations " - "(LedgerHash,NodePubKey,SignTime,RawData) VALUES ('%s','%s','%u',%s);"); + std::string insVal ("INSERT INTO Validations " + "(InitialSeq, LedgerSeq, LedgerHash,NodePubKey,SignTime,RawData) " + "VALUES (:initialSeq, :ledgerSeq, :ledgerHash,:nodePubKey,:signTime,:rawData);"); + std::string findSeq("SELECT LedgerSeq FROM Ledgers WHERE Ledgerhash=:ledgerHash;"); ScopedLockType sl (mLock); assert (mWriting); @@ -484,13 +486,34 @@ private: { s.erase (); it->add (s); - *db << boost::str ( - insVal % to_string (it->getLedgerHash ()) % - toBase58( - TokenType::TOKEN_NODE_PUBLIC, - it->getSignerPublic ()) % - it->getSignTime().time_since_epoch().count() % - sqlEscape (s.peekData ())); + + auto const ledgerHash = to_string(it->getLedgerHash()); + + boost::optional ledgerSeq; + *db << findSeq, soci::use(ledgerHash), + soci::into(ledgerSeq); + + auto const initialSeq = ledgerSeq.value_or( + app_.getLedgerMaster().getCurrentLedgerIndex()); + auto const nodePubKey = toBase58( + TokenType::TOKEN_NODE_PUBLIC, + it->getSignerPublic()); + auto const signTime = + it->getSignTime().time_since_epoch().count(); + + soci::blob rawData(*db); + rawData.append(reinterpret_cast( + s.peekData().data()), s.peekData().size()); + assert(rawData.get_len() == s.peekData().size()); + + *db << + insVal, + soci::use(initialSeq), + soci::use(ledgerSeq), + soci::use(ledgerHash), + soci::use(nodePubKey), + soci::use(signTime), + soci::use(rawData); } tr.commit (); diff --git a/src/ripple/app/tests/SHAMapStore_test.cpp b/src/ripple/app/tests/SHAMapStore_test.cpp new file mode 100644 index 000000000..4060df43c --- /dev/null +++ b/src/ripple/app/tests/SHAMapStore_test.cpp @@ -0,0 +1,654 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012-2015 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include +#include +#include + +namespace ripple { +namespace test { + +class SHAMapStore_test : public beast::unit_test::suite +{ + static auto const deleteInterval = 8; + + static + std::unique_ptr + makeConfig() + { + auto p = std::make_unique(); + setupConfigForUnitTests(*p); + p->LEDGER_HISTORY = deleteInterval; + auto& section = p->section(ConfigSection::nodeDatabase()); + section.set("online_delete", to_string(deleteInterval)); + //section.set("age_threshold", "60"); + return p; + } + + static + std::unique_ptr + makeConfigAdvisory() + { + auto p = makeConfig(); + auto& section = p->section(ConfigSection::nodeDatabase()); + section.set("advisory_delete", "1"); + return p; + } + + bool goodLedger(jtx::Env& env, Json::Value const& json, + std::string ledgerID, bool checkDB = false) + { + auto good = json.isMember(jss::result) + && !RPC::contains_error(json[jss::result]) + && json[jss::result][jss::ledger][jss::ledger_index] == ledgerID; + if (!good || !checkDB) + return good; + + auto const seq = json[jss::result][jss::ledger_index].asUInt(); + std::string outHash; + LedgerIndex outSeq; + std::string outParentHash; + std::string outDrops; + std::uint64_t outCloseTime; + std::uint64_t outParentCloseTime; + std::uint64_t outCloseTimeResolution; + std::uint64_t outCloseFlags; + std::string outAccountHash; + std::string outTxHash; + + { + auto db = env.app().getLedgerDB().checkoutDb(); + + *db << "SELECT LedgerHash,LedgerSeq,PrevHash,TotalCoins, " + "ClosingTime,PrevClosingTime,CloseTimeRes,CloseFlags, " + "AccountSetHash,TransSetHash " + "FROM Ledgers " + "WHERE LedgerSeq = :seq", + soci::use(seq), + soci::into(outHash), + soci::into(outSeq), + soci::into(outParentHash), + soci::into(outDrops), + soci::into(outCloseTime), + soci::into(outParentCloseTime), + soci::into(outCloseTimeResolution), + soci::into(outCloseFlags), + soci::into(outAccountHash), + soci::into(outTxHash); + } + + auto const& ledger = json[jss::result][jss::ledger]; + return outHash == ledger[jss::hash].asString() && + outSeq == seq && + outParentHash == ledger[jss::parent_hash].asString() && + outDrops == ledger[jss::total_coins].asString() && + outCloseTime == ledger[jss::close_time].asUInt() && + outParentCloseTime == ledger[jss::parent_close_time].asUInt() && + outCloseTimeResolution == ledger[jss::close_time_resolution].asUInt() && + outCloseFlags == ledger[jss::close_flags].asUInt() && + outAccountHash == ledger[jss::account_hash].asString() && + outTxHash == ledger[jss::transaction_hash].asString(); + } + + bool bad(Json::Value const& json, error_code_i error = rpcLGR_NOT_FOUND) + { + return json.isMember(jss::result) + && RPC::contains_error(json[jss::result]) + && json[jss::result][jss::error_code] == error; + } + + std::string getHash(Json::Value const& json) + { + expect(json.isMember(jss::result) && + json[jss::result].isMember(jss::ledger) && + json[jss::result][jss::ledger].isMember(jss::hash) && + json[jss::result][jss::ledger][jss::hash].isString()); + return json[jss::result][jss::ledger][jss::hash].asString(); + } + + void validationCheck(jtx::Env& env, int const expected) + { + auto db = env.app().getLedgerDB().checkoutDb(); + + int actual; + *db << "SELECT count(*) AS rows FROM Validations;", + soci::into(actual); + + expect(actual == expected); + + } + + void ledgerCheck(jtx::Env& env, int const rows, + int const first) + { + auto db = env.app().getLedgerDB().checkoutDb(); + + int actualRows, actualFirst, actualLast; + *db << "SELECT count(*) AS rows, " + "min(LedgerSeq) as first, " + "max(LedgerSeq) as last " + "FROM Ledgers;", + soci::into(actualRows), + soci::into(actualFirst), + soci::into(actualLast); + + expect(actualRows == rows); + expect(actualFirst == first); + expect(actualLast == first + rows - 1); + + } + + void transactionCheck(jtx::Env& env, int const rows) + { + auto db = env.app().getTxnDB().checkoutDb(); + + int actualRows; + *db << "SELECT count(*) AS rows " + "FROM Transactions;", + soci::into(actualRows); + + expect(actualRows == rows); + } + + void accountTransactionCheck(jtx::Env& env, int const rows) + { + auto db = env.app().getTxnDB().checkoutDb(); + + int actualRows; + *db << "SELECT count(*) AS rows " + "FROM AccountTransactions;", + soci::into(actualRows); + + expect(actualRows == rows); + } + + int waitForReady(jtx::Env& env) + { + using namespace std::chrono_literals; + + auto& store = env.app().getSHAMapStore(); + + int ledgerSeq = 3; + while (!store.getLastRotated()) + { + env.close(); + std::this_thread::sleep_for(100ms); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq++))); + } + return ledgerSeq; + } + +public: + void testClear() + { + using namespace std::chrono_literals; + + testcase("clearPrior"); + using namespace jtx; + + Env env(*this, makeConfig()); + + auto store = dynamic_cast( + &env.app().getSHAMapStore()); + expect(store); + env.fund(XRP(10000), noripple("alice")); + + validationCheck(env, 0); + ledgerCheck(env, 1, 2); + transactionCheck(env, 0); + accountTransactionCheck(env, 0); + + std::map ledgers; + + auto ledgerTmp = env.rpc("ledger", "0"); + expect(bad(ledgerTmp)); + + ledgers.emplace(std::make_pair(1, env.rpc("ledger", "1"))); + expect(goodLedger(env, ledgers[1], "1")); + + ledgers.emplace(std::make_pair(2, env.rpc("ledger", "2"))); + expect(goodLedger(env, ledgers[2], "2")); + + ledgerTmp = env.rpc("ledger", "current"); + expect(goodLedger(env, ledgerTmp, "3")); + + ledgerTmp = env.rpc("ledger", "4"); + expect(bad(ledgerTmp)); + + ledgerTmp = env.rpc("ledger", "100"); + expect(bad(ledgerTmp)); + + for (auto i = 4; i < deleteInterval + 4; ++i) + { + env.fund(XRP(10000), noripple("test" + to_string(i))); + env.close(); + + ledgerTmp = env.rpc("ledger", "current"); + expect(goodLedger(env, ledgerTmp, to_string(i))); + } + assert(store->getLastRotated() == 3); + + for (auto i = 3; i < deleteInterval + 3; ++i) + { + ledgers.emplace(std::make_pair(i, + env.rpc("ledger", to_string(i)))); + expect(goodLedger(env, ledgers[i], to_string(i), true) && + getHash(ledgers[i]).length()); + } + + validationCheck(env, 0); + ledgerCheck(env, deleteInterval + 1, 2); + transactionCheck(env, deleteInterval + 1); + accountTransactionCheck(env, 2*(deleteInterval + 1)); + + { + // Since standalone doesn't _do_ validations, manually + // insert some into the table. Create some with the + // hashes from our real ledgers, and some with fake + // hashes to represent validations that never ended up + // in a validated ledger. + char lh[65]; + memset(lh, 'a', 64); + lh[64] = '\0'; + std::vector preSeqLedgerHashes({ + lh + }); + std::vector badLedgerHashes; + std::vector badLedgerSeqs; + std::vector ledgerHashes; + std::vector ledgerSeqs; + for (auto const& lgr : ledgers) + { + ledgerHashes.emplace_back(getHash(lgr.second)); + ledgerSeqs.emplace_back(lgr.second[jss::result][jss::ledger_index].asUInt()); + } + for (auto i = 0; i < 10; ++i) + { + ++lh[30]; + preSeqLedgerHashes.emplace_back(lh); + ++lh[20]; + badLedgerHashes.emplace_back(lh); + badLedgerSeqs.emplace_back(i + 1); + } + + auto db = env.app().getLedgerDB().checkoutDb(); + + // Pre-migration validation - no sequence numbers. + *db << "INSERT INTO Validations " + "(LedgerHash) " + "VALUES " + "(:ledgerHash);", + soci::use(preSeqLedgerHashes); + // Post-migration orphan validation - InitalSeq, + // but no LedgerSeq + *db << "INSERT INTO Validations " + "(LedgerHash, InitialSeq) " + "VALUES " + "(:ledgerHash, :initialSeq);", + soci::use(badLedgerHashes), + soci::use(badLedgerSeqs); + // Post-migration validated ledger. + *db << "INSERT INTO Validations " + "(LedgerHash, LedgerSeq) " + "VALUES " + "(:ledgerHash, :ledgerSeq);", + soci::use(ledgerHashes), + soci::use(ledgerSeqs); + } + + validationCheck(env, deleteInterval + 23); + ledgerCheck(env, deleteInterval + 1, 2); + transactionCheck(env, deleteInterval + 1); + accountTransactionCheck(env, 2 * (deleteInterval + 1)); + + { + // Closing one more ledger triggers a rotate + env.close(); + + auto ledger = env.rpc("ledger", "current"); + expect(goodLedger(env, ledger, to_string(deleteInterval + 4))); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + expect(store->getLastRotated() == deleteInterval + 3); + auto const lastRotated = store->getLastRotated(); + expect(lastRotated == 11, to_string(lastRotated)); + + // That took care of the fake hashes + validationCheck(env, deleteInterval + 8); + ledgerCheck(env, deleteInterval + 1, 3); + transactionCheck(env, deleteInterval + 1); + accountTransactionCheck(env, 2 * (deleteInterval + 1)); + + // The last iteration of this loop should trigger a rotate + for (auto i = lastRotated - 1; i < lastRotated + deleteInterval - 1; ++i) + { + validationCheck(env, deleteInterval + i + 1 - lastRotated + 8); + + env.close(); + + ledgerTmp = env.rpc("ledger", "current"); + expect(goodLedger(env, ledgerTmp, to_string(i + 3))); + + ledgers.emplace(std::make_pair(i, + env.rpc("ledger", to_string(i)))); + expect(store->getLastRotated() == lastRotated || + i == lastRotated + deleteInterval - 2, "lastRotated"); + expect(goodLedger(env, ledgers[i], to_string(i), true) && + getHash(ledgers[i]).length(), to_string(ledgers[i])); + + std::vector ledgerHashes({ + getHash(ledgers[i]) + }); + std::vector ledgerSeqs({ + ledgers[i][jss::result][jss::ledger_index].asUInt() + }); + auto db = env.app().getLedgerDB().checkoutDb(); + + *db << "INSERT INTO Validations " + "(LedgerHash, LedgerSeq) " + "VALUES " + "(:ledgerHash, :ledgerSeq);", + soci::use(ledgerHashes), + soci::use(ledgerSeqs); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + expect(store->getLastRotated() == deleteInterval + lastRotated); + + validationCheck(env, deleteInterval - 1); + ledgerCheck(env, deleteInterval + 1, lastRotated); + transactionCheck(env, 0); + accountTransactionCheck(env, 0); + + } + + void testAutomatic() + { + testcase("automatic online_delete"); + using namespace jtx; + using namespace std::chrono_literals; + + Env env(*this, makeConfig()); + auto store = dynamic_cast( + &env.app().getSHAMapStore()); + expect(store); + + auto ledgerSeq = waitForReady(env); + auto lastRotated = ledgerSeq - 1; + expect(store->getLastRotated() == lastRotated, + to_string(store->getLastRotated())); + expect(lastRotated != 2); + + // Because advisory_delete is unset, + // "can_delete" is disabled. + auto const canDelete = env.rpc("can_delete"); + expect(bad(canDelete, rpcNOT_ENABLED)); + + // Close ledgers without triggering a rotate + for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq) + { + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq), true)); + } + + while(store->rotating()) + std::this_thread::sleep_for(1ms); + + // The database will always have back to ledger 2, + // regardless of lastRotated. + validationCheck(env, 0); + ledgerCheck(env, ledgerSeq - 2, 2); + expect(lastRotated == store->getLastRotated()); + + { + // Closing one more ledger triggers a rotate + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq++), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + validationCheck(env, 0); + ledgerCheck(env, ledgerSeq - lastRotated, lastRotated); + expect(lastRotated != store->getLastRotated()); + + lastRotated = store->getLastRotated(); + + // Close enough ledgers to trigger another rotate + for (; ledgerSeq < lastRotated + deleteInterval + 1; ++ledgerSeq) + { + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + validationCheck(env, 0); + ledgerCheck(env, deleteInterval + 1, lastRotated); + expect(lastRotated != store->getLastRotated()); + } + + void testCanDelete() + { + testcase("online_delete with advisory_delete"); + using namespace jtx; + using namespace std::chrono_literals; + + // Same config with advisory_delete enabled + Env env(*this, makeConfigAdvisory()); + auto store = dynamic_cast( + &env.app().getSHAMapStore()); + expect(store); + + auto ledgerSeq = waitForReady(env); + auto lastRotated = ledgerSeq - 1; + expect(store->getLastRotated() == lastRotated, + to_string(store->getLastRotated())); + expect(lastRotated != 2); + + auto canDelete = env.rpc("can_delete"); + expect(!RPC::contains_error(canDelete[jss::result])); + expect(canDelete[jss::result][jss::can_delete] == 0); + + canDelete = env.rpc("can_delete", "never"); + expect(!RPC::contains_error(canDelete[jss::result])); + expect(canDelete[jss::result][jss::can_delete] == 0); + + auto const firstBatch = deleteInterval + ledgerSeq; + for (; ledgerSeq < firstBatch; ++ledgerSeq) + { + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + validationCheck(env, 0); + ledgerCheck(env, ledgerSeq - 2, 2); + expect(lastRotated == store->getLastRotated()); + + // This does not kick off a cleanup + canDelete = env.rpc("can_delete", to_string( + ledgerSeq + deleteInterval / 2)); + expect(!RPC::contains_error(canDelete[jss::result])); + expect(canDelete[jss::result][jss::can_delete] == + ledgerSeq + deleteInterval / 2); + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + validationCheck(env, 0); + ledgerCheck(env, ledgerSeq - 2, 2); + expect(store->getLastRotated() == lastRotated); + + { + // This kicks off a cleanup, but it stays small. + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq++), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + validationCheck(env, 0); + ledgerCheck(env, ledgerSeq - lastRotated, lastRotated); + + expect(store->getLastRotated() == ledgerSeq - 1); + lastRotated = ledgerSeq - 1; + + for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq) + { + // No cleanups in this loop. + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + expect(store->getLastRotated() == lastRotated); + + { + // This kicks off another cleanup. + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq++), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + validationCheck(env, 0); + ledgerCheck(env, ledgerSeq - firstBatch, firstBatch); + + expect(store->getLastRotated() == ledgerSeq - 1); + lastRotated = ledgerSeq - 1; + + // This does not kick off a cleanup + canDelete = env.rpc("can_delete", "always"); + expect(!RPC::contains_error(canDelete[jss::result])); + expect(canDelete[jss::result][jss::can_delete] == + std::numeric_limits ::max()); + + for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq) + { + // No cleanups in this loop. + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + expect(store->getLastRotated() == lastRotated); + + { + // This kicks off another cleanup. + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq++), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + validationCheck(env, 0); + ledgerCheck(env, ledgerSeq - lastRotated, lastRotated); + + expect(store->getLastRotated() == ledgerSeq - 1); + lastRotated = ledgerSeq - 1; + + // This does not kick off a cleanup + canDelete = env.rpc("can_delete", "now"); + expect(!RPC::contains_error(canDelete[jss::result])); + expect(canDelete[jss::result][jss::can_delete] == ledgerSeq - 1); + + for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq) + { + // No cleanups in this loop. + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + expect(store->getLastRotated() == lastRotated); + + { + // This kicks off another cleanup. + env.close(); + + auto ledger = env.rpc("ledger", "validated"); + expect(goodLedger(env, ledger, to_string(ledgerSeq++), true)); + } + + while (store->rotating()) + std::this_thread::sleep_for(1ms); + + validationCheck(env, 0); + ledgerCheck(env, ledgerSeq - lastRotated, lastRotated); + + expect(store->getLastRotated() == ledgerSeq - 1); + lastRotated = ledgerSeq - 1; + } + + void run() + { + testClear(); + testAutomatic(); + testCanDelete(); + } +}; + +BEAST_DEFINE_TESTSUITE(SHAMapStore,app,ripple); + +} +} diff --git a/src/ripple/core/impl/DatabaseCon.cpp b/src/ripple/core/impl/DatabaseCon.cpp index 4b3797932..ec44ed4b2 100644 --- a/src/ripple/core/impl/DatabaseCon.cpp +++ b/src/ripple/core/impl/DatabaseCon.cpp @@ -46,7 +46,9 @@ DatabaseCon::DatabaseCon ( { try { - session_ << initStrings[i]; + soci::statement st = session_.prepare << + initStrings[i]; + st.execute(true); } catch (soci::soci_error&) { diff --git a/src/ripple/rpc/handlers/CanDelete.cpp b/src/ripple/rpc/handlers/CanDelete.cpp index fa15f918c..9ae1b5f3b 100644 --- a/src/ripple/rpc/handlers/CanDelete.cpp +++ b/src/ripple/rpc/handlers/CanDelete.cpp @@ -70,10 +70,11 @@ Json::Value doCanDelete (RPC::Context& context) { canDeleteSeq = context.app.getSHAMapStore().getLastRotated(); if (!canDeleteSeq) - return RPC::make_error (rpcNOT_READY); } - else if (canDeleteStr.size() == 64 && - canDeleteStr.find_first_not_of("0123456789abcdef") == - std::string::npos) + return RPC::make_error (rpcNOT_READY); + } + else if (canDeleteStr.size() == 64 && + canDeleteStr.find_first_not_of("0123456789abcdef") == + std::string::npos) { auto ledger = context.ledgerMaster.getLedgerByHash ( from_hex_text(canDeleteStr)); diff --git a/src/ripple/unity/app_tests.cpp b/src/ripple/unity/app_tests.cpp index ab7d17a15..973497643 100644 --- a/src/ripple/unity/app_tests.cpp +++ b/src/ripple/unity/app_tests.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include