diff --git a/Builds/VisualStudio2015/RippleD.vcxproj b/Builds/VisualStudio2015/RippleD.vcxproj
index 5b8eeb824..4d66dc499 100644
--- a/Builds/VisualStudio2015/RippleD.vcxproj
+++ b/Builds/VisualStudio2015/RippleD.vcxproj
@@ -1531,6 +1531,10 @@
True
True
+
+ True
+ True
+
True
True
diff --git a/Builds/VisualStudio2015/RippleD.vcxproj.filters b/Builds/VisualStudio2015/RippleD.vcxproj.filters
index ad78f1211..9b6454663 100644
--- a/Builds/VisualStudio2015/RippleD.vcxproj.filters
+++ b/Builds/VisualStudio2015/RippleD.vcxproj.filters
@@ -2112,6 +2112,9 @@
ripple\app\tests
+
+ ripple\app\tests
+
ripple\app\tests
diff --git a/src/ripple/app/ledger/Ledger.cpp b/src/ripple/app/ledger/Ledger.cpp
index 3e163dc08..f54d34796 100644
--- a/src/ripple/app/ledger/Ledger.cpp
+++ b/src/ripple/app/ledger/Ledger.cpp
@@ -890,16 +890,6 @@ static bool saveValidatedLedger (
"DELETE FROM AccountTransactions WHERE LedgerSeq = %u;");
static boost::format deleteAcctTrans (
"DELETE FROM AccountTransactions WHERE TransID = '%s';");
- static boost::format transExists (
- "SELECT Status FROM Transactions WHERE TransID = '%s';");
- static boost::format updateTx (
- "UPDATE Transactions SET LedgerSeq = %u, Status = '%c', TxnMeta = %s "
- "WHERE TransID = '%s';");
- static boost::format addLedger (
- "INSERT OR REPLACE INTO Ledgers "
- "(LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime,"
- "CloseTimeRes,CloseFlags,AccountSetHash,TransSetHash) VALUES "
- "('%s','%u','%s','%s','%u','%u','%d','%u','%s','%s');");
auto seq = ledger->info().seq;
@@ -1032,18 +1022,53 @@ static bool saveValidatedLedger (
}
{
+ static std::string addLedger(
+ R"sql(INSERT OR REPLACE INTO Ledgers
+ (LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime,
+ CloseTimeRes,CloseFlags,AccountSetHash,TransSetHash)
+ VALUES
+ (:ledgerHash,:ledgerSeq,:prevHash,:totalCoins,:closingTime,:prevClosingTime,
+ :closeTimeRes,:closeFlags,:accountSetHash,:transSetHash);)sql");
+ static std::string updateVal(
+ R"sql(UPDATE Validations SET LedgerSeq = :ledgerSeq, InitialSeq = :initialSeq
+ WHERE LedgerHash = :ledgerHash;)sql");
+
auto db (app.getLedgerDB ().checkoutDb ());
- // TODO(tom): ARG!
- *db << boost::str (
- addLedger %
- to_string (ledger->info().hash) % seq % to_string (ledger->info().parentHash) %
- to_string (ledger->info().drops) %
- ledger->info().closeTime.time_since_epoch().count() %
- ledger->info().parentCloseTime.time_since_epoch().count() %
- ledger->info().closeTimeResolution.count() %
- ledger->info().closeFlags % to_string (ledger->info().accountHash) %
- to_string (ledger->info().txHash));
+ soci::transaction tr(*db);
+
+ auto const hash = to_string (ledger->info().hash);
+ auto const parentHash = to_string (ledger->info().parentHash);
+ auto const drops = to_string (ledger->info().drops);
+ auto const closeTime =
+ ledger->info().closeTime.time_since_epoch().count();
+ auto const parentCloseTime =
+ ledger->info().parentCloseTime.time_since_epoch().count();
+ auto const closeTimeResolution =
+ ledger->info().closeTimeResolution.count();
+ auto const closeFlags = ledger->info().closeFlags;
+ auto const accountHash = to_string (ledger->info().accountHash);
+ auto const txHash = to_string (ledger->info().txHash);
+
+ *db << addLedger,
+ soci::use(hash),
+ soci::use(seq),
+ soci::use(parentHash),
+ soci::use(drops),
+ soci::use(closeTime),
+ soci::use(parentCloseTime),
+ soci::use(closeTimeResolution),
+ soci::use(closeFlags),
+ soci::use(accountHash),
+ soci::use(txHash);
+
+
+ *db << updateVal,
+ soci::use(seq),
+ soci::use(seq),
+ soci::use(hash);
+
+ tr.commit();
}
// Clients can now trust the database for
diff --git a/src/ripple/app/ledger/LedgerHistory.cpp b/src/ripple/app/ledger/LedgerHistory.cpp
index 714392b91..0bdaa048c 100644
--- a/src/ripple/app/ledger/LedgerHistory.cpp
+++ b/src/ripple/app/ledger/LedgerHistory.cpp
@@ -499,7 +499,9 @@ void LedgerHistory::clearLedgerCachePrior (LedgerIndex seq)
{
for (LedgerHash it: m_ledgers_by_hash.getKeys())
{
- if (getLedgerByHash (it)->info().seq < seq)
+ auto const ledger = getLedgerByHash (it);
+ assert(ledger);
+ if (!ledger || ledger->info().seq < seq)
m_ledgers_by_hash.del (it, false);
}
}
diff --git a/src/ripple/app/ledger/impl/LedgerMaster.cpp b/src/ripple/app/ledger/impl/LedgerMaster.cpp
index 244b855cd..9144a6960 100644
--- a/src/ripple/app/ledger/impl/LedgerMaster.cpp
+++ b/src/ripple/app/ledger/impl/LedgerMaster.cpp
@@ -984,7 +984,7 @@ LedgerMaster::getLedgerHashForHistory (LedgerIndex index)
if (! ret)
ret = walkHashBySeq (index);
- return *ret;
+ return ret;
}
bool
diff --git a/src/ripple/app/main/DBInit.cpp b/src/ripple/app/main/DBInit.cpp
index 96e9824e9..2a404fa3d 100644
--- a/src/ripple/app/main/DBInit.cpp
+++ b/src/ripple/app/main/DBInit.cpp
@@ -90,14 +90,29 @@ const char* LedgerDBInit[] =
);",
"CREATE INDEX IF NOT EXISTS SeqLedger ON Ledgers(LedgerSeq);",
+ // InitialSeq field is the current ledger seq when the row
+ // is inserted. Only relevant during online delete
"CREATE TABLE IF NOT EXISTS Validations ( \
+ LedgerSeq BIGINT UNSIGNED, \
+ InitialSeq BIGINT UNSIGNED, \
LedgerHash CHARACTER(64), \
NodePubKey CHARACTER(56), \
SignTime BIGINT UNSIGNED, \
RawData BLOB \
);",
+ // This will error out if the column already exists,
+ // but DatabaseCon intentionally ignores errors.
+ "ALTER TABLE Validations \
+ ADD COLUMN LedgerSeq BIGINT UNSIGNED;",
+ "ALTER TABLE Validations \
+ ADD COLUMN InitialSeq BIGINT UNSIGNED;",
+
"CREATE INDEX IF NOT EXISTS ValidationsByHash ON \
Validations(LedgerHash);",
+ "CREATE INDEX IF NOT EXISTS ValidationsBySeq ON \
+ Validations(LedgerSeq);",
+ "CREATE INDEX IF NOT EXISTS ValidationsByInitialSeq ON \
+ Validations(InitialSeq, LedgerSeq);",
"CREATE INDEX IF NOT EXISTS ValidationsByTime ON \
Validations(SignTime);",
diff --git a/src/ripple/app/misc/SHAMapStore.h b/src/ripple/app/misc/SHAMapStore.h
index 742e7541e..ad85c87cd 100644
--- a/src/ripple/app/misc/SHAMapStore.h
+++ b/src/ripple/app/misc/SHAMapStore.h
@@ -41,6 +41,7 @@ class SHAMapStore
public:
struct Setup
{
+ bool standalone = false;
std::uint32_t deleteInterval = 0;
bool advisoryDelete = false;
std::uint32_t ledgerHistory = 0;
diff --git a/src/ripple/app/misc/SHAMapStoreImp.cpp b/src/ripple/app/misc/SHAMapStoreImp.cpp
index 41ff380e9..3eeb001a9 100644
--- a/src/ripple/app/misc/SHAMapStoreImp.cpp
+++ b/src/ripple/app/misc/SHAMapStoreImp.cpp
@@ -29,6 +29,7 @@
#include
#include
#include
+#include
namespace ripple {
void SHAMapStoreImp::SavedStateDB::init (BasicConfig const& config,
@@ -179,21 +180,24 @@ SHAMapStoreImp::SHAMapStoreImp (
, scheduler_ (scheduler)
, journal_ (journal)
, nodeStoreJournal_ (nodeStoreJournal)
+ , rotating_(false)
, transactionMaster_ (transactionMaster)
, canDelete_ (std::numeric_limits ::max())
{
if (setup_.deleteInterval)
{
- if (setup_.deleteInterval < minimumDeletionInterval_)
+ auto const minInterval = setup.standalone ?
+ minimumDeletionIntervalSA_ : minimumDeletionInterval_;
+ if (setup_.deleteInterval < minInterval)
{
Throw ("online_delete must be at least " +
- std::to_string (minimumDeletionInterval_));
+ std::to_string (minInterval));
}
if (setup_.ledgerHistory > setup_.deleteInterval)
{
Throw (
- "online_delete must be less than ledger_history (currently " +
+ "online_delete must not be less than ledger_history (currently " +
std::to_string (setup_.ledgerHistory) + ")");
}
@@ -283,7 +287,8 @@ SHAMapStoreImp::run()
while (1)
{
healthy_ = true;
- validatedLedger_.reset();
+ std::shared_ptr validatedLedger;
+ rotating_ = false;
{
std::unique_lock lock (mutex_);
@@ -294,12 +299,15 @@ SHAMapStoreImp::run()
}
cond_.wait (lock);
if (newLedger_)
- validatedLedger_ = std::move (newLedger_);
+ {
+ rotating_ = true;
+ validatedLedger = std::move(newLedger_);
+ }
else
continue;
}
- LedgerIndex validatedSeq = validatedLedger_->info().seq;
+ LedgerIndex validatedSeq = validatedLedger->info().seq;
if (!lastRotated)
{
lastRotated = validatedSeq;
@@ -340,7 +348,7 @@ SHAMapStoreImp::run()
}
std::uint64_t nodeCount = 0;
- validatedLedger_->stateMap().snapShot (
+ validatedLedger->stateMap().snapShot (
false)->visitNodes (
std::bind (&SHAMapStoreImp::copyNode, this,
std::ref(nodeCount), std::placeholders::_1));
@@ -504,7 +512,7 @@ SHAMapStoreImp::makeDatabaseRotating (std::string const& name,
readThreads, writableBackend, archiveBackend, nodeStoreJournal_);
}
-void
+bool
SHAMapStoreImp::clearSql (DatabaseCon& database,
LedgerIndex lastRotated,
std::string const& minQuery,
@@ -517,12 +525,12 @@ SHAMapStoreImp::clearSql (DatabaseCon& database,
boost::optional m;
*db << minQuery, soci::into(m);
if (!m)
- return;
+ return false;
min = *m;
}
- if (health() != Health::ok)
- return;
+ if(min > lastRotated || health() != Health::ok)
+ return false;
boost::format formattedDeleteQuery (deleteQuery);
@@ -530,19 +538,19 @@ SHAMapStoreImp::clearSql (DatabaseCon& database,
"start: " << deleteQuery << " from " << min << " to " << lastRotated;
while (min < lastRotated)
{
- min = (min + setup_.deleteBatch >= lastRotated) ? lastRotated :
- min + setup_.deleteBatch;
+ min = std::min(lastRotated, min + setup_.deleteBatch);
{
auto db = database.checkoutDb ();
*db << boost::str (formattedDeleteQuery % min);
}
if (health())
- return;
+ return true;
if (min < lastRotated)
std::this_thread::sleep_for (
std::chrono::milliseconds (setup_.backOff));
}
JLOG(journal_.debug) << "finished: " << deleteQuery;
+ return true;
}
void
@@ -566,25 +574,10 @@ SHAMapStoreImp::freshenCaches()
void
SHAMapStoreImp::clearPrior (LedgerIndex lastRotated)
{
- ledgerMaster_->clearPriorLedgers (lastRotated);
if (health())
return;
- // TODO This won't remove validations for ledgers that do not get
- // validated. That will likely require inserting LedgerSeq into
- // the validations table.
- //
- // This query has poor performance with large data sets.
- // The schema needs to be redesigned to avoid the JOIN, or an
- // RDBMS that supports concurrency should be used.
- /*
- clearSql (*ledgerDb_, lastRotated,
- "SELECT MIN(LedgerSeq) FROM Ledgers;",
- "DELETE FROM Validations WHERE LedgerHash IN "
- "(SELECT Ledgers.LedgerHash FROM Validations JOIN Ledgers ON "
- "Validations.LedgerHash=Ledgers.LedgerHash WHERE Ledgers.LedgerSeq < %u);");
- */
-
+ ledgerMaster_->clearPriorLedgers (lastRotated);
if (health())
return;
@@ -594,6 +587,118 @@ SHAMapStoreImp::clearPrior (LedgerIndex lastRotated)
if (health())
return;
+ {
+ /*
+ Steps for migration:
+ Assume: online_delete = 100, lastRotated = 1000,
+ Last shutdown was at ledger # 1080.
+ The current network validated ledger is 1090.
+ Implies: Ledgers has entries from 900 to 1080.
+ Validations has entries for all 1080 ledgers,
+ including orphan validations that were not included
+ in a validated ledger.
+ 1) Columns are created in Validations with default NULL values.
+ 2) During syncing, Ledgers and Validations for 1080 - 1090
+ are received from the network. Records are created in
+ Validations with InitialSeq approximately 1080 (exact value
+ doesn't matter), and later validated with the matching
+ LedgerSeq value.
+ 3) rippled participates in ledgers 1091-1100. Validations
+ received are created with InitialSeq in that range, and
+ appropriate LedgerSeqs. Maybe some of those ledgers are
+ not accepted, so LedgerSeq stays null.
+ 4) At ledger 1100, this function is called with
+ lastRotated = 1000. The first query tries to delete
+ rows WHERE LedgerSeq < 1000. It finds none.
+ 5) The second round of deletions does not run.
+ 6) Ledgers continue to advance from 1100-1200 as described
+ in step 3.
+ 7) At ledger 1200, this function is called again with
+ lastRotated = 1100. The first query again tries to delete
+ rows WHERE LedgerSeq < 1100. It finds the rows for 1080-1099.
+ 8) The second round of deletions runs. It gets
+ WHERE v.LedgerSeq is NULL AND
+ (v.InitialSeq IS NULL OR v.InitialSeq < 1100)
+ The rows that are found include (a) ALL of the Validations
+ for the first 1080 ledgers. (b) Any orphan validations that
+ were created in step 3.
+ 9) This continues. The next rotation cycle does the same as steps
+ 7 & 8, except that none of the original Validations (8a) exist
+ anymore, and 8b gets the orphans from step 6.
+ */
+
+ static auto anyValDeleted = false;
+ auto const valDeleted = clearSql(*ledgerDb_, lastRotated,
+ "SELECT MIN(LedgerSeq) FROM Validations;",
+ "DELETE FROM Validations WHERE LedgerSeq < %u;");
+ anyValDeleted |= valDeleted;
+
+ if (health())
+ return;
+
+ if (anyValDeleted)
+ {
+ /* Delete the old NULL LedgerSeqs - the Validations that
+ aren't linked to a validated ledger - but only if we
+ deleted rows in the matching `clearSql` call, and only
+ for those created with an old InitialSeq.
+ */
+ using namespace std::chrono;
+ auto const deleteBatch = setup_.deleteBatch;
+ auto const continueLimit = (deleteBatch + 1) / 2;
+
+ std::string const deleteQuery(
+ R"sql(DELETE FROM Validations
+ WHERE LedgerHash IN
+ (
+ SELECT v.LedgerHash
+ FROM Validations v
+ WHERE v.LedgerSeq is NULL AND
+ (v.InitialSeq IS NULL OR v.InitialSeq < )sql" +
+ std::to_string(lastRotated) +
+ ") LIMIT " +
+ std::to_string (deleteBatch) +
+ ");");
+
+ JLOG(journal_.debug) << "start: " << deleteQuery << " of "
+ << deleteBatch << " rows.";
+ long long totalRowsAffected = 0;
+ long long rowsAffected;
+ soci::statement st = [&]
+ {
+ auto db = ledgerDb_->checkoutDb();
+ return (db->prepare << deleteQuery);
+ }();
+ if (health())
+ return;
+ do
+ {
+ {
+ auto db = ledgerDb_->checkoutDb();
+ auto const start = high_resolution_clock::now();
+ st.execute(true);
+ rowsAffected = st.get_affected_rows();
+ totalRowsAffected += rowsAffected;
+ auto const ms = duration_cast(
+ high_resolution_clock::now() - start).count();
+ JLOG(journal_.trace) << "step: deleted " << rowsAffected
+ << " rows in " << ms << "ms.";
+ }
+ if (health())
+ return;
+ if (rowsAffected >= continueLimit)
+ std::this_thread::sleep_for(
+ std::chrono::milliseconds(setup_.backOff));
+ }
+ while (rowsAffected && rowsAffected >= continueLimit);
+ JLOG(journal_.debug) << "finished: " << deleteQuery << ". Deleted "
+ << totalRowsAffected << " rows.";
+ }
+ }
+
+ if (health())
+ return;
+
clearSql (*transactionDb_, lastRotated,
"SELECT MIN(LedgerSeq) FROM Transactions;",
"DELETE FROM Transactions WHERE LedgerSeq < %u;");
@@ -675,6 +780,8 @@ setup_SHAMapStore (Config const& c)
{
SHAMapStore::Setup setup;
+ setup.standalone = c.RUN_STANDALONE;
+
// Get existing settings and add some default values if not specified:
setup.nodeDatabase = c.section (ConfigSection::nodeDatabase ());
diff --git a/src/ripple/app/misc/SHAMapStoreImp.h b/src/ripple/app/misc/SHAMapStoreImp.h
index db38a62d1..797db777c 100644
--- a/src/ripple/app/misc/SHAMapStoreImp.h
+++ b/src/ripple/app/misc/SHAMapStoreImp.h
@@ -80,7 +80,9 @@ private:
// check health/stop status as records are copied
std::uint64_t const checkHealthInterval_ = 1000;
// minimum # of ledgers to maintain for health of network
- std::uint32_t minimumDeletionInterval_ = 256;
+ static std::uint32_t const minimumDeletionInterval_ = 256;
+ // minimum # of ledgers required for standalone mode.
+ static std::uint32_t const minimumDeletionIntervalSA_ = 8;
Setup setup_;
NodeStore::Scheduler& scheduler_;
@@ -94,7 +96,7 @@ private:
mutable std::condition_variable cond_;
mutable std::mutex mutex_;
std::shared_ptr newLedger_;
- std::shared_ptr validatedLedger_;
+ std::atomic rotating_;
TransactionMaster& transactionMaster_;
std::atomic canDelete_;
// these do not exist upon SHAMapStore creation, but do exist
@@ -106,6 +108,12 @@ private:
DatabaseCon* transactionDb_ = nullptr;
DatabaseCon* ledgerDb_ = nullptr;
+public:
+ bool rotating() const
+ {
+ return rotating_;
+ }
+
public:
SHAMapStoreImp (Application& app,
Setup const& setup,
@@ -203,8 +211,10 @@ private:
/** delete from sqlite table in batches to not lock the db excessively
* pause briefly to extend access time to other users
* call with mutex object unlocked
+ * @return true if any deletable rows were found (though not
+ * necessarily deleted.
*/
- void clearSql (DatabaseCon& database, LedgerIndex lastRotated,
+ bool clearSql (DatabaseCon& database, LedgerIndex lastRotated,
std::string const& minQuery, std::string const& deleteQuery);
void clearCaches (LedgerIndex validatedSeq);
void freshenCaches();
diff --git a/src/ripple/app/misc/Validations.cpp b/src/ripple/app/misc/Validations.cpp
index 856536529..dcf241717 100644
--- a/src/ripple/app/misc/Validations.cpp
+++ b/src/ripple/app/misc/Validations.cpp
@@ -461,8 +461,10 @@ private:
void doWrite ()
{
LoadEvent::autoptr event (app_.getJobQueue ().getLoadEventAP (jtDISK, "ValidationWrite"));
- boost::format insVal ("INSERT INTO Validations "
- "(LedgerHash,NodePubKey,SignTime,RawData) VALUES ('%s','%s','%u',%s);");
+ std::string insVal ("INSERT INTO Validations "
+ "(InitialSeq, LedgerSeq, LedgerHash,NodePubKey,SignTime,RawData) "
+ "VALUES (:initialSeq, :ledgerSeq, :ledgerHash,:nodePubKey,:signTime,:rawData);");
+ std::string findSeq("SELECT LedgerSeq FROM Ledgers WHERE Ledgerhash=:ledgerHash;");
ScopedLockType sl (mLock);
assert (mWriting);
@@ -484,13 +486,34 @@ private:
{
s.erase ();
it->add (s);
- *db << boost::str (
- insVal % to_string (it->getLedgerHash ()) %
- toBase58(
- TokenType::TOKEN_NODE_PUBLIC,
- it->getSignerPublic ()) %
- it->getSignTime().time_since_epoch().count() %
- sqlEscape (s.peekData ()));
+
+ auto const ledgerHash = to_string(it->getLedgerHash());
+
+ boost::optional ledgerSeq;
+ *db << findSeq, soci::use(ledgerHash),
+ soci::into(ledgerSeq);
+
+ auto const initialSeq = ledgerSeq.value_or(
+ app_.getLedgerMaster().getCurrentLedgerIndex());
+ auto const nodePubKey = toBase58(
+ TokenType::TOKEN_NODE_PUBLIC,
+ it->getSignerPublic());
+ auto const signTime =
+ it->getSignTime().time_since_epoch().count();
+
+ soci::blob rawData(*db);
+ rawData.append(reinterpret_cast(
+ s.peekData().data()), s.peekData().size());
+ assert(rawData.get_len() == s.peekData().size());
+
+ *db <<
+ insVal,
+ soci::use(initialSeq),
+ soci::use(ledgerSeq),
+ soci::use(ledgerHash),
+ soci::use(nodePubKey),
+ soci::use(signTime),
+ soci::use(rawData);
}
tr.commit ();
diff --git a/src/ripple/app/tests/SHAMapStore_test.cpp b/src/ripple/app/tests/SHAMapStore_test.cpp
new file mode 100644
index 000000000..4060df43c
--- /dev/null
+++ b/src/ripple/app/tests/SHAMapStore_test.cpp
@@ -0,0 +1,654 @@
+//------------------------------------------------------------------------------
+/*
+ This file is part of rippled: https://github.com/ripple/rippled
+ Copyright (c) 2012-2015 Ripple Labs Inc.
+
+ Permission to use, copy, modify, and/or distribute this software for any
+ purpose with or without fee is hereby granted, provided that the above
+ copyright notice and this permission notice appear in all copies.
+
+ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+namespace ripple {
+namespace test {
+
+class SHAMapStore_test : public beast::unit_test::suite
+{
+ static auto const deleteInterval = 8;
+
+ static
+ std::unique_ptr
+ makeConfig()
+ {
+ auto p = std::make_unique();
+ setupConfigForUnitTests(*p);
+ p->LEDGER_HISTORY = deleteInterval;
+ auto& section = p->section(ConfigSection::nodeDatabase());
+ section.set("online_delete", to_string(deleteInterval));
+ //section.set("age_threshold", "60");
+ return p;
+ }
+
+ static
+ std::unique_ptr
+ makeConfigAdvisory()
+ {
+ auto p = makeConfig();
+ auto& section = p->section(ConfigSection::nodeDatabase());
+ section.set("advisory_delete", "1");
+ return p;
+ }
+
+ bool goodLedger(jtx::Env& env, Json::Value const& json,
+ std::string ledgerID, bool checkDB = false)
+ {
+ auto good = json.isMember(jss::result)
+ && !RPC::contains_error(json[jss::result])
+ && json[jss::result][jss::ledger][jss::ledger_index] == ledgerID;
+ if (!good || !checkDB)
+ return good;
+
+ auto const seq = json[jss::result][jss::ledger_index].asUInt();
+ std::string outHash;
+ LedgerIndex outSeq;
+ std::string outParentHash;
+ std::string outDrops;
+ std::uint64_t outCloseTime;
+ std::uint64_t outParentCloseTime;
+ std::uint64_t outCloseTimeResolution;
+ std::uint64_t outCloseFlags;
+ std::string outAccountHash;
+ std::string outTxHash;
+
+ {
+ auto db = env.app().getLedgerDB().checkoutDb();
+
+ *db << "SELECT LedgerHash,LedgerSeq,PrevHash,TotalCoins, "
+ "ClosingTime,PrevClosingTime,CloseTimeRes,CloseFlags, "
+ "AccountSetHash,TransSetHash "
+ "FROM Ledgers "
+ "WHERE LedgerSeq = :seq",
+ soci::use(seq),
+ soci::into(outHash),
+ soci::into(outSeq),
+ soci::into(outParentHash),
+ soci::into(outDrops),
+ soci::into(outCloseTime),
+ soci::into(outParentCloseTime),
+ soci::into(outCloseTimeResolution),
+ soci::into(outCloseFlags),
+ soci::into(outAccountHash),
+ soci::into(outTxHash);
+ }
+
+ auto const& ledger = json[jss::result][jss::ledger];
+ return outHash == ledger[jss::hash].asString() &&
+ outSeq == seq &&
+ outParentHash == ledger[jss::parent_hash].asString() &&
+ outDrops == ledger[jss::total_coins].asString() &&
+ outCloseTime == ledger[jss::close_time].asUInt() &&
+ outParentCloseTime == ledger[jss::parent_close_time].asUInt() &&
+ outCloseTimeResolution == ledger[jss::close_time_resolution].asUInt() &&
+ outCloseFlags == ledger[jss::close_flags].asUInt() &&
+ outAccountHash == ledger[jss::account_hash].asString() &&
+ outTxHash == ledger[jss::transaction_hash].asString();
+ }
+
+ bool bad(Json::Value const& json, error_code_i error = rpcLGR_NOT_FOUND)
+ {
+ return json.isMember(jss::result)
+ && RPC::contains_error(json[jss::result])
+ && json[jss::result][jss::error_code] == error;
+ }
+
+ std::string getHash(Json::Value const& json)
+ {
+ expect(json.isMember(jss::result) &&
+ json[jss::result].isMember(jss::ledger) &&
+ json[jss::result][jss::ledger].isMember(jss::hash) &&
+ json[jss::result][jss::ledger][jss::hash].isString());
+ return json[jss::result][jss::ledger][jss::hash].asString();
+ }
+
+ void validationCheck(jtx::Env& env, int const expected)
+ {
+ auto db = env.app().getLedgerDB().checkoutDb();
+
+ int actual;
+ *db << "SELECT count(*) AS rows FROM Validations;",
+ soci::into(actual);
+
+ expect(actual == expected);
+
+ }
+
+ void ledgerCheck(jtx::Env& env, int const rows,
+ int const first)
+ {
+ auto db = env.app().getLedgerDB().checkoutDb();
+
+ int actualRows, actualFirst, actualLast;
+ *db << "SELECT count(*) AS rows, "
+ "min(LedgerSeq) as first, "
+ "max(LedgerSeq) as last "
+ "FROM Ledgers;",
+ soci::into(actualRows),
+ soci::into(actualFirst),
+ soci::into(actualLast);
+
+ expect(actualRows == rows);
+ expect(actualFirst == first);
+ expect(actualLast == first + rows - 1);
+
+ }
+
+ void transactionCheck(jtx::Env& env, int const rows)
+ {
+ auto db = env.app().getTxnDB().checkoutDb();
+
+ int actualRows;
+ *db << "SELECT count(*) AS rows "
+ "FROM Transactions;",
+ soci::into(actualRows);
+
+ expect(actualRows == rows);
+ }
+
+ void accountTransactionCheck(jtx::Env& env, int const rows)
+ {
+ auto db = env.app().getTxnDB().checkoutDb();
+
+ int actualRows;
+ *db << "SELECT count(*) AS rows "
+ "FROM AccountTransactions;",
+ soci::into(actualRows);
+
+ expect(actualRows == rows);
+ }
+
+ int waitForReady(jtx::Env& env)
+ {
+ using namespace std::chrono_literals;
+
+ auto& store = env.app().getSHAMapStore();
+
+ int ledgerSeq = 3;
+ while (!store.getLastRotated())
+ {
+ env.close();
+ std::this_thread::sleep_for(100ms);
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq++)));
+ }
+ return ledgerSeq;
+ }
+
+public:
+ void testClear()
+ {
+ using namespace std::chrono_literals;
+
+ testcase("clearPrior");
+ using namespace jtx;
+
+ Env env(*this, makeConfig());
+
+ auto store = dynamic_cast(
+ &env.app().getSHAMapStore());
+ expect(store);
+ env.fund(XRP(10000), noripple("alice"));
+
+ validationCheck(env, 0);
+ ledgerCheck(env, 1, 2);
+ transactionCheck(env, 0);
+ accountTransactionCheck(env, 0);
+
+ std::map ledgers;
+
+ auto ledgerTmp = env.rpc("ledger", "0");
+ expect(bad(ledgerTmp));
+
+ ledgers.emplace(std::make_pair(1, env.rpc("ledger", "1")));
+ expect(goodLedger(env, ledgers[1], "1"));
+
+ ledgers.emplace(std::make_pair(2, env.rpc("ledger", "2")));
+ expect(goodLedger(env, ledgers[2], "2"));
+
+ ledgerTmp = env.rpc("ledger", "current");
+ expect(goodLedger(env, ledgerTmp, "3"));
+
+ ledgerTmp = env.rpc("ledger", "4");
+ expect(bad(ledgerTmp));
+
+ ledgerTmp = env.rpc("ledger", "100");
+ expect(bad(ledgerTmp));
+
+ for (auto i = 4; i < deleteInterval + 4; ++i)
+ {
+ env.fund(XRP(10000), noripple("test" + to_string(i)));
+ env.close();
+
+ ledgerTmp = env.rpc("ledger", "current");
+ expect(goodLedger(env, ledgerTmp, to_string(i)));
+ }
+ assert(store->getLastRotated() == 3);
+
+ for (auto i = 3; i < deleteInterval + 3; ++i)
+ {
+ ledgers.emplace(std::make_pair(i,
+ env.rpc("ledger", to_string(i))));
+ expect(goodLedger(env, ledgers[i], to_string(i), true) &&
+ getHash(ledgers[i]).length());
+ }
+
+ validationCheck(env, 0);
+ ledgerCheck(env, deleteInterval + 1, 2);
+ transactionCheck(env, deleteInterval + 1);
+ accountTransactionCheck(env, 2*(deleteInterval + 1));
+
+ {
+ // Since standalone doesn't _do_ validations, manually
+ // insert some into the table. Create some with the
+ // hashes from our real ledgers, and some with fake
+ // hashes to represent validations that never ended up
+ // in a validated ledger.
+ char lh[65];
+ memset(lh, 'a', 64);
+ lh[64] = '\0';
+ std::vector preSeqLedgerHashes({
+ lh
+ });
+ std::vector badLedgerHashes;
+ std::vector badLedgerSeqs;
+ std::vector ledgerHashes;
+ std::vector ledgerSeqs;
+ for (auto const& lgr : ledgers)
+ {
+ ledgerHashes.emplace_back(getHash(lgr.second));
+ ledgerSeqs.emplace_back(lgr.second[jss::result][jss::ledger_index].asUInt());
+ }
+ for (auto i = 0; i < 10; ++i)
+ {
+ ++lh[30];
+ preSeqLedgerHashes.emplace_back(lh);
+ ++lh[20];
+ badLedgerHashes.emplace_back(lh);
+ badLedgerSeqs.emplace_back(i + 1);
+ }
+
+ auto db = env.app().getLedgerDB().checkoutDb();
+
+ // Pre-migration validation - no sequence numbers.
+ *db << "INSERT INTO Validations "
+ "(LedgerHash) "
+ "VALUES "
+ "(:ledgerHash);",
+ soci::use(preSeqLedgerHashes);
+ // Post-migration orphan validation - InitalSeq,
+ // but no LedgerSeq
+ *db << "INSERT INTO Validations "
+ "(LedgerHash, InitialSeq) "
+ "VALUES "
+ "(:ledgerHash, :initialSeq);",
+ soci::use(badLedgerHashes),
+ soci::use(badLedgerSeqs);
+ // Post-migration validated ledger.
+ *db << "INSERT INTO Validations "
+ "(LedgerHash, LedgerSeq) "
+ "VALUES "
+ "(:ledgerHash, :ledgerSeq);",
+ soci::use(ledgerHashes),
+ soci::use(ledgerSeqs);
+ }
+
+ validationCheck(env, deleteInterval + 23);
+ ledgerCheck(env, deleteInterval + 1, 2);
+ transactionCheck(env, deleteInterval + 1);
+ accountTransactionCheck(env, 2 * (deleteInterval + 1));
+
+ {
+ // Closing one more ledger triggers a rotate
+ env.close();
+
+ auto ledger = env.rpc("ledger", "current");
+ expect(goodLedger(env, ledger, to_string(deleteInterval + 4)));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ expect(store->getLastRotated() == deleteInterval + 3);
+ auto const lastRotated = store->getLastRotated();
+ expect(lastRotated == 11, to_string(lastRotated));
+
+ // That took care of the fake hashes
+ validationCheck(env, deleteInterval + 8);
+ ledgerCheck(env, deleteInterval + 1, 3);
+ transactionCheck(env, deleteInterval + 1);
+ accountTransactionCheck(env, 2 * (deleteInterval + 1));
+
+ // The last iteration of this loop should trigger a rotate
+ for (auto i = lastRotated - 1; i < lastRotated + deleteInterval - 1; ++i)
+ {
+ validationCheck(env, deleteInterval + i + 1 - lastRotated + 8);
+
+ env.close();
+
+ ledgerTmp = env.rpc("ledger", "current");
+ expect(goodLedger(env, ledgerTmp, to_string(i + 3)));
+
+ ledgers.emplace(std::make_pair(i,
+ env.rpc("ledger", to_string(i))));
+ expect(store->getLastRotated() == lastRotated ||
+ i == lastRotated + deleteInterval - 2, "lastRotated");
+ expect(goodLedger(env, ledgers[i], to_string(i), true) &&
+ getHash(ledgers[i]).length(), to_string(ledgers[i]));
+
+ std::vector ledgerHashes({
+ getHash(ledgers[i])
+ });
+ std::vector ledgerSeqs({
+ ledgers[i][jss::result][jss::ledger_index].asUInt()
+ });
+ auto db = env.app().getLedgerDB().checkoutDb();
+
+ *db << "INSERT INTO Validations "
+ "(LedgerHash, LedgerSeq) "
+ "VALUES "
+ "(:ledgerHash, :ledgerSeq);",
+ soci::use(ledgerHashes),
+ soci::use(ledgerSeqs);
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ expect(store->getLastRotated() == deleteInterval + lastRotated);
+
+ validationCheck(env, deleteInterval - 1);
+ ledgerCheck(env, deleteInterval + 1, lastRotated);
+ transactionCheck(env, 0);
+ accountTransactionCheck(env, 0);
+
+ }
+
+ void testAutomatic()
+ {
+ testcase("automatic online_delete");
+ using namespace jtx;
+ using namespace std::chrono_literals;
+
+ Env env(*this, makeConfig());
+ auto store = dynamic_cast(
+ &env.app().getSHAMapStore());
+ expect(store);
+
+ auto ledgerSeq = waitForReady(env);
+ auto lastRotated = ledgerSeq - 1;
+ expect(store->getLastRotated() == lastRotated,
+ to_string(store->getLastRotated()));
+ expect(lastRotated != 2);
+
+ // Because advisory_delete is unset,
+ // "can_delete" is disabled.
+ auto const canDelete = env.rpc("can_delete");
+ expect(bad(canDelete, rpcNOT_ENABLED));
+
+ // Close ledgers without triggering a rotate
+ for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
+ {
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+ }
+
+ while(store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ // The database will always have back to ledger 2,
+ // regardless of lastRotated.
+ validationCheck(env, 0);
+ ledgerCheck(env, ledgerSeq - 2, 2);
+ expect(lastRotated == store->getLastRotated());
+
+ {
+ // Closing one more ledger triggers a rotate
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ validationCheck(env, 0);
+ ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
+ expect(lastRotated != store->getLastRotated());
+
+ lastRotated = store->getLastRotated();
+
+ // Close enough ledgers to trigger another rotate
+ for (; ledgerSeq < lastRotated + deleteInterval + 1; ++ledgerSeq)
+ {
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ validationCheck(env, 0);
+ ledgerCheck(env, deleteInterval + 1, lastRotated);
+ expect(lastRotated != store->getLastRotated());
+ }
+
+ void testCanDelete()
+ {
+ testcase("online_delete with advisory_delete");
+ using namespace jtx;
+ using namespace std::chrono_literals;
+
+ // Same config with advisory_delete enabled
+ Env env(*this, makeConfigAdvisory());
+ auto store = dynamic_cast(
+ &env.app().getSHAMapStore());
+ expect(store);
+
+ auto ledgerSeq = waitForReady(env);
+ auto lastRotated = ledgerSeq - 1;
+ expect(store->getLastRotated() == lastRotated,
+ to_string(store->getLastRotated()));
+ expect(lastRotated != 2);
+
+ auto canDelete = env.rpc("can_delete");
+ expect(!RPC::contains_error(canDelete[jss::result]));
+ expect(canDelete[jss::result][jss::can_delete] == 0);
+
+ canDelete = env.rpc("can_delete", "never");
+ expect(!RPC::contains_error(canDelete[jss::result]));
+ expect(canDelete[jss::result][jss::can_delete] == 0);
+
+ auto const firstBatch = deleteInterval + ledgerSeq;
+ for (; ledgerSeq < firstBatch; ++ledgerSeq)
+ {
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ validationCheck(env, 0);
+ ledgerCheck(env, ledgerSeq - 2, 2);
+ expect(lastRotated == store->getLastRotated());
+
+ // This does not kick off a cleanup
+ canDelete = env.rpc("can_delete", to_string(
+ ledgerSeq + deleteInterval / 2));
+ expect(!RPC::contains_error(canDelete[jss::result]));
+ expect(canDelete[jss::result][jss::can_delete] ==
+ ledgerSeq + deleteInterval / 2);
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ validationCheck(env, 0);
+ ledgerCheck(env, ledgerSeq - 2, 2);
+ expect(store->getLastRotated() == lastRotated);
+
+ {
+ // This kicks off a cleanup, but it stays small.
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ validationCheck(env, 0);
+ ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
+
+ expect(store->getLastRotated() == ledgerSeq - 1);
+ lastRotated = ledgerSeq - 1;
+
+ for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
+ {
+ // No cleanups in this loop.
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ expect(store->getLastRotated() == lastRotated);
+
+ {
+ // This kicks off another cleanup.
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ validationCheck(env, 0);
+ ledgerCheck(env, ledgerSeq - firstBatch, firstBatch);
+
+ expect(store->getLastRotated() == ledgerSeq - 1);
+ lastRotated = ledgerSeq - 1;
+
+ // This does not kick off a cleanup
+ canDelete = env.rpc("can_delete", "always");
+ expect(!RPC::contains_error(canDelete[jss::result]));
+ expect(canDelete[jss::result][jss::can_delete] ==
+ std::numeric_limits ::max());
+
+ for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
+ {
+ // No cleanups in this loop.
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ expect(store->getLastRotated() == lastRotated);
+
+ {
+ // This kicks off another cleanup.
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ validationCheck(env, 0);
+ ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
+
+ expect(store->getLastRotated() == ledgerSeq - 1);
+ lastRotated = ledgerSeq - 1;
+
+ // This does not kick off a cleanup
+ canDelete = env.rpc("can_delete", "now");
+ expect(!RPC::contains_error(canDelete[jss::result]));
+ expect(canDelete[jss::result][jss::can_delete] == ledgerSeq - 1);
+
+ for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
+ {
+ // No cleanups in this loop.
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ expect(store->getLastRotated() == lastRotated);
+
+ {
+ // This kicks off another cleanup.
+ env.close();
+
+ auto ledger = env.rpc("ledger", "validated");
+ expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
+ }
+
+ while (store->rotating())
+ std::this_thread::sleep_for(1ms);
+
+ validationCheck(env, 0);
+ ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
+
+ expect(store->getLastRotated() == ledgerSeq - 1);
+ lastRotated = ledgerSeq - 1;
+ }
+
+ void run()
+ {
+ testClear();
+ testAutomatic();
+ testCanDelete();
+ }
+};
+
+BEAST_DEFINE_TESTSUITE(SHAMapStore,app,ripple);
+
+}
+}
diff --git a/src/ripple/core/impl/DatabaseCon.cpp b/src/ripple/core/impl/DatabaseCon.cpp
index 4b3797932..ec44ed4b2 100644
--- a/src/ripple/core/impl/DatabaseCon.cpp
+++ b/src/ripple/core/impl/DatabaseCon.cpp
@@ -46,7 +46,9 @@ DatabaseCon::DatabaseCon (
{
try
{
- session_ << initStrings[i];
+ soci::statement st = session_.prepare <<
+ initStrings[i];
+ st.execute(true);
}
catch (soci::soci_error&)
{
diff --git a/src/ripple/rpc/handlers/CanDelete.cpp b/src/ripple/rpc/handlers/CanDelete.cpp
index fa15f918c..9ae1b5f3b 100644
--- a/src/ripple/rpc/handlers/CanDelete.cpp
+++ b/src/ripple/rpc/handlers/CanDelete.cpp
@@ -70,10 +70,11 @@ Json::Value doCanDelete (RPC::Context& context)
{
canDeleteSeq = context.app.getSHAMapStore().getLastRotated();
if (!canDeleteSeq)
- return RPC::make_error (rpcNOT_READY); }
- else if (canDeleteStr.size() == 64 &&
- canDeleteStr.find_first_not_of("0123456789abcdef") ==
- std::string::npos)
+ return RPC::make_error (rpcNOT_READY);
+ }
+ else if (canDeleteStr.size() == 64 &&
+ canDeleteStr.find_first_not_of("0123456789abcdef") ==
+ std::string::npos)
{
auto ledger = context.ledgerMaster.getLedgerByHash (
from_hex_text(canDeleteStr));
diff --git a/src/ripple/unity/app_tests.cpp b/src/ripple/unity/app_tests.cpp
index ab7d17a15..973497643 100644
--- a/src/ripple/unity/app_tests.cpp
+++ b/src/ripple/unity/app_tests.cpp
@@ -29,6 +29,7 @@
#include
#include
#include
+#include
#include
#include
#include