ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-12-06 17:27:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clear old Validations during online delete (RIPD-870):

Browse Source 
* Add Validations.LedgerSeq and .InitialSeq fields.
* Clean up logging.
* Lower online delete minimum for standalone mode.
* Unit tests of online_delete.
This commit is contained in:
Edward Hennis
2015-12-09 19:30:05 -05:00
committed by Nik Bougalis
parent 70d5c4eca7
commit eb62959216
14 changed files with 917 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Builds/VisualStudio2015/RippleD.vcxproj
View File

@@ -1531,6 +1531,10 @@
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\src\ripple\app\tests\SHAMapStore_test.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\src\ripple\app\tests\SusPay_test.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='debug|x64'">True</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='release|x64'">True</ExcludedFromBuild>

3
Builds/VisualStudio2015/RippleD.vcxproj.filters
View File

@@ -2112,6 +2112,9 @@
<ClCompile Include="..\..\src\ripple\app\tests\SetAuth_test.cpp">
<Filter>ripple\app\tests</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ripple\app\tests\SHAMapStore_test.cpp">
<Filter>ripple\app\tests</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ripple\app\tests\SusPay_test.cpp">
<Filter>ripple\app\tests</Filter>
</ClCompile>

65
src/ripple/app/ledger/Ledger.cpp
View File

@@ -890,16 +890,6 @@ static bool saveValidatedLedger (
"DELETE FROM AccountTransactions WHERE LedgerSeq = %u;");
static boost::format deleteAcctTrans (
"DELETE FROM AccountTransactions WHERE TransID = '%s';");
static boost::format transExists (
"SELECT Status FROM Transactions WHERE TransID = '%s';");
static boost::format updateTx (
"UPDATE Transactions SET LedgerSeq = %u, Status = '%c', TxnMeta = %s "
"WHERE TransID = '%s';");
static boost::format addLedger (
"INSERT OR REPLACE INTO Ledgers "
"(LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime,"
"CloseTimeRes,CloseFlags,AccountSetHash,TransSetHash) VALUES "
"('%s','%u','%s','%s','%u','%u','%d','%u','%s','%s');");
auto seq = ledger->info().seq;
@@ -1032,18 +1022,53 @@ static bool saveValidatedLedger (
}
{
static std::string addLedger(
R"sql(INSERT OR REPLACE INTO Ledgers
(LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime,
CloseTimeRes,CloseFlags,AccountSetHash,TransSetHash)
VALUES
(:ledgerHash,:ledgerSeq,:prevHash,:totalCoins,:closingTime,:prevClosingTime,
:closeTimeRes,:closeFlags,:accountSetHash,:transSetHash);)sql");
static std::string updateVal(
R"sql(UPDATE Validations SET LedgerSeq = :ledgerSeq, InitialSeq = :initialSeq
WHERE LedgerHash = :ledgerHash;)sql");
auto db (app.getLedgerDB ().checkoutDb ());
// TODO(tom): ARG!
*db << boost::str (
addLedger %
to_string (ledger->info().hash) % seq % to_string (ledger->info().parentHash) %
to_string (ledger->info().drops) %
ledger->info().closeTime.time_since_epoch().count() %
ledger->info().parentCloseTime.time_since_epoch().count() %
ledger->info().closeTimeResolution.count() %
ledger->info().closeFlags % to_string (ledger->info().accountHash) %
to_string (ledger->info().txHash));
soci::transaction tr(*db);
auto const hash = to_string (ledger->info().hash);
auto const parentHash = to_string (ledger->info().parentHash);
auto const drops = to_string (ledger->info().drops);
auto const closeTime =
ledger->info().closeTime.time_since_epoch().count();
auto const parentCloseTime =
ledger->info().parentCloseTime.time_since_epoch().count();
auto const closeTimeResolution =
ledger->info().closeTimeResolution.count();
auto const closeFlags = ledger->info().closeFlags;
auto const accountHash = to_string (ledger->info().accountHash);
auto const txHash = to_string (ledger->info().txHash);
*db << addLedger,
soci::use(hash),
soci::use(seq),
soci::use(parentHash),
soci::use(drops),
soci::use(closeTime),
soci::use(parentCloseTime),
soci::use(closeTimeResolution),
soci::use(closeFlags),
soci::use(accountHash),
soci::use(txHash);
*db << updateVal,
soci::use(seq),
soci::use(seq),
soci::use(hash);
tr.commit();
}
// Clients can now trust the database for

4
src/ripple/app/ledger/LedgerHistory.cpp
View File

@@ -499,7 +499,9 @@ void LedgerHistory::clearLedgerCachePrior (LedgerIndex seq)
{
for (LedgerHash it: m_ledgers_by_hash.getKeys())
{
if (getLedgerByHash (it)->info().seq < seq)
auto const ledger = getLedgerByHash (it);
assert(ledger);
if (!ledger || ledger->info().seq < seq)
m_ledgers_by_hash.del (it, false);
}
}

2
src/ripple/app/ledger/impl/LedgerMaster.cpp
View File

@@ -984,7 +984,7 @@ LedgerMaster::getLedgerHashForHistory (LedgerIndex index)
if (! ret)
ret = walkHashBySeq (index);
return *ret;
return ret;
}
bool

15
src/ripple/app/main/DBInit.cpp
View File

@@ -90,14 +90,29 @@ const char* LedgerDBInit[] =
);",
"CREATE INDEX IF NOT EXISTS SeqLedger ON Ledgers(LedgerSeq);",
// InitialSeq field is the current ledger seq when the row
// is inserted. Only relevant during online delete
"CREATE TABLE IF NOT EXISTS Validations ( \
LedgerSeq BIGINT UNSIGNED, \
InitialSeq BIGINT UNSIGNED, \
LedgerHash CHARACTER(64), \
NodePubKey CHARACTER(56), \
SignTime BIGINT UNSIGNED, \
RawData BLOB \
);",
// This will error out if the column already exists,
// but DatabaseCon intentionally ignores errors.
"ALTER TABLE Validations \
ADD COLUMN LedgerSeq BIGINT UNSIGNED;",
"ALTER TABLE Validations \
ADD COLUMN InitialSeq BIGINT UNSIGNED;",
"CREATE INDEX IF NOT EXISTS ValidationsByHash ON \
Validations(LedgerHash);",
"CREATE INDEX IF NOT EXISTS ValidationsBySeq ON \
Validations(LedgerSeq);",
"CREATE INDEX IF NOT EXISTS ValidationsByInitialSeq ON \
Validations(InitialSeq, LedgerSeq);",
"CREATE INDEX IF NOT EXISTS ValidationsByTime ON \
Validations(SignTime);",

1
src/ripple/app/misc/SHAMapStore.h
View File

@@ -41,6 +41,7 @@ class SHAMapStore
public:
struct Setup
{
bool standalone = false;
std::uint32_t deleteInterval = 0;
bool advisoryDelete = false;
std::uint32_t ledgerHistory = 0;

167
src/ripple/app/misc/SHAMapStoreImp.cpp
View File

@@ -29,6 +29,7 @@
#include <boost/format.hpp>
#include <boost/optional.hpp>
#include <memory>
#include <chrono>
namespace ripple {
void SHAMapStoreImp::SavedStateDB::init (BasicConfig const& config,
@@ -179,21 +180,24 @@ SHAMapStoreImp::SHAMapStoreImp (
, scheduler_ (scheduler)
, journal_ (journal)
, nodeStoreJournal_ (nodeStoreJournal)
, rotating_(false)
, transactionMaster_ (transactionMaster)
, canDelete_ (std::numeric_limits <LedgerIndex>::max())
{
if (setup_.deleteInterval)
{
if (setup_.deleteInterval < minimumDeletionInterval_)
auto const minInterval = setup.standalone ?
minimumDeletionIntervalSA_ : minimumDeletionInterval_;
if (setup_.deleteInterval < minInterval)
{
Throw<std::runtime_error> ("online_delete must be at least " +
std::to_string (minimumDeletionInterval_));
std::to_string (minInterval));
}
if (setup_.ledgerHistory > setup_.deleteInterval)
{
Throw<std::runtime_error> (
"online_delete must be less than ledger_history (currently " +
"online_delete must not be less than ledger_history (currently " +
std::to_string (setup_.ledgerHistory) + ")");
}
@@ -283,7 +287,8 @@ SHAMapStoreImp::run()
while (1)
{
healthy_ = true;
validatedLedger_.reset();
std::shared_ptr<Ledger const> validatedLedger;
rotating_ = false;
{
std::unique_lock <std::mutex> lock (mutex_);
@@ -294,12 +299,15 @@ SHAMapStoreImp::run()
}
cond_.wait (lock);
if (newLedger_)
validatedLedger_ = std::move (newLedger_);
{
rotating_ = true;
validatedLedger = std::move(newLedger_);
}
else
continue;
}
LedgerIndex validatedSeq = validatedLedger_->info().seq;
LedgerIndex validatedSeq = validatedLedger->info().seq;
if (!lastRotated)
{
lastRotated = validatedSeq;
@@ -340,7 +348,7 @@ SHAMapStoreImp::run()
}
std::uint64_t nodeCount = 0;
validatedLedger_->stateMap().snapShot (
validatedLedger->stateMap().snapShot (
false)->visitNodes (
std::bind (&SHAMapStoreImp::copyNode, this,
std::ref(nodeCount), std::placeholders::_1));
@@ -504,7 +512,7 @@ SHAMapStoreImp::makeDatabaseRotating (std::string const& name,
readThreads, writableBackend, archiveBackend, nodeStoreJournal_);
}
void
bool
SHAMapStoreImp::clearSql (DatabaseCon& database,
LedgerIndex lastRotated,
std::string const& minQuery,
@@ -517,12 +525,12 @@ SHAMapStoreImp::clearSql (DatabaseCon& database,
boost::optional<std::uint64_t> m;
*db << minQuery, soci::into(m);
if (!m)
return;
return false;
min = *m;
}
if (health() != Health::ok)
return;
if(min > lastRotated || health() != Health::ok)
return false;
boost::format formattedDeleteQuery (deleteQuery);
@@ -530,19 +538,19 @@ SHAMapStoreImp::clearSql (DatabaseCon& database,
"start: " << deleteQuery << " from " << min << " to " << lastRotated;
while (min < lastRotated)
{
min = (min + setup_.deleteBatch >= lastRotated) ? lastRotated :
min + setup_.deleteBatch;
min = std::min(lastRotated, min + setup_.deleteBatch);
{
auto db = database.checkoutDb ();
*db << boost::str (formattedDeleteQuery % min);
}
if (health())
return;
return true;
if (min < lastRotated)
std::this_thread::sleep_for (
std::chrono::milliseconds (setup_.backOff));
}
JLOG(journal_.debug) << "finished: " << deleteQuery;
return true;
}
void
@@ -566,25 +574,10 @@ SHAMapStoreImp::freshenCaches()
void
SHAMapStoreImp::clearPrior (LedgerIndex lastRotated)
{
ledgerMaster_->clearPriorLedgers (lastRotated);
if (health())
return;
// TODO This won't remove validations for ledgers that do not get
// validated. That will likely require inserting LedgerSeq into
// the validations table.
//
// This query has poor performance with large data sets.
// The schema needs to be redesigned to avoid the JOIN, or an
// RDBMS that supports concurrency should be used.
/*
clearSql (*ledgerDb_, lastRotated,
"SELECT MIN(LedgerSeq) FROM Ledgers;",
"DELETE FROM Validations WHERE LedgerHash IN "
"(SELECT Ledgers.LedgerHash FROM Validations JOIN Ledgers ON "
"Validations.LedgerHash=Ledgers.LedgerHash WHERE Ledgers.LedgerSeq < %u);");
*/
ledgerMaster_->clearPriorLedgers (lastRotated);
if (health())
return;
@@ -594,6 +587,118 @@ SHAMapStoreImp::clearPrior (LedgerIndex lastRotated)
if (health())
return;
{
/*
Steps for migration:
Assume: online_delete = 100, lastRotated = 1000,
Last shutdown was at ledger # 1080.
The current network validated ledger is 1090.
Implies: Ledgers has entries from 900 to 1080.
Validations has entries for all 1080 ledgers,
including orphan validations that were not included
in a validated ledger.
1) Columns are created in Validations with default NULL values.
2) During syncing, Ledgers and Validations for 1080 - 1090
are received from the network. Records are created in
Validations with InitialSeq approximately 1080 (exact value
doesn't matter), and later validated with the matching
LedgerSeq value.
3) rippled participates in ledgers 1091-1100. Validations
received are created with InitialSeq in that range, and
appropriate LedgerSeqs. Maybe some of those ledgers are
not accepted, so LedgerSeq stays null.
4) At ledger 1100, this function is called with
lastRotated = 1000. The first query tries to delete
rows WHERE LedgerSeq < 1000. It finds none.
5) The second round of deletions does not run.
6) Ledgers continue to advance from 1100-1200 as described
in step 3.
7) At ledger 1200, this function is called again with
lastRotated = 1100. The first query again tries to delete
rows WHERE LedgerSeq < 1100. It finds the rows for 1080-1099.
8) The second round of deletions runs. It gets
WHERE v.LedgerSeq is NULL AND
(v.InitialSeq IS NULL OR v.InitialSeq < 1100)
The rows that are found include (a) ALL of the Validations
for the first 1080 ledgers. (b) Any orphan validations that
were created in step 3.
9) This continues. The next rotation cycle does the same as steps
7 & 8, except that none of the original Validations (8a) exist
anymore, and 8b gets the orphans from step 6.
*/
static auto anyValDeleted = false;
auto const valDeleted = clearSql(*ledgerDb_, lastRotated,
"SELECT MIN(LedgerSeq) FROM Validations;",
"DELETE FROM Validations WHERE LedgerSeq < %u;");
anyValDeleted |= valDeleted;
if (health())
return;
if (anyValDeleted)
{
/* Delete the old NULL LedgerSeqs - the Validations that
aren't linked to a validated ledger - but only if we
deleted rows in the matching `clearSql` call, and only
for those created with an old InitialSeq.
*/
using namespace std::chrono;
auto const deleteBatch = setup_.deleteBatch;
auto const continueLimit = (deleteBatch + 1) / 2;
std::string const deleteQuery(
R"sql(DELETE FROM Validations
WHERE LedgerHash IN
(
SELECT v.LedgerHash
FROM Validations v
WHERE v.LedgerSeq is NULL AND
(v.InitialSeq IS NULL OR v.InitialSeq < )sql" +
std::to_string(lastRotated) +
") LIMIT " +
std::to_string (deleteBatch) +
");");
JLOG(journal_.debug) << "start: " << deleteQuery << " of "
<< deleteBatch << " rows.";
long long totalRowsAffected = 0;
long long rowsAffected;
soci::statement st = [&]
{
auto db = ledgerDb_->checkoutDb();
return (db->prepare << deleteQuery);
}();
if (health())
return;
do
{
{
auto db = ledgerDb_->checkoutDb();
auto const start = high_resolution_clock::now();
st.execute(true);
rowsAffected = st.get_affected_rows();
totalRowsAffected += rowsAffected;
auto const ms = duration_cast<milliseconds>(
high_resolution_clock::now() - start).count();
JLOG(journal_.trace) << "step: deleted " << rowsAffected
<< " rows in " << ms << "ms.";
}
if (health())
return;
if (rowsAffected >= continueLimit)
std::this_thread::sleep_for(
std::chrono::milliseconds(setup_.backOff));
}
while (rowsAffected && rowsAffected >= continueLimit);
JLOG(journal_.debug) << "finished: " << deleteQuery << ". Deleted "
<< totalRowsAffected << " rows.";
}
}
if (health())
return;
clearSql (*transactionDb_, lastRotated,
"SELECT MIN(LedgerSeq) FROM Transactions;",
"DELETE FROM Transactions WHERE LedgerSeq < %u;");
@@ -675,6 +780,8 @@ setup_SHAMapStore (Config const& c)
{
SHAMapStore::Setup setup;
setup.standalone = c.RUN_STANDALONE;
// Get existing settings and add some default values if not specified:
setup.nodeDatabase = c.section (ConfigSection::nodeDatabase ());

16
src/ripple/app/misc/SHAMapStoreImp.h
View File

@@ -80,7 +80,9 @@ private:
// check health/stop status as records are copied
std::uint64_t const checkHealthInterval_ = 1000;
// minimum # of ledgers to maintain for health of network
std::uint32_t minimumDeletionInterval_ = 256;
static std::uint32_t const minimumDeletionInterval_ = 256;
// minimum # of ledgers required for standalone mode.
static std::uint32_t const minimumDeletionIntervalSA_ = 8;
Setup setup_;
NodeStore::Scheduler& scheduler_;
@@ -94,7 +96,7 @@ private:
mutable std::condition_variable cond_;
mutable std::mutex mutex_;
std::shared_ptr<Ledger const> newLedger_;
std::shared_ptr<Ledger const> validatedLedger_;
std::atomic<bool> rotating_;
TransactionMaster& transactionMaster_;
std::atomic <LedgerIndex> canDelete_;
// these do not exist upon SHAMapStore creation, but do exist
@@ -106,6 +108,12 @@ private:
DatabaseCon* transactionDb_ = nullptr;
DatabaseCon* ledgerDb_ = nullptr;
public:
bool rotating() const
{
return rotating_;
}
public:
SHAMapStoreImp (Application& app,
Setup const& setup,
@@ -203,8 +211,10 @@ private:
/** delete from sqlite table in batches to not lock the db excessively
* pause briefly to extend access time to other users
* call with mutex object unlocked
* @return true if any deletable rows were found (though not
* necessarily deleted.
*/
void clearSql (DatabaseCon& database, LedgerIndex lastRotated,
bool clearSql (DatabaseCon& database, LedgerIndex lastRotated,
std::string const& minQuery, std::string const& deleteQuery);
void clearCaches (LedgerIndex validatedSeq);
void freshenCaches();

41
src/ripple/app/misc/Validations.cpp
View File

@@ -461,8 +461,10 @@ private:
void doWrite ()
{
LoadEvent::autoptr event (app_.getJobQueue ().getLoadEventAP (jtDISK, "ValidationWrite"));
boost::format insVal ("INSERT INTO Validations "
"(LedgerHash,NodePubKey,SignTime,RawData) VALUES ('%s','%s','%u',%s);");
std::string insVal ("INSERT INTO Validations "
"(InitialSeq, LedgerSeq, LedgerHash,NodePubKey,SignTime,RawData) "
"VALUES (:initialSeq, :ledgerSeq, :ledgerHash,:nodePubKey,:signTime,:rawData);");
std::string findSeq("SELECT LedgerSeq FROM Ledgers WHERE Ledgerhash=:ledgerHash;");
ScopedLockType sl (mLock);
assert (mWriting);
@@ -484,13 +486,34 @@ private:
{
s.erase ();
it->add (s);
*db << boost::str (
insVal % to_string (it->getLedgerHash ()) %
toBase58(
TokenType::TOKEN_NODE_PUBLIC,
it->getSignerPublic ()) %
it->getSignTime().time_since_epoch().count() %
sqlEscape (s.peekData ()));
auto const ledgerHash = to_string(it->getLedgerHash());
boost::optional<std::uint64_t> ledgerSeq;
*db << findSeq, soci::use(ledgerHash),
soci::into(ledgerSeq);
auto const initialSeq = ledgerSeq.value_or(
app_.getLedgerMaster().getCurrentLedgerIndex());
auto const nodePubKey = toBase58(
TokenType::TOKEN_NODE_PUBLIC,
it->getSignerPublic());
auto const signTime =
it->getSignTime().time_since_epoch().count();
soci::blob rawData(*db);
rawData.append(reinterpret_cast<const char*>(
s.peekData().data()), s.peekData().size());
assert(rawData.get_len() == s.peekData().size());
*db <<
insVal,
soci::use(initialSeq),
soci::use(ledgerSeq),
soci::use(ledgerHash),
soci::use(nodePubKey),
soci::use(signTime),
soci::use(rawData);
}
tr.commit ();

654
src/ripple/app/tests/SHAMapStore_test.cpp Normal file
View File

@@ -0,0 +1,654 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012-2015 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <BeastConfig.h>
#include <ripple/app/main/Application.h>
#include <ripple/app/misc/SHAMapStoreImp.h>
#include <ripple/core/ConfigSections.h>
#include <ripple/protocol/JsonFields.h>
#include <ripple/test/jtx.h>
namespace ripple {
namespace test {
class SHAMapStore_test : public beast::unit_test::suite
{
static auto const deleteInterval = 8;
static
std::unique_ptr<Config>
makeConfig()
{
auto p = std::make_unique<Config>();
setupConfigForUnitTests(*p);
p->LEDGER_HISTORY = deleteInterval;
auto& section = p->section(ConfigSection::nodeDatabase());
section.set("online_delete", to_string(deleteInterval));
//section.set("age_threshold", "60");
return p;
}
static
std::unique_ptr<Config>
makeConfigAdvisory()
{
auto p = makeConfig();
auto& section = p->section(ConfigSection::nodeDatabase());
section.set("advisory_delete", "1");
return p;
}
bool goodLedger(jtx::Env& env, Json::Value const& json,
std::string ledgerID, bool checkDB = false)
{
auto good = json.isMember(jss::result)
&& !RPC::contains_error(json[jss::result])
&& json[jss::result][jss::ledger][jss::ledger_index] == ledgerID;
if (!good || !checkDB)
return good;
auto const seq = json[jss::result][jss::ledger_index].asUInt();
std::string outHash;
LedgerIndex outSeq;
std::string outParentHash;
std::string outDrops;
std::uint64_t outCloseTime;
std::uint64_t outParentCloseTime;
std::uint64_t outCloseTimeResolution;
std::uint64_t outCloseFlags;
std::string outAccountHash;
std::string outTxHash;
{
auto db = env.app().getLedgerDB().checkoutDb();
*db << "SELECT LedgerHash,LedgerSeq,PrevHash,TotalCoins, "
"ClosingTime,PrevClosingTime,CloseTimeRes,CloseFlags, "
"AccountSetHash,TransSetHash "
"FROM Ledgers "
"WHERE LedgerSeq = :seq",
soci::use(seq),
soci::into(outHash),
soci::into(outSeq),
soci::into(outParentHash),
soci::into(outDrops),
soci::into(outCloseTime),
soci::into(outParentCloseTime),
soci::into(outCloseTimeResolution),
soci::into(outCloseFlags),
soci::into(outAccountHash),
soci::into(outTxHash);
}
auto const& ledger = json[jss::result][jss::ledger];
return outHash == ledger[jss::hash].asString() &&
outSeq == seq &&
outParentHash == ledger[jss::parent_hash].asString() &&
outDrops == ledger[jss::total_coins].asString() &&
outCloseTime == ledger[jss::close_time].asUInt() &&
outParentCloseTime == ledger[jss::parent_close_time].asUInt() &&
outCloseTimeResolution == ledger[jss::close_time_resolution].asUInt() &&
outCloseFlags == ledger[jss::close_flags].asUInt() &&
outAccountHash == ledger[jss::account_hash].asString() &&
outTxHash == ledger[jss::transaction_hash].asString();
}
bool bad(Json::Value const& json, error_code_i error = rpcLGR_NOT_FOUND)
{
return json.isMember(jss::result)
&& RPC::contains_error(json[jss::result])
&& json[jss::result][jss::error_code] == error;
}
std::string getHash(Json::Value const& json)
{
expect(json.isMember(jss::result) &&
json[jss::result].isMember(jss::ledger) &&
json[jss::result][jss::ledger].isMember(jss::hash) &&
json[jss::result][jss::ledger][jss::hash].isString());
return json[jss::result][jss::ledger][jss::hash].asString();
}
void validationCheck(jtx::Env& env, int const expected)
{
auto db = env.app().getLedgerDB().checkoutDb();
int actual;
*db << "SELECT count(*) AS rows FROM Validations;",
soci::into(actual);
expect(actual == expected);
}
void ledgerCheck(jtx::Env& env, int const rows,
int const first)
{
auto db = env.app().getLedgerDB().checkoutDb();
int actualRows, actualFirst, actualLast;
*db << "SELECT count(*) AS rows, "
"min(LedgerSeq) as first, "
"max(LedgerSeq) as last "
"FROM Ledgers;",
soci::into(actualRows),
soci::into(actualFirst),
soci::into(actualLast);
expect(actualRows == rows);
expect(actualFirst == first);
expect(actualLast == first + rows - 1);
}
void transactionCheck(jtx::Env& env, int const rows)
{
auto db = env.app().getTxnDB().checkoutDb();
int actualRows;
*db << "SELECT count(*) AS rows "
"FROM Transactions;",
soci::into(actualRows);
expect(actualRows == rows);
}
void accountTransactionCheck(jtx::Env& env, int const rows)
{
auto db = env.app().getTxnDB().checkoutDb();
int actualRows;
*db << "SELECT count(*) AS rows "
"FROM AccountTransactions;",
soci::into(actualRows);
expect(actualRows == rows);
}
int waitForReady(jtx::Env& env)
{
using namespace std::chrono_literals;
auto& store = env.app().getSHAMapStore();
int ledgerSeq = 3;
while (!store.getLastRotated())
{
env.close();
std::this_thread::sleep_for(100ms);
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq++)));
}
return ledgerSeq;
}
public:
void testClear()
{
using namespace std::chrono_literals;
testcase("clearPrior");
using namespace jtx;
Env env(*this, makeConfig());
auto store = dynamic_cast<SHAMapStoreImp*>(
&env.app().getSHAMapStore());
expect(store);
env.fund(XRP(10000), noripple("alice"));
validationCheck(env, 0);
ledgerCheck(env, 1, 2);
transactionCheck(env, 0);
accountTransactionCheck(env, 0);
std::map<std::uint32_t, Json::Value const> ledgers;
auto ledgerTmp = env.rpc("ledger", "0");
expect(bad(ledgerTmp));
ledgers.emplace(std::make_pair(1, env.rpc("ledger", "1")));
expect(goodLedger(env, ledgers[1], "1"));
ledgers.emplace(std::make_pair(2, env.rpc("ledger", "2")));
expect(goodLedger(env, ledgers[2], "2"));
ledgerTmp = env.rpc("ledger", "current");
expect(goodLedger(env, ledgerTmp, "3"));
ledgerTmp = env.rpc("ledger", "4");
expect(bad(ledgerTmp));
ledgerTmp = env.rpc("ledger", "100");
expect(bad(ledgerTmp));
for (auto i = 4; i < deleteInterval + 4; ++i)
{
env.fund(XRP(10000), noripple("test" + to_string(i)));
env.close();
ledgerTmp = env.rpc("ledger", "current");
expect(goodLedger(env, ledgerTmp, to_string(i)));
}
assert(store->getLastRotated() == 3);
for (auto i = 3; i < deleteInterval + 3; ++i)
{
ledgers.emplace(std::make_pair(i,
env.rpc("ledger", to_string(i))));
expect(goodLedger(env, ledgers[i], to_string(i), true) &&
getHash(ledgers[i]).length());
}
validationCheck(env, 0);
ledgerCheck(env, deleteInterval + 1, 2);
transactionCheck(env, deleteInterval + 1);
accountTransactionCheck(env, 2*(deleteInterval + 1));
{
// Since standalone doesn't _do_ validations, manually
// insert some into the table. Create some with the
// hashes from our real ledgers, and some with fake
// hashes to represent validations that never ended up
// in a validated ledger.
char lh[65];
memset(lh, 'a', 64);
lh[64] = '\0';
std::vector<std::string> preSeqLedgerHashes({
lh
});
std::vector<std::string> badLedgerHashes;
std::vector<LedgerIndex> badLedgerSeqs;
std::vector<std::string> ledgerHashes;
std::vector<LedgerIndex> ledgerSeqs;
for (auto const& lgr : ledgers)
{
ledgerHashes.emplace_back(getHash(lgr.second));
ledgerSeqs.emplace_back(lgr.second[jss::result][jss::ledger_index].asUInt());
}
for (auto i = 0; i < 10; ++i)
{
++lh[30];
preSeqLedgerHashes.emplace_back(lh);
++lh[20];
badLedgerHashes.emplace_back(lh);
badLedgerSeqs.emplace_back(i + 1);
}
auto db = env.app().getLedgerDB().checkoutDb();
// Pre-migration validation - no sequence numbers.
*db << "INSERT INTO Validations "
"(LedgerHash) "
"VALUES "
"(:ledgerHash);",
soci::use(preSeqLedgerHashes);
// Post-migration orphan validation - InitalSeq,
// but no LedgerSeq
*db << "INSERT INTO Validations "
"(LedgerHash, InitialSeq) "
"VALUES "
"(:ledgerHash, :initialSeq);",
soci::use(badLedgerHashes),
soci::use(badLedgerSeqs);
// Post-migration validated ledger.
*db << "INSERT INTO Validations "
"(LedgerHash, LedgerSeq) "
"VALUES "
"(:ledgerHash, :ledgerSeq);",
soci::use(ledgerHashes),
soci::use(ledgerSeqs);
}
validationCheck(env, deleteInterval + 23);
ledgerCheck(env, deleteInterval + 1, 2);
transactionCheck(env, deleteInterval + 1);
accountTransactionCheck(env, 2 * (deleteInterval + 1));
{
// Closing one more ledger triggers a rotate
env.close();
auto ledger = env.rpc("ledger", "current");
expect(goodLedger(env, ledger, to_string(deleteInterval + 4)));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
expect(store->getLastRotated() == deleteInterval + 3);
auto const lastRotated = store->getLastRotated();
expect(lastRotated == 11, to_string(lastRotated));
// That took care of the fake hashes
validationCheck(env, deleteInterval + 8);
ledgerCheck(env, deleteInterval + 1, 3);
transactionCheck(env, deleteInterval + 1);
accountTransactionCheck(env, 2 * (deleteInterval + 1));
// The last iteration of this loop should trigger a rotate
for (auto i = lastRotated - 1; i < lastRotated + deleteInterval - 1; ++i)
{
validationCheck(env, deleteInterval + i + 1 - lastRotated + 8);
env.close();
ledgerTmp = env.rpc("ledger", "current");
expect(goodLedger(env, ledgerTmp, to_string(i + 3)));
ledgers.emplace(std::make_pair(i,
env.rpc("ledger", to_string(i))));
expect(store->getLastRotated() == lastRotated ||
i == lastRotated + deleteInterval - 2, "lastRotated");
expect(goodLedger(env, ledgers[i], to_string(i), true) &&
getHash(ledgers[i]).length(), to_string(ledgers[i]));
std::vector<std::string> ledgerHashes({
getHash(ledgers[i])
});
std::vector<LedgerIndex> ledgerSeqs({
ledgers[i][jss::result][jss::ledger_index].asUInt()
});
auto db = env.app().getLedgerDB().checkoutDb();
*db << "INSERT INTO Validations "
"(LedgerHash, LedgerSeq) "
"VALUES "
"(:ledgerHash, :ledgerSeq);",
soci::use(ledgerHashes),
soci::use(ledgerSeqs);
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
expect(store->getLastRotated() == deleteInterval + lastRotated);
validationCheck(env, deleteInterval - 1);
ledgerCheck(env, deleteInterval + 1, lastRotated);
transactionCheck(env, 0);
accountTransactionCheck(env, 0);
}
void testAutomatic()
{
testcase("automatic online_delete");
using namespace jtx;
using namespace std::chrono_literals;
Env env(*this, makeConfig());
auto store = dynamic_cast<SHAMapStoreImp*>(
&env.app().getSHAMapStore());
expect(store);
auto ledgerSeq = waitForReady(env);
auto lastRotated = ledgerSeq - 1;
expect(store->getLastRotated() == lastRotated,
to_string(store->getLastRotated()));
expect(lastRotated != 2);
// Because advisory_delete is unset,
// "can_delete" is disabled.
auto const canDelete = env.rpc("can_delete");
expect(bad(canDelete, rpcNOT_ENABLED));
// Close ledgers without triggering a rotate
for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
{
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
}
while(store->rotating())
std::this_thread::sleep_for(1ms);
// The database will always have back to ledger 2,
// regardless of lastRotated.
validationCheck(env, 0);
ledgerCheck(env, ledgerSeq - 2, 2);
expect(lastRotated == store->getLastRotated());
{
// Closing one more ledger triggers a rotate
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
validationCheck(env, 0);
ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
expect(lastRotated != store->getLastRotated());
lastRotated = store->getLastRotated();
// Close enough ledgers to trigger another rotate
for (; ledgerSeq < lastRotated + deleteInterval + 1; ++ledgerSeq)
{
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
validationCheck(env, 0);
ledgerCheck(env, deleteInterval + 1, lastRotated);
expect(lastRotated != store->getLastRotated());
}
void testCanDelete()
{
testcase("online_delete with advisory_delete");
using namespace jtx;
using namespace std::chrono_literals;
// Same config with advisory_delete enabled
Env env(*this, makeConfigAdvisory());
auto store = dynamic_cast<SHAMapStoreImp*>(
&env.app().getSHAMapStore());
expect(store);
auto ledgerSeq = waitForReady(env);
auto lastRotated = ledgerSeq - 1;
expect(store->getLastRotated() == lastRotated,
to_string(store->getLastRotated()));
expect(lastRotated != 2);
auto canDelete = env.rpc("can_delete");
expect(!RPC::contains_error(canDelete[jss::result]));
expect(canDelete[jss::result][jss::can_delete] == 0);
canDelete = env.rpc("can_delete", "never");
expect(!RPC::contains_error(canDelete[jss::result]));
expect(canDelete[jss::result][jss::can_delete] == 0);
auto const firstBatch = deleteInterval + ledgerSeq;
for (; ledgerSeq < firstBatch; ++ledgerSeq)
{
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
validationCheck(env, 0);
ledgerCheck(env, ledgerSeq - 2, 2);
expect(lastRotated == store->getLastRotated());
// This does not kick off a cleanup
canDelete = env.rpc("can_delete", to_string(
ledgerSeq + deleteInterval / 2));
expect(!RPC::contains_error(canDelete[jss::result]));
expect(canDelete[jss::result][jss::can_delete] ==
ledgerSeq + deleteInterval / 2);
while (store->rotating())
std::this_thread::sleep_for(1ms);
validationCheck(env, 0);
ledgerCheck(env, ledgerSeq - 2, 2);
expect(store->getLastRotated() == lastRotated);
{
// This kicks off a cleanup, but it stays small.
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
validationCheck(env, 0);
ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
expect(store->getLastRotated() == ledgerSeq - 1);
lastRotated = ledgerSeq - 1;
for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
{
// No cleanups in this loop.
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
expect(store->getLastRotated() == lastRotated);
{
// This kicks off another cleanup.
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
validationCheck(env, 0);
ledgerCheck(env, ledgerSeq - firstBatch, firstBatch);
expect(store->getLastRotated() == ledgerSeq - 1);
lastRotated = ledgerSeq - 1;
// This does not kick off a cleanup
canDelete = env.rpc("can_delete", "always");
expect(!RPC::contains_error(canDelete[jss::result]));
expect(canDelete[jss::result][jss::can_delete] ==
std::numeric_limits <unsigned int>::max());
for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
{
// No cleanups in this loop.
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
expect(store->getLastRotated() == lastRotated);
{
// This kicks off another cleanup.
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
validationCheck(env, 0);
ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
expect(store->getLastRotated() == ledgerSeq - 1);
lastRotated = ledgerSeq - 1;
// This does not kick off a cleanup
canDelete = env.rpc("can_delete", "now");
expect(!RPC::contains_error(canDelete[jss::result]));
expect(canDelete[jss::result][jss::can_delete] == ledgerSeq - 1);
for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
{
// No cleanups in this loop.
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
expect(store->getLastRotated() == lastRotated);
{
// This kicks off another cleanup.
env.close();
auto ledger = env.rpc("ledger", "validated");
expect(goodLedger(env, ledger, to_string(ledgerSeq++), true));
}
while (store->rotating())
std::this_thread::sleep_for(1ms);
validationCheck(env, 0);
ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
expect(store->getLastRotated() == ledgerSeq - 1);
lastRotated = ledgerSeq - 1;
}
void run()
{
testClear();
testAutomatic();
testCanDelete();
}
};
BEAST_DEFINE_TESTSUITE(SHAMapStore,app,ripple);
}
}

4
src/ripple/core/impl/DatabaseCon.cpp
View File

@@ -46,7 +46,9 @@ DatabaseCon::DatabaseCon (
{
try
{
session_ << initStrings[i];
soci::statement st = session_.prepare <<
initStrings[i];
st.execute(true);
}
catch (soci::soci_error&)
{

9
src/ripple/rpc/handlers/CanDelete.cpp
View File

@@ -70,10 +70,11 @@ Json::Value doCanDelete (RPC::Context& context)
{
canDeleteSeq = context.app.getSHAMapStore().getLastRotated();
if (!canDeleteSeq)
return RPC::make_error (rpcNOT_READY); }
else if (canDeleteStr.size() == 64 &&
canDeleteStr.find_first_not_of("0123456789abcdef") ==
std::string::npos)
return RPC::make_error (rpcNOT_READY);
}
else if (canDeleteStr.size() == 64 &&
canDeleteStr.find_first_not_of("0123456789abcdef") ==
std::string::npos)
{
auto ledger = context.ledgerMaster.getLedgerByHash (
from_hex_text<uint256>(canDeleteStr));

1
src/ripple/unity/app_tests.cpp
View File

@@ -29,6 +29,7 @@
#include <ripple/app/tests/Offer.test.cpp>
#include <ripple/app/tests/Path_test.cpp>
#include <ripple/app/tests/Regression_test.cpp>
#include <ripple/app/tests/SHAMapStore_test.cpp>
#include <ripple/app/tests/SusPay_test.cpp>
#include <ripple/app/tests/SetAuth_test.cpp>
#include <ripple/app/tests/OversizeMeta_test.cpp>