From 5cc9314d03a641f6ea63d809b7ccc2d8e2fb524c Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 1 Apr 2013 19:03:28 -0700 Subject: [PATCH] Add a framework for detecting databases that need updating. Put transaction from account_tx command in transaction application sequence. CAUTION: This modifies your databases and will cause a delay on your next startup. --- newcoin.vcxproj | 1 + newcoin.vcxproj.filters | 3 + ripple2010.vcxproj | 1 + ripple2010.vcxproj.filters | 3 + src/cpp/ripple/AcceptedLedger.h | 1 + src/cpp/ripple/Application.cpp | 3 + src/cpp/ripple/Application.h | 1 + src/cpp/ripple/DBInit.cpp | 3 +- src/cpp/ripple/Ledger.cpp | 4 +- src/cpp/ripple/LedgerMaster.cpp | 2 +- src/cpp/ripple/NetworkOPs.cpp | 4 +- src/cpp/ripple/UpdateTables.cpp | 108 ++++++++++++++++++++++++++++++++ 12 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 src/cpp/ripple/UpdateTables.cpp diff --git a/newcoin.vcxproj b/newcoin.vcxproj index 07ca73e105..cd9990a880 100644 --- a/newcoin.vcxproj +++ b/newcoin.vcxproj @@ -186,6 +186,7 @@ + diff --git a/newcoin.vcxproj.filters b/newcoin.vcxproj.filters index 483cb0c2d0..a4f8be012f 100644 --- a/newcoin.vcxproj.filters +++ b/newcoin.vcxproj.filters @@ -270,6 +270,9 @@ Source Files + + Source Files + Source Files diff --git a/ripple2010.vcxproj b/ripple2010.vcxproj index d3e9aff5c2..e044141116 100644 --- a/ripple2010.vcxproj +++ b/ripple2010.vcxproj @@ -185,6 +185,7 @@ + diff --git a/ripple2010.vcxproj.filters b/ripple2010.vcxproj.filters index bbdf822554..14f56c6328 100644 --- a/ripple2010.vcxproj.filters +++ b/ripple2010.vcxproj.filters @@ -267,6 +267,9 @@ Source Files + + Source Files + Source Files diff --git a/src/cpp/ripple/AcceptedLedger.h b/src/cpp/ripple/AcceptedLedger.h index dcc662b959..04f917a788 100644 --- a/src/cpp/ripple/AcceptedLedger.h +++ b/src/cpp/ripple/AcceptedLedger.h @@ -31,6 +31,7 @@ public: uint256 getTransactionID() const { return mTxn->getTransactionID(); } TransactionType getTxnType() const { return mTxn->getTxnType(); } TER getResult() const { return mResult; } + uint32 getTxnSeq() const { return mMeta->getIndex(); } bool isApplied() const { return !!mMeta; } int getIndex() const { return mMeta ? mMeta->getIndex() : 0; } diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index a9cef25392..a82d08cdf9 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -143,6 +143,9 @@ void Application::setup() mLedgerDB->getDB()->setupCheckpointing(&mJobQueue); mHashNodeDB->getDB()->setupCheckpointing(&mJobQueue); + if (!theConfig.RUN_STANDALONE) + updateTables(); + if (theConfig.START_UP == Config::FRESH) { cLog(lsINFO) << "Starting new Ledger"; diff --git a/src/cpp/ripple/Application.h b/src/cpp/ripple/Application.h index a467202665..ad09c620d7 100644 --- a/src/cpp/ripple/Application.h +++ b/src/cpp/ripple/Application.h @@ -90,6 +90,7 @@ class Application volatile bool mShutdown; + void updateTables(); void startNewLedger(); void loadOldLedger(const std::string&); diff --git a/src/cpp/ripple/DBInit.cpp b/src/cpp/ripple/DBInit.cpp index 6df0611fa5..1ec5554591 100644 --- a/src/cpp/ripple/DBInit.cpp +++ b/src/cpp/ripple/DBInit.cpp @@ -26,11 +26,12 @@ const char *TxnDBInit[] = { TransID CHARACTER(64), \ Account CHARACTER(64), \ LedgerSeq BIGINT UNSIGNED \ + TxnSeq INTEGER \ );", "CREATE INDEX AcctTxIDIndex ON \ AccountTransactions(TransID);", "CREATE INDEX AcctTxIndex ON \ - AccountTransactions(Account, LedgerSeq, TransID);", + AccountTransactions(Account, LedgerSeq, TxnSeq, TransID);", "CREATE INDEX AcctLgrIndex ON \ AccountTransactions(LedgerSeq, Account, TransID);", diff --git a/src/cpp/ripple/Ledger.cpp b/src/cpp/ripple/Ledger.cpp index dea0b4b882..a05876a1c0 100644 --- a/src/cpp/ripple/Ledger.cpp +++ b/src/cpp/ripple/Ledger.cpp @@ -479,7 +479,7 @@ void Ledger::saveAcceptedLedger(Job&, bool fromConsensus) const std::vector& accts = vt.second.getAffected(); if (!accts.empty()) { - std::string sql = "INSERT INTO AccountTransactions (TransID, Account, LedgerSeq) VALUES "; + std::string sql = "INSERT INTO AccountTransactions (TransID, Account, LedgerSeq, TxnSeq) VALUES "; bool first = true; for (std::vector::const_iterator it = accts.begin(), end = accts.end(); it != end; ++it) { @@ -495,6 +495,8 @@ void Ledger::saveAcceptedLedger(Job&, bool fromConsensus) sql += it->humanAccountID(); sql += "',"; sql += boost::lexical_cast(getLedgerSeq()); + sql += ","; + sql += boost::lexical_cast(vt.second.getTxnSeq()); sql += ")"; } sql += ";"; diff --git a/src/cpp/ripple/LedgerMaster.cpp b/src/cpp/ripple/LedgerMaster.cpp index 67381e195e..7fa42d7d35 100644 --- a/src/cpp/ripple/LedgerMaster.cpp +++ b/src/cpp/ripple/LedgerMaster.cpp @@ -106,7 +106,7 @@ Ledger::pointer LedgerMaster::closeLedger(bool recover) ++recovers; } catch (...) - { + { // CHECKME: We got a few of these cLog(lsWARNING) << "Held transaction throws"; } } diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index 4ad0042228..7415a605b3 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -1093,13 +1093,15 @@ std::string boost::str(boost::format("SELECT %s FROM " "AccountTransactions INNER JOIN Transactions ON Transactions.TransID = AccountTransactions.TransID " "WHERE Account = '%s' %s %s " - "ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TransID %s LIMIT %u, %u;") + "ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s, AccountTransactions.TransID %s " + "LIMIT %u, %u;") % selection % account.humanAccountID() % maxClause % minClause % (descending ? "DESC" : "ASC") % (descending ? "DESC" : "ASC") + % (descending ? "DESC" : "ASC") % boost::lexical_cast(offset) % boost::lexical_cast(numberOfResults) ); diff --git a/src/cpp/ripple/UpdateTables.cpp b/src/cpp/ripple/UpdateTables.cpp new file mode 100644 index 0000000000..b5302b09e1 --- /dev/null +++ b/src/cpp/ripple/UpdateTables.cpp @@ -0,0 +1,108 @@ + +#include "Application.h" +#include "Log.h" + +static std::vector getSchema(DatabaseCon* dbc, const std::string& dbName) +{ + std::vector schema; + + std::string sql = "SELECT sql FROM sqlite_master WHERE tbl_name='"; + sql += dbName; + sql += "';"; + + SQL_FOREACH(dbc->getDB(), sql) + { + dbc->getDB()->getStr("sql", sql); + schema.push_back(sql); + } + + return schema; +} + +static bool schemaHas(DatabaseCon* dbc, const std::string& dbName, int line, const std::string& content) +{ + std::vector schema = getSchema(dbc, dbName); + if (schema.size() <= line) + { + Log(lsFATAL) << "Schema for " << dbName << " has too few lines"; + throw std::runtime_error("bad schema"); + } + return schema[line].find(content) != std::string::npos; +} + +static void addTxnSeqField() +{ + if (schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "TxnSeq")) + return; + Log(lsWARNING) << "Transaction sequence field is missing"; + + Database* db = theApp->getTxnDB()->getDB(); + + std::vector< std::pair > txIDs; + txIDs.reserve(300000); + + Log(lsINFO) << "Parsing transactions"; + int i = 0; + uint256 transID; + SQL_FOREACH(db, "SELECT TransID,TxnMeta FROM Transactions;") + { + std::vector rawMeta; + int metaSize = 2048; + rawMeta.resize(metaSize); + metaSize = db->getBinary("TxnMeta", &*rawMeta.begin(), rawMeta.size()); + if (metaSize > rawMeta.size()) + { + rawMeta.resize(metaSize); + db->getBinary("TxnMeta", &*rawMeta.begin(), rawMeta.size()); + } + else rawMeta.resize(metaSize); + + std::string tid; + db->getStr("TransID", tid); + transID.SetHex(tid, true); + + if (rawMeta.size() == 0) + { + txIDs.push_back(std::make_pair(transID, -1)); + Log(lsINFO) << "No metadata for " << transID; + } + else + { + TransactionMetaSet m(transID, 0, rawMeta); + txIDs.push_back(std::make_pair(transID, m.getIndex())); + } + + if ((++i % 1000) == 0) + Log(lsINFO) << i << " transactions read"; + } + + Log(lsINFO) << "All " << i << " transactions read"; + + db->executeSQL("BEGIN TRANSACTION;"); + + Log(lsINFO) << "Dropping old index"; + db->executeSQL("DROP INDEX AcctTxIndex;"); + + Log(lsINFO) << "Altering table"; + db->executeSQL("ALTER TABLE AccountTransactions ADD COLUMN TxnSeq INTEGER;"); + + typedef std::pair u256_int_pair_t; + boost::format fmt("UPDATE AccountTransactions SET TxnSeq = %d WHERE TransID = '%s';"); + i = 0; + BOOST_FOREACH(u256_int_pair_t& t, txIDs) + { + db->executeSQL(boost::str(fmt % t.second % t.first.GetHex())); + if ((++i % 1000) == 0) + Log(lsINFO) << i << " transactions updated"; + } + + db->executeSQL("CREATE INDEX AcctTxIndex ON AccountTransactions(Account, LedgerSeq, TxnSeq, TransID);"); + db->executeSQL("END TRANSACTION;"); +} + +void Application::updateTables() +{ // perform any needed table updates + assert(schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "TransID")); + assert(!schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "foobar")); + addTxnSeqField(); +}